From 792f961fb96bb8533e540970c54f43b958d77296 Mon Sep 17 00:00:00 2001 From: Javier Sancho Date: Tue, 28 Jan 2014 16:52:02 +0100 Subject: [PATCH] Separate classes and functionality into various files --- MySQL.py | 4 +- __init__.py | 22 ---- collection.py | 97 +++++++++++++++++ connection.py | 85 +++++++++++++++ cursor.py | 122 +++++++++++++++++++++ database.py | 45 ++++++++ mojo.py | 286 -------------------------------------------------- 7 files changed, 351 insertions(+), 310 deletions(-) delete mode 100644 __init__.py create mode 100644 collection.py create mode 100644 connection.py create mode 100644 cursor.py create mode 100644 database.py delete mode 100644 mojo.py diff --git a/MySQL.py b/MySQL.py index 39c3435..d4604d3 100644 --- a/MySQL.py +++ b/MySQL.py @@ -19,7 +19,7 @@ # ############################################################################## -import mojo +import connection import MySQLdb SQL_FIELD_TYPES = { @@ -28,7 +28,7 @@ SQL_FIELD_TYPES = { 'float': 'DOUBLE', } -class Connection(mojo.Connection): +class Connection(connection.Connection): def __init__(self, *args, **kwargs): self._db_con = MySQLdb.connect(*args, **kwargs) self._db_con_autocommit = MySQLdb.connect(*args, **kwargs) diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 36efb41..0000000 --- a/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# mojo, a Python library for implementing data flows in MongoDB -# Copyright (C) 2013 by Javier Sancho Fernandez -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -############################################################################## - -from mojo import Connection diff --git a/collection.py b/collection.py new file mode 100644 index 0000000..664961a --- /dev/null +++ b/collection.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# mojo, a Python library for implementing document based databases +# Copyright (C) 2013-2014 by Javier Sancho Fernandez +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## + +import cPickle +from cursor import Cursor +import uuid + +class Collection(object): + def __init__(self, database, table_name): + self.database = database + self.table_name = unicode(table_name) + + def __repr__(self): + return "Collection(%r, %r)" % (self.database, self.table_name) + + def exists(self): + return (self.database.exists() and self.table_name in self.database.collection_names()) + + def _create_table(self): + fields = [ + {'name': 'id', 'type': 'char', 'size': 32, 'primary': True}, + ] + return self.database.connection._create_table(self.database.db_name, '%s$_id' % self.table_name, fields) + + def _create_field(self, field_name): + fields = [ + {'name': 'id', 'type': 'char', 'size': 32, 'primary': True}, + {'name': 'value', 'type': 'text', 'null': False}, + {'name': 'number', 'type': 'float'}, + ] + return self.database.connection._create_table(self.database.db_name, '%s$%s' % (self.table_name, field_name), fields) + + def _get_fields(self): + tables = self.database.connection._get_tables(self.database.db_name) + return [unicode(x[x.find('$')+1:]) for x in filter(lambda x: x.startswith('%s$' % self.table_name), tables)] + + def count(self): + return self.database.connection._count(self.database.db_name, self.table_name) + + def find(self, *args, **kwargs): + return Cursor(self, *args, **kwargs) + + def insert(self, doc_or_docs): + if not self.database.db_name in self.database.connection.database_names(): + self.database._create_database() + if not self.table_name in self.database.collection_names(): + self._create_table() + + if not type(doc_or_docs) in (list, tuple): + docs = [doc_or_docs] + else: + docs = doc_or_docs + for doc in docs: + if not '_id' in doc: + doc['_id'] = uuid.uuid4().hex + self._insert_document(doc) + + if type(doc_or_docs) in (list, tuple): + return [d['_id'] for d in docs] + else: + return docs[0]['_id'] + + def _insert_document(self, doc): + table_id = '%s$_id' % self.table_name + fields = self._get_fields() + self.database.connection._insert(self.database.db_name, table_id, {'id': doc['_id']}) + for f in doc: + if f == '_id': + continue + if not f in fields: + self._create_field(f) + table_f = '%s$%s' % (self.table_name, f) + values = { + 'id': doc['_id'], + 'value': cPickle.dumps(doc[f]), + } + if type(doc[f]) in (int, float): + values['number'] = doc[f] + self.database.connection._insert(self.database.db_name, table_f, values) diff --git a/connection.py b/connection.py new file mode 100644 index 0000000..ca67b9e --- /dev/null +++ b/connection.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# mojo, a Python library for implementing document based databases +# Copyright (C) 2013-2014 by Javier Sancho Fernandez +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## + +from database import Database + +class Connection(object): + def __init__(self, *args, **kwargs): + self._db_con = None + + def __getattr__(self, db_name): + return Database(self, db_name) + + def __getitem__(self, *args, **kwargs): + return self.__getattr__(*args, **kwargs) + + def __repr__(self): + return "Connection(%s)" % self._db_con + + def _get_databases(self): + return [] + + def database_names(self): + try: + return [unicode(x) for x in self._get_databases()] + except: + return [] + + def _get_tables(self, db_name): + return [] + + def collection_names(self, db_name): + try: + return list(set([unicode(x.split('$')[0]) for x in filter(lambda x: '$' in x, self._get_tables(db_name))])) + except: + return [] + + def _count_rows(self, db_name, table_name): + return 0 + + def _count(self, db_name, table_name): + try: + return self._count_rows(db_name, table_name + '$_id') + except: + return 0 + + def _create_database(self, db_name): + return None + + def _create_table(self, db_name, table_name, fields): + # [{'name': 'id', 'type': 'char', 'size': 20, 'primary': True}] + return None + + def _get_cursor(self, db_name, query): + # {'select': [('t1$_id', 'id'), {'select': [('t1$c1', 'value')], 'from': ['t1$c1'], 'where': [(('t1$c1', 'id'), '=', ('t1$_id', 'id'))]}], 'from': ['t1$_id']} + return None + + def _next(self, cursor): + return None + + def _insert(self, db_name, table_name, values): + return None + + def commit(self): + pass + + def rollback(self): + pass diff --git a/cursor.py b/cursor.py new file mode 100644 index 0000000..73a0c13 --- /dev/null +++ b/cursor.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# mojo, a Python library for implementing document based databases +# Copyright (C) 2013-2014 by Javier Sancho Fernandez +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## + +import cPickle + +class Cursor(object): + def __init__(self, collection, spec=None, fields=None, **kwargs): + if spec and not type(spec) is dict: + raise Exception("spec must be an instance of dict") + + self.collection = collection + self.spec = spec + if self.collection.exists(): + self.fields = self._get_fields(fields) + self.cursor = self._get_cursor() + else: + self.fields = None + self.cursor = None + + def __iter__(self): + return self + + def _get_fields(self, fields): + set_all_fields = set(self.collection._get_fields()) + if fields is None: + res_fields = list(set_all_fields) + elif type(fields) is dict: + fields_without_id = filter(lambda x: x[0] != '_id', fields.iteritems()) + if fields_without_id[0][1]: + first = True + res_fields = set() + else: + first = False + res_fields = set(set_all_fields) + for f in fields_without_id: + if f[1] and f[0] in set_all_fields: + if first: + res_fields.add(f[0]) + else: + raise Exception("You cannot currently mix including and excluding fields. Contact us if this is an issue.") + elif not f[1]: + if not first: + res_fields.discard(f[0]) + else: + raise Exception("You cannot currently mix including and excluding fields. Contact us if this is an issue.") + if '_id' in fields and not fields['_id']: + res_fields.discard('_id') + else: + res_fields.add('_id') + res_fields = list(res_fields) + else: + set_fields = set(list(fields)) + set_fields.add('_id') + res_fields = list(set_all_fields.intersection(set_fields)) + + return res_fields + + def _get_cursor(self): + query = {} + table_id = '%s$_id' % self.collection.table_name + + query['select'] = [(table_id, 'id')] + for f in filter(lambda x: x != '_id', self.fields): + table_f = '%s$%s' % (self.collection.table_name, f) + q = self._get_cursor_field(table_id, table_f) + query['select'].append(q) + + query['from'] = [table_id] + + if self.spec: + query['where'] = [] + for k, v in self.spec.iteritems(): + table_f = '%s$%s' % (self.collection.table_name, k) + field_q = self._get_cursor_field(table_id, table_f) + query['where'].append((field_q, '=', v)) + + return self.collection.database.connection._get_cursor(self.collection.database.db_name, query) + + def _get_cursor_field(self, table_id, table_field): + return { + 'select': [(table_field, 'value')], + 'from': [table_field], + 'where': [((table_field, 'id'), '=', (table_id, 'id'))], + } + + def next(self): + if self.cursor is None: + raise StopIteration + + if self.cursor: + res = self.collection.database.connection._next(self.cursor) + if res is None: + raise StopIteration + else: + document = {} + if '_id' in self.fields: + document['_id'] = res[0] + fields_without_id = filter(lambda x: x != '_id', self.fields) + for i in xrange(len(fields_without_id)): + if not res[i + 1] is None: + document[fields_without_id[i]] = cPickle.loads(res[i + 1]) + return document + else: + return None diff --git a/database.py b/database.py new file mode 100644 index 0000000..1b378bf --- /dev/null +++ b/database.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# mojo, a Python library for implementing document based databases +# Copyright (C) 2013-2014 by Javier Sancho Fernandez +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## + +from collection import Collection + +class Database(object): + def __init__(self, connection, db_name): + self.connection = connection + self.db_name = unicode(db_name) + + def __getattr__(self, table_name): + return Collection(self, table_name) + + def __getitem__(self, *args, **kwargs): + return self.__getattr__(*args, **kwargs) + + def __repr__(self): + return "Database(%r, %r)" % (self.connection, self.db_name) + + def _create_database(self): + return self.connection._create_database(self.db_name) + + def exists(self): + return (self.db_name in self.connection.database_names()) + + def collection_names(self): + return self.connection.collection_names(self.db_name) diff --git a/mojo.py b/mojo.py deleted file mode 100644 index f3aa744..0000000 --- a/mojo.py +++ /dev/null @@ -1,286 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# mojo, a Python library for implementing document based databases -# Copyright (C) 2013-2014 by Javier Sancho Fernandez -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -############################################################################## - -import cPickle -import uuid - - -class Connection(object): - def __init__(self, *args, **kwargs): - self._db_con = None - - def __getattr__(self, db_name): - return Database(self, db_name) - - def __getitem__(self, *args, **kwargs): - return self.__getattr__(*args, **kwargs) - - def __repr__(self): - return "Connection(%s)" % self._db_con - - def _get_databases(self): - return [] - - def database_names(self): - try: - return [unicode(x) for x in self._get_databases()] - except: - return [] - - def _get_tables(self, db_name): - return [] - - def collection_names(self, db_name): - try: - return list(set([unicode(x.split('$')[0]) for x in filter(lambda x: '$' in x, self._get_tables(db_name))])) - except: - return [] - - def _count_rows(self, db_name, table_name): - return 0 - - def _count(self, db_name, table_name): - try: - return self._count_rows(db_name, table_name + '$_id') - except: - return 0 - - def _create_database(self, db_name): - return None - - def _create_table(self, db_name, table_name, fields): - # [{'name': 'id', 'type': 'char', 'size': 20, 'primary': True}] - return None - - def _get_cursor(self, db_name, query): - # {'select': [('t1$_id', 'id'), {'select': [('t1$c1', 'value')], 'from': ['t1$c1'], 'where': [(('t1$c1', 'id'), '=', ('t1$_id', 'id'))]}], 'from': ['t1$_id']} - return None - - def _next(self, cursor): - return None - - def _insert(self, db_name, table_name, values): - return None - - def commit(self): - pass - - def rollback(self): - pass - - -class Database(object): - def __init__(self, connection, db_name): - self.connection = connection - self.db_name = unicode(db_name) - - def __getattr__(self, table_name): - return Collection(self, table_name) - - def __getitem__(self, *args, **kwargs): - return self.__getattr__(*args, **kwargs) - - def __repr__(self): - return "Database(%r, %r)" % (self.connection, self.db_name) - - def _create_database(self): - return self.connection._create_database(self.db_name) - - def exists(self): - return (self.db_name in self.connection.database_names()) - - def collection_names(self): - return self.connection.collection_names(self.db_name) - - -class Collection(object): - def __init__(self, database, table_name): - self.database = database - self.table_name = unicode(table_name) - - def __repr__(self): - return "Collection(%r, %r)" % (self.database, self.table_name) - - def exists(self): - return (self.database.exists() and self.table_name in self.database.collection_names()) - - def _create_table(self): - fields = [ - {'name': 'id', 'type': 'char', 'size': 32, 'primary': True}, - ] - return self.database.connection._create_table(self.database.db_name, '%s$_id' % self.table_name, fields) - - def _create_field(self, field_name): - fields = [ - {'name': 'id', 'type': 'char', 'size': 32, 'primary': True}, - {'name': 'value', 'type': 'text', 'null': False}, - {'name': 'number', 'type': 'float'}, - ] - return self.database.connection._create_table(self.database.db_name, '%s$%s' % (self.table_name, field_name), fields) - - def _get_fields(self): - tables = self.database.connection._get_tables(self.database.db_name) - return [unicode(x[x.find('$')+1:]) for x in filter(lambda x: x.startswith('%s$' % self.table_name), tables)] - - def count(self): - return self.database.connection._count(self.database.db_name, self.table_name) - - def find(self, *args, **kwargs): - return Cursor(self, *args, **kwargs) - - def insert(self, doc_or_docs): - if not self.database.db_name in self.database.connection.database_names(): - self.database._create_database() - if not self.table_name in self.database.collection_names(): - self._create_table() - - if not type(doc_or_docs) in (list, tuple): - docs = [doc_or_docs] - else: - docs = doc_or_docs - for doc in docs: - if not '_id' in doc: - doc['_id'] = uuid.uuid4().hex - self._insert_document(doc) - - if type(doc_or_docs) in (list, tuple): - return [d['_id'] for d in docs] - else: - return docs[0]['_id'] - - def _insert_document(self, doc): - table_id = '%s$_id' % self.table_name - fields = self._get_fields() - self.database.connection._insert(self.database.db_name, table_id, {'id': doc['_id']}) - for f in doc: - if f == '_id': - continue - if not f in fields: - self._create_field(f) - table_f = '%s$%s' % (self.table_name, f) - values = { - 'id': doc['_id'], - 'value': cPickle.dumps(doc[f]), - } - if type(doc[f]) in (int, float): - values['number'] = doc[f] - self.database.connection._insert(self.database.db_name, table_f, values) - - -class Cursor(object): - def __init__(self, collection, spec=None, fields=None, **kwargs): - if spec and not type(spec) is dict: - raise Exception("spec must be an instance of dict") - - self.collection = collection - self.spec = spec - if self.collection.exists(): - self.fields = self._get_fields(fields) - self.cursor = self._get_cursor() - else: - self.fields = None - self.cursor = None - - def __iter__(self): - return self - - def _get_fields(self, fields): - set_all_fields = set(self.collection._get_fields()) - if fields is None: - res_fields = list(set_all_fields) - elif type(fields) is dict: - fields_without_id = filter(lambda x: x[0] != '_id', fields.iteritems()) - if fields_without_id[0][1]: - first = True - res_fields = set() - else: - first = False - res_fields = set(set_all_fields) - for f in fields_without_id: - if f[1] and f[0] in set_all_fields: - if first: - res_fields.add(f[0]) - else: - raise Exception("You cannot currently mix including and excluding fields. Contact us if this is an issue.") - elif not f[1]: - if not first: - res_fields.discard(f[0]) - else: - raise Exception("You cannot currently mix including and excluding fields. Contact us if this is an issue.") - if '_id' in fields and not fields['_id']: - res_fields.discard('_id') - else: - res_fields.add('_id') - res_fields = list(res_fields) - else: - set_fields = set(list(fields)) - set_fields.add('_id') - res_fields = list(set_all_fields.intersection(set_fields)) - - return res_fields - - def _get_cursor(self): - query = {} - table_id = '%s$_id' % self.collection.table_name - - query['select'] = [(table_id, 'id')] - for f in filter(lambda x: x != '_id', self.fields): - table_f = '%s$%s' % (self.collection.table_name, f) - q = self._get_cursor_field(table_id, table_f) - query['select'].append(q) - - query['from'] = [table_id] - - if self.spec: - query['where'] = [] - for k, v in self.spec.iteritems(): - table_f = '%s$%s' % (self.collection.table_name, k) - field_q = self._get_cursor_field(table_id, table_f) - query['where'].append((field_q, '=', v)) - - return self.collection.database.connection._get_cursor(self.collection.database.db_name, query) - - def _get_cursor_field(self, table_id, table_field): - return { - 'select': [(table_field, 'value')], - 'from': [table_field], - 'where': [((table_field, 'id'), '=', (table_id, 'id'))], - } - - def next(self): - if self.cursor is None: - raise StopIteration - - if self.cursor: - res = self.collection.database.connection._next(self.cursor) - if res is None: - raise StopIteration - else: - document = {} - if '_id' in self.fields: - document['_id'] = res[0] - fields_without_id = filter(lambda x: x != '_id', self.fields) - for i in xrange(len(fields_without_id)): - if not res[i + 1] is None: - document[fields_without_id[i]] = cPickle.loads(res[i + 1]) - return document - else: - return None -- 2.39.2