Merge pull request web2py#348 from gi0baro/master

Optimizations in parsing

Author: gi0baro

pydal/adapters/base.py

@@ -1,13 +1,14 @@
 import copy
 import sys
 import types
+from collections import defaultdict
 from .._compat import PY2, with_metaclass, iterkeys, iteritems, hashlib_md5, \
     integer_types
 from .._globals import IDENTITY
 from ..connection import ConnectionPool
 from ..exceptions import NotOnNOSQLError
 from ..helpers.classes import Reference, ExecutionHandler, SQLCustomType, \
-    SQLALL, NullDriver, BasicStorage
+    SQLALL, NullDriver
 from ..helpers.methods import use_common_filters, xorify
 from ..helpers.regex import REGEX_SELECT_AS_PARSER
 from ..migrator import Migrator
@@ -210,8 +211,7 @@ def _regex_select_as_parser(self, colname):
 
     def _parse(self, row, tmps, fields, colnames, blob_decode,
                cacheable, fields_virtual, fields_lazy):
-        new_row = self.db.Row(dict(
-            (tablename, self.db.Row()) for tablename in fields_virtual.keys()))
+        new_row = defaultdict(self.db.Row)
         extras = self.db.Row()
         #: let's loop over columns
         for (j, colname) in enumerate(colnames):
@@ -221,10 +221,7 @@ def _parse(self, row, tmps, fields, colnames, blob_decode,
             #: do we have a real column?
             if tmp:
                 (tablename, fieldname, table, field, ft) = tmp
-                #: add table to row object if needed
-                colset = new_row.get(tablename, None)
-                if colset is None:
-                    colset = new_row[tablename] = self.db.Row()
+                colset = new_row[tablename]
                 #: parse value
                 value = self.parse_value(value, ft, blob_decode)
                 if field.filter_out:
@@ -246,11 +243,12 @@ def _parse(self, row, tmps, fields, colnames, blob_decode,
                 new_column_name = self._regex_select_as_parser(colname)
                 if new_column_name is not None:
                     column_name = new_column_name.groups(0)
-                    setattr(new_row, column_name[0], value)
+                    new_row[column_name[0]] = value
         #: add extras if needed (eg. operations results)
         if extras:
             new_row['_extra'] = extras
         #: add virtuals
+        new_row = self.db.Row(**new_row)
         for tablename in fields_virtual.keys():
             for f, v in fields_virtual[tablename]:
                 try:

pydal/parsers/__init__.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from .._compat import with_metaclass, iteritems
 from .._gae import gae
 from ..helpers._internals import Dispatcher
@@ -82,7 +83,7 @@ def __init__(self, adapter):
         self._before_registry_ = {}
         for name, obj in iteritems(self._declared_before_):
             self._before_registry_[obj.field_type] = obj.f
-        self.registered = {}
+        self.registered = defaultdict(lambda self=self: self._default)
         for name, obj in iteritems(self._declared_parsers_):
             if obj.field_type in self._before_registry_:
                 self.registered[obj.field_type] = ParserMethodWrapper(
@@ -98,7 +99,7 @@ def _default(self, value, field_type):
 
     def get_parser(self, field_type):
         key = REGEX_TYPE.match(field_type).group(0)
-        return self.registered.get(key, self._default)
+        return self.registered[key]
 
     def parse(self, value, field_type):
         return self.get_parser(field_type)(value, field_type)

pydal/parsers/base.py

@@ -62,14 +62,16 @@ def _bigint(self, value):
         return self.registered['integer'](value, 'bigint')
 
 
-class DateTimeParser(Parser):
+class DateParser(Parser):
     @for_type('date')
     def _date(self, value):
         if isinstance(value, datetime):
             return value.date()
         (y, m, d) = map(int, str(value)[:10].strip().split('-'))
         return date(y, m, d)
 
+
+class TimeParser(Parser):
     @for_type('time')
     def _time(self, value):
         if isinstance(value, datetime):
@@ -81,18 +83,20 @@ def _time(self, value):
             (h, mi, s) = time_items + [0]
         return time(h, mi, s)
 
+
+class DateTimeParser(Parser):
     @for_type('datetime')
     def _datetime(self, value):
         value = str(value)
         date_part, time_part, timezone = value[:10], value[11:19], value[19:]
         if '+' in timezone:
             ms, tz = timezone.split('+')
             h, m = tz.split(':')
-            dt = timedelta(seconds=3600*int(h)+60*int(m))
+            dt = timedelta(seconds=3600 * int(h) + 60 * int(m))
         elif '-' in timezone:
             ms, tz = timezone.split('-')
             h, m = tz.split(':')
-            dt = -timedelta(seconds=3600*int(h)+60*int(m))
+            dt = -timedelta(seconds=3600 * int(h) + 60 * int(m))
         else:
             ms = timezone.upper().split('Z')[0]
             dt = None
@@ -146,5 +150,8 @@ def _list_references(self, value, field_type):
 
 
 @parsers.register_for(SQLAdapter)
-class Commonparser(ListsParser, DateTimeParser, DecimalParser, JSONParser):
+class Commonparser(
+    ListsParser, DateParser, TimeParser, DateTimeParser, DecimalParser,
+    JSONParser
+):
     pass

pydal/parsers/sqlite.py

@@ -1,11 +1,11 @@
 from decimal import Decimal
 from ..adapters.sqlite import SQLite
-from .base import ListsParser, DateTimeParser, JSONParser
+from .base import ListsParser, TimeParser, JSONParser
 from . import parsers, for_type, before_parse
 
 
 @parsers.register_for(SQLite)
-class SQLiteParser(ListsParser, DateTimeParser, JSONParser):
+class SQLiteParser(ListsParser, TimeParser, JSONParser):
     @before_parse('decimal')
     def decimal_extras(self, field_type):
         return {'decimals': field_type[8:-1].split(',')[-1]}

pydal/representers/__init__.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from .._compat import PY2, with_metaclass, iteritems, to_unicode
 from .._gae import gae
 from ..helpers._internals import Dispatcher
@@ -182,7 +183,7 @@ def __init__(self, adapter):
         self._tbefore_registry_ = {}
         for name, obj in iteritems(self._declared_tbefore_):
             self._tbefore_registry_[obj.field_type] = obj.f
-        self.registered_t = {}
+        self.registered_t = defaultdict(lambda self=self: self._default)
         for name, obj in iteritems(self._declared_trepresenters_):
             if obj.field_type in self._tbefore_registry_:
                 self.registered_t[obj.field_type] = TReprMethodWrapper(
@@ -217,7 +218,7 @@ def get_representer_for_instance(self, value):
 
     def get_representer_for_type(self, field_type):
         key = REGEX_TYPE.match(field_type).group(0)
-        return self.registered_t.get(key, self._default)
+        return self.registered_t[key]
 
     def adapt(self, value):
         if PY2: