fix #2: try inserting one by one if exception occurs in django

Author: chfw

pyexcel_io/base.py

@@ -275,3 +275,12 @@ def from_query_sets(column_names, query_sets):
 
 def is_empty_array(array):
     return len([x for x in array if x != '']) == 0
+
+
+def swap_empty_string_for_none(array):
+    def swap(x):
+        if x == '':
+            return None
+        else:
+            return x
+    return [swap(x) for x in array]

pyexcel_io/constants.py

@@ -20,6 +20,8 @@
 MESSAGE_CANNOT_READ_FILE_TYPE_FORMATTER = "Cannot read content of file type %s from file %s"
 MESSAGE_LOADING_FORMATTER = "The plugin for file type %s is not installed. Please install %s"
 MESSAGE_EMPTY_ARRAY = "One empty row is found"
+MESSAGE_IGNORE_ROW = "One row is ignored"
+MESSAGE_DB_EXCEPTION = "Warning: Bulk insertion got below exception. Trying to do it one by one slowly."
 
 FILE_FORMAT_CSV = 'csv'
 FILE_FORMAT_TSV = 'tsv'

pyexcel_io/djangobook.py

@@ -8,14 +8,19 @@
     :license: New BSD License, see LICENSE for more details
 """
 from ._compact import OrderedDict
-from .constants import MESSAGE_EMPTY_ARRAY
+from .constants import (
+    MESSAGE_EMPTY_ARRAY,
+    MESSAGE_DB_EXCEPTION,
+    MESSAGE_IGNORE_ROW
+)
 from .base import (
     BookReaderBase,
     SheetReaderBase,
     BookWriter,
     SheetWriter,
     from_query_sets,
-    is_empty_array
+    is_empty_array,
+    swap_empty_string_for_none
 )
 
 
@@ -77,12 +82,25 @@ def write_row(self, array):
         if is_empty_array(array):
             print(MESSAGE_EMPTY_ARRAY)
         else:
+            new_array = swap_empty_string_for_none(array)
             self.objs.append(self.mymodel(**dict(
-                zip(self.column_names, self.initializer(array))
+                zip(self.column_names, self.initializer(new_array))
             )))
 
     def close(self):
-        self.mymodel.objects.bulk_create(self.objs, batch_size=self.batch_size)
+        try:
+            self.mymodel.objects.bulk_create(self.objs, batch_size=self.batch_size)
+        except Exception as e:
+            print(MESSAGE_DB_EXCEPTION)
+            print(e)
+            for object in self.objs:
+                try:
+                    object.save()
+                except Exception as e2:
+                    print(MESSAGE_IGNORE_ROW)
+                    print(e2)
+                    print(object)
+                    continue
 
 
 class DjangoBookWriter(BookWriter):

pyexcel_io/sqlbook.py

@@ -18,7 +18,8 @@
     BookWriter,
     SheetWriter,
     from_query_sets,
-    is_empty_array
+    is_empty_array,
+    swap_empty_string_for_none
 )
 
 
@@ -78,7 +79,8 @@ def write_row(self, array):
         if is_empty_array(array):
             print(MESSAGE_EMPTY_ARRAY)
         else:
-            self._write_row(array)
+            new_array = swap_empty_string_for_none(array)
+            self._write_row(new_array)
 
     def _write_row(self, array):
         row = dict(zip(self.column_names, array))

tests/test_django_book.py

@@ -2,28 +2,46 @@
 from pyexcel_io.djangobook import DjangoModelReader, DjangoModelWriter, DjangoBookReader, DjangoBookWriter
 
 
+class Package:
+    def __init__(self, raiseException=False, **keywords):
+        self.keywords = keywords
+        self.raiseException = raiseException
+
+    def get_content(self):
+        return self.keywords
+
+    def save(self):
+        if self.raiseException:
+            raise Exception("test exception")
+        else:
+            pass
+
+
 class Attributable:
     def __init__(self, adict):
         self.mydict = adict
 
     def __getattr__(self, field):
         return self.mydict[field]
 
+
 class Objects:
     def __init__(self):
         self.objs = []
 
     def bulk_create(self, objs, batch_size):
-        self.objs = objs
+        self.objs = [ o.get_content() for o in objs ]
         self.batch_size = batch_size
 
     def all(self):
         return [Attributable(o) for o in self.objs]
 
+
 class Field:
     def __init__(self, name):
         self.attname = name
 
+
 class Meta:
     def __init__(self):
         self.model_name = "test"
@@ -33,13 +51,54 @@ def update(self, data):
         for f in data:
             self.concrete_fields.append(Field(f))
 
+
 class FakeDjangoModel:
     def __init__(self):
         self.objects = Objects()
         self._meta = Meta()
 
     def __call__(self, **keywords):
-        return keywords
+        return Package(**keywords)
+
+
+class ExceptionObjects(Objects):
+    def bulk_create(self, objs, batch_size):
+        raise Exception("faked exception")
+
+
+class FakeExceptionDjangoModel(FakeDjangoModel):
+    def __init__(self, raiseException=False):
+        self.objects = ExceptionObjects()
+        self._meta = Meta()
+        self.raiseException = raiseException
+
+    def __call__(self, **keywords):
+        return Package(raiseExcpetion=self.raiseException,
+                       **keywords)
+
+
+class TestException:
+    def setUp(self):
+        self.data  = [
+            ["X", "Y", "Z"],
+            [1, 2, 3],
+            [4, 5, 6]
+        ]
+        self.result = [
+            {'Y': 2, 'X': 1, 'Z': 3},
+            {'Y': 5, 'X': 4, 'Z': 6}
+        ]
+        
+    def test_sheet_save_to_django_model(self):
+        model=FakeExceptionDjangoModel()
+        writer = DjangoModelWriter([model, self.data[0], None, None])
+        writer.write_array(self.data[1:])
+        writer.close()
+        # now raise excpetion
+        model=FakeExceptionDjangoModel(raiseException=True)
+        writer = DjangoModelWriter([model, self.data[0], None, None])
+        writer.write_array(self.data[1:])
+        writer.close()
 
 
 class TestSheet:

tests/test_sql_book.py

@@ -22,7 +22,7 @@
 class Pyexcel(Base):
     __tablename__='pyexcel'
     id=Column(Integer, primary_key=True)
-    name=Column(String)
+    name=Column(String, unique=True)
     weight=Column(Float)
     birth=Column(Date)
 
@@ -138,6 +138,24 @@ def test_one_table_with_empty_rows(self):
         assert results == self.results
         mysession.close()
 
+    def test_one_table_with_empty_string_in_unique_field(self):
+        mysession = Session()
+        data = [
+            ['birth', 'id', 'name', 'weight'],
+            [datetime.date(2014, 11, 11), 0, '', 11.25],
+            [datetime.date(2014, 11, 12), 1, '', 12.25]
+        ]
+        writer = SQLTableWriter(mysession,
+                                [Pyexcel,data[0], None, None])
+        writer.write_array(data[1:])
+        writer.close()
+        query_sets=mysession.query(Pyexcel).all()
+        results = from_query_sets(data[0], query_sets)
+        assert results == [['birth', 'id', 'name', 'weight'],
+                           ['2014-11-11', 0, None, 11.25],
+                           ['2014-11-12', 1, None, 12.25]]
+        mysession.close()
+
     def test_one_table_using_mapdict_as_array(self):
         mysession = Session()
         self.data = [