cli: adds replace option

 * Closes #1836.

Signed-off-by: Ioannis Tsanaktsidis <[email protected]>

Author: ioannistsanaktsidis

cernopendata/modules/fixtures/cli.py

@@ -58,8 +58,10 @@ def fixtures():
 @click.option('--profile', is_flag=True,
               help='Output profiling information.')
 @click.option('--verbose', is_flag=True, default=False)
+@click.option('--mode', required=True, type=click.Choice(
+    ['insert', 'replace']))
 @with_appcontext
-def records(skip_files, files, profile, verbose):
+def records(skip_files, files, profile, verbose, mode):
     """Load all records."""
     if profile:
         import cProfile
@@ -77,13 +79,16 @@ def records(skip_files, files, profile, verbose):
     from invenio_files_rest.models import \
         Bucket, FileInstance, ObjectVersion
     from invenio_records_files.models import RecordsBuckets
+    from invenio_pidstore.models import PersistentIdentifier
+    from invenio_pidstore.errors import PIDDoesNotExistError
 
     indexer = RecordIndexer()
     schema = current_app.extensions['invenio-jsonschemas'].path_to_url(
         'records/record-v1.0.0.json'
     )
     data = pkg_resources.resource_filename('cernopendata',
                                            'modules/fixtures/data/records')
+
     if files:
         record_json = files
     else:
@@ -103,48 +108,56 @@ def records(skip_files, files, profile, verbose):
                                format(data.get('recid')))
 
                 files = data.get('files', [])
-
-                bucket = Bucket.create()
-
-                for file in files:
-                    if skip_files:
-                        break
-                    assert 'uri' in file
-                    assert 'size' in file
-                    assert 'checksum' in file
-
+                if mode == 'insert':
+                    bucket = Bucket.create()
+
+                    for file in files:
+                        if skip_files:
+                            break
+                        assert 'uri' in file
+                        assert 'size' in file
+                        assert 'checksum' in file
+
+                        try:
+                            f = FileInstance.create()
+                            filename = file.get("uri").split('/')[-1:][0]
+                            f.set_uri(file.get("uri"), file.get(
+                                "size"), file.get("checksum"))
+                            obj = ObjectVersion.create(
+                                bucket,
+                                filename,
+                                _file_id=f.id
+                            )
+
+                            file.update({
+                                'bucket': str(obj.bucket_id),
+                                'checksum': obj.file.checksum,
+                                'key': obj.key,
+                                'version_id': str(obj.version_id),
+                            })
+
+                        except Exception as e:
+                            click.echo(
+                                'Recid {0} file {1} could not be loaded due '
+                                'to {2}.'.format(data.get('recid'), filename,
+                                                 str(e)))
+                            continue
+
+                    id = uuid.uuid4()
+                    cernopendata_recid_minter(id, data)
+                    record = Record.create(data, id_=id)
+                    record['$schema'] = schema
+                    RecordsBuckets.create(
+                        record=record.model, bucket=bucket)
+                else:
                     try:
-                        f = FileInstance.create()
-                        filename = file.get("uri").split('/')[-1:][0]
-                        f.set_uri(file.get("uri"), file.get(
-                            "size"), file.get("checksum"))
-                        obj = ObjectVersion.create(
-                            bucket,
-                            filename,
-                            _file_id=f.id
-                        )
-
-                        file.update({
-                            'bucket': str(obj.bucket_id),
-                            'checksum': obj.file.checksum,
-                            'key': obj.key,
-                            'version_id': str(obj.version_id),
-                        })
-
-                    except Exception as e:
-                        click.echo(
-                            'Recid {0} file {1} could not be loaded due '
-                            'to {2}.'.format(data.get('recid'), filename,
-                                             str(e)))
-                        continue
-
-                id = uuid.uuid4()
-                cernopendata_recid_minter(id, data)
-                record = Record.create(data, id_=id)
-                record['$schema'] = schema
-                RecordsBuckets.create(
-                    record=record.model, bucket=bucket)
-
+                        pid = PersistentIdentifier.get('recid', data['recid'])
+                    except PIDDoesNotExistError:
+                        return None
+                    record = Record.get_record(pid.object_uuid)
+                    record['$schema'] = schema
+                    record.update(data)
+                    record.commit()
                 db.session.commit()
                 indexer.index(record)
                 db.session.expunge_all()
@@ -180,7 +193,8 @@ def glossary_terms():
         with open(filename, 'rb') as source:
             for data in json.load(source):
                 if "collections" not in data and \
-                   not isinstance(data.get("collections", None), basestring):
+                    not isinstance(
+                        data.get("collections", None), basestring):
                     data["collections"] = []
                 data["collections"].append({"primary": "Terms"})
                 id = uuid.uuid4()
@@ -228,7 +242,8 @@ def articles():
                 with open(content_filename) as body_field:
                     data["body"]["content"] = body_field.read()
                 if "collections" not in data and \
-                   not isinstance(data.get("collections", None), basestring):
+                    not isinstance(
+                        data.get("collections", None), basestring):
                     data["collections"] = []
                 id = uuid.uuid4()
                 cernopendata_articleid_minter(id, data)
@@ -254,6 +269,7 @@ def data_policies(skip_files):
         Bucket, FileInstance, ObjectVersion
     from invenio_records_files.models import RecordsBuckets
     from invenio_records_files.api import Record
+
     from invenio_records.models import RecordMetadata
 
     indexer = RecordIndexer()

scripts/populate-instance.sh

@@ -127,12 +127,12 @@ if [[ "$@" = *"--skip-records"* ]]; then
 else
     if [[ "$@" = *"--skip-files"* ]]; then
         echo "[INFO] Skipping loading of record files."
-        ${INVENIO_WEB_INSTANCE} fixtures records --skip-files
+        ${INVENIO_WEB_INSTANCE} fixtures records --skip-files --mode insert
     else
         # Prevent memory leak which happens when all fixtures are loaded at once
         for recordfile in $(ls -Sr cernopendata/modules/fixtures/data/records/*.json);
         do
-            ${INVENIO_WEB_INSTANCE} fixtures records -f "$recordfile" --verbose
+            ${INVENIO_WEB_INSTANCE} fixtures records -f "$recordfile" --verbose --mode insert
         done
     fi
 fi