Merge pull request #261 from scaleoutsystems/release/v0.6.0-1

Release/v0.6.0 1

Author: Wrede

components/studio/models/apps.py

@@ -2,5 +2,6 @@
 
 
 class ModelsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
     name = 'models'
-    verbose_name = 'Scaleout Models'
+    verbose_name = 'Studio Models'

components/studio/models/fixtures/object_type.json

@@ -0,0 +1,47 @@
+[
+  {
+    "model": "models.objecttype",
+    "pk": 1,
+    "fields": {
+      "name": "Default Model",
+      "slug": "default",
+      "app_slug": "default-serve"
+    }
+  },
+  {
+    "model": "models.objecttype",
+    "pk": 2,
+    "fields": {
+      "name": "Tensorflow Model",
+      "slug": "tensorflow",
+      "app_slug": "tensorflow-serve"
+    }
+  },
+  {
+    "model": "models.objecttype",
+    "pk": 3,
+    "fields": {
+      "name": "PyTorch Model",
+      "slug": "pytorch",
+      "app_slug": "pytorch-serve"
+    }
+  },
+  {
+    "model": "models.objecttype",
+    "pk": 4,
+    "fields": {
+      "name": "MLflow Model",
+      "slug": "mlflow",
+      "app_slug": "mlflow-serve"
+    }
+  },
+  {
+    "model": "models.objecttype",
+    "pk": 5,
+    "fields": {
+      "name": "Python Model Deployment",
+      "slug": "python",
+      "app_slug": "fastapi-serve"
+    }
+  }
+]

components/studio/models/fixtures/objecttype_fixtures.json

@@ -5,10 +5,9 @@
 class ModelForm(forms.ModelForm):
     class Meta:
         model = Model
-        fields = ('uid', 'name', 'description', 'url', 'project')
-        widgets = {
-            'uid': forms.HiddenInput(),
-            'project': forms.HiddenInput()
+        fields = ('name', 'description', 'release_type', 'version', 'access', 'path')
+        labels = {
+            'path': ('Current folder name of your trained model*'),
         }
 
 class ModelLogForm(forms.ModelForm):

components/studio/models/forms.py

@@ -1,33 +1,9 @@
 from .models import Model
 from minio import Minio
-import s3fs
 from portal.models import PublicModelObject, PublishedModel
 
-def get_download_url(model_id):
-    model = Model.objects.get(pk=model_id)
-    bucket = model.bucket
-    path = model.path
-    uid = model.uid
-
-    minio_host = model.s3.host
-    minio_access_key = model.s3.access_key
-    minio_secret_key = model.s3.secret_key
-    minio_region = model.s3.region
-
-    download_url = ""
-    try:
-        client = Minio(
-            minio_host,
-            access_key=minio_access_key,
-            secret_key=minio_secret_key,
-            region=minio_region,
-        )
-
-        download_url = client.presigned_get_object(bucket, path + uid)
-    except Exception as e:
-        print(e)
-
-    return download_url
+import io
+import s3fs
 
 def add_pmo_to_publish(mdl, pmodel):
     print(mdl.name)
@@ -63,4 +39,95 @@ def add_pmo_to_publish(mdl, pmodel):
         print(err)
 
     print("Created public model object")
-    pmodel.model_obj.add(pmo)
+    pmodel.model_obj.add(pmo)
+
+
+def get_download_url(model_id):
+    model = Model.objects.get(pk=model_id)
+    bucket = model.bucket
+    path = model.path
+    uid = model.uid
+
+    minio_host = model.s3.host
+    minio_access_key = model.s3.access_key
+    minio_secret_key = model.s3.secret_key
+    minio_region = model.s3.region
+
+    download_url = ""
+    try:
+        client = Minio(
+            minio_host,
+            access_key=minio_access_key,
+            secret_key=minio_secret_key,
+            region=minio_region,
+        )
+
+        download_url = client.presigned_get_object(bucket, path + uid)
+    except Exception as e:
+        print(e)
+
+    return download_url
+
+
+# This Method use Minio Python API to create a minio client and connect it to a minio server instance
+def create_client(S3_storage, secure_mode=True):
+    try:
+        access_key = S3_storage.access_key
+    except Exception:
+        print("No access key could be found with the current S3 storage instance: {}".format(S3_storage))
+        return []
+    try:
+        secret_key = S3_storage.secret_key
+    except Exception:
+        print("No secret key could be found with the current S3 storage instance: {}".format(S3_storage))
+        return []
+
+    # API connection does not want scheme in the minio URL
+    # Yet we need the URL to have the scheme included when some app charts use the minio client mc
+    if 'http://' in S3_storage.host:
+        minio_url = S3_storage.host.replace('http://', '')
+    elif 'https://' in S3_storage.host:
+        minio_url = S3_storage.host.replace('https://', '')
+    else:
+        minio_url = S3_storage.host
+
+    if not secure_mode:
+        client = Minio(minio_url, access_key=access_key, secret_key=secret_key, secure=secure_mode)
+    else:
+        client = Minio(minio_url, access_key=access_key, secret_key=secret_key)
+
+    return client
+
+
+# This Method use Minio Python API to save an artificat into a running minio server instance
+def set_artifact(artifact_name, artifact_file, bucket, S3_storage, is_file=False, secure_mode=True):
+    """ Instance must be a byte-like object. """
+    client = create_client(S3_storage, secure_mode)
+
+    try:
+        found = client.bucket_exists(bucket)
+    except Exception as err:
+        print('EXCEPTION LOG: Client could not verify if bucket exists')
+        return False
+    
+    if not found:
+        try:
+            client.make_bucket(bucket)
+        except Exception as err:
+            print('Bucket does not exist, and failed to create bucket.')
+            return False
+
+    if is_file == True:
+        try:
+            client.fput_object(bucket, artifact_name, artifact_file)
+        except Exception as e:
+            print('Client method fput_object failed')
+            return False
+    else:
+        try:
+            client.put_object(bucket, artifact_name, io.BytesIO(artifact_file), len(artifact_file))
+        except Exception as e:
+            print('Client method put_object failed')
+            return False
+
+    return True

components/studio/models/helpers.py

@@ -1,40 +1,97 @@
-# Generated by Django 2.2.13 on 2020-08-31 08:56
+# Generated by Django 4.0 on 2021-12-13 08:51
 
 from django.db import migrations, models
-import django.db.models.deletion
 import django.db.models.manager
+import tagulous.models.models
 
 
 class Migration(migrations.Migration):
 
     initial = True
 
     dependencies = [
-        ('projects', '0001_initial'),
+        ('apps', '0001_initial'),
     ]
 
     operations = [
+        migrations.CreateModel(
+            name='Metadata',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('run_id', models.CharField(max_length=32)),
+                ('trained_model', models.CharField(default='', max_length=32)),
+                ('project', models.CharField(default='', max_length=255)),
+                ('model_details', models.TextField(blank=True)),
+                ('parameters', models.TextField(blank=True)),
+                ('metrics', models.TextField(blank=True)),
+            ],
+        ),
         migrations.CreateModel(
             name='Model',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('uid', models.CharField(max_length=255)),
                 ('name', models.CharField(max_length=255)),
                 ('version', models.CharField(max_length=255)),
                 ('release_type', models.CharField(max_length=255)),
                 ('description', models.CharField(blank=True, max_length=255, null=True)),
+                ('model_card', models.TextField(blank=True, null=True)),
                 ('access', models.CharField(choices=[('PR', 'Private'), ('LI', 'Limited'), ('PU', 'Public')], default='PR', max_length=2)),
                 ('resource', models.URLField(blank=True, max_length=2048, null=True)),
                 ('url', models.URLField(blank=True, max_length=512, null=True)),
+                ('bucket', models.CharField(blank=True, default='models', max_length=200, null=True)),
+                ('path', models.CharField(blank=True, default='', max_length=200, null=True)),
                 ('uploaded_at', models.DateTimeField(auto_now_add=True)),
                 ('status', models.CharField(choices=[('CR', 'Created'), ('DP', 'Deployed'), ('AR', 'Archived')], default='CR', max_length=2)),
-                ('project', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='model_owner', to='projects.Project')),
+                ('model_card_headline', models.ImageField(blank=True, default=None, null=True, upload_to='models/image')),
             ],
-            options={
-                'unique_together': {('name', 'version', 'project')},
-            },
             managers=[
                 ('objects_version', django.db.models.manager.Manager()),
             ],
         ),
+        migrations.CreateModel(
+            name='Tagulous_Model_tags',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, unique=True)),
+                ('slug', models.SlugField()),
+                ('count', models.IntegerField(default=0, help_text='Internal counter of how many times this tag is in use')),
+                ('protected', models.BooleanField(default=False, help_text='Will not be deleted when the count reaches 0')),
+            ],
+            options={
+                'ordering': ('name',),
+                'abstract': False,
+                'unique_together': {('slug',)},
+            },
+            bases=(tagulous.models.models.BaseTagModel, models.Model),
+        ),
+        migrations.CreateModel(
+            name='ObjectType',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(blank=True, default='Model', max_length=100, null=True)),
+                ('slug', models.CharField(blank=True, default='model', max_length=100, null=True)),
+                ('apps', models.ManyToManyField(blank=True, to='apps.Apps')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ModelLog',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('run_id', models.CharField(max_length=32)),
+                ('trained_model', models.CharField(default='', max_length=32)),
+                ('project', models.CharField(default='', max_length=255)),
+                ('training_started_at', models.CharField(max_length=255)),
+                ('execution_time', models.CharField(default='', max_length=255)),
+                ('code_version', models.CharField(default='', max_length=255)),
+                ('current_git_repo', models.CharField(default='', max_length=255)),
+                ('latest_git_commit', models.CharField(default='', max_length=255)),
+                ('system_details', models.TextField(blank=True)),
+                ('cpu_details', models.TextField(blank=True)),
+                ('training_status', models.CharField(choices=[('ST', 'Started'), ('DO', 'Done'), ('FA', 'Failed')], default='ST', max_length=2)),
+            ],
+            options={
+                'unique_together': {('run_id', 'trained_model')},
+            },
+        ),
     ]

components/studio/models/migrations/0001_initial.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.0 on 2021-12-13 08:51
+
+from django.db import migrations, models
+import django.db.models.deletion
+import tagulous.models.fields
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('models', '0001_initial'),
+        ('projects', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='model',
+            name='docker_image',
+            field=models.OneToOneField(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='projects.environment'),
+        ),
+        migrations.AddField(
+            model_name='model',
+            name='object_type',
+            field=models.ManyToManyField(blank=True, to='models.ObjectType'),
+        ),
+        migrations.AddField(
+            model_name='model',
+            name='project',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='model_owner', to='projects.project'),
+        ),
+        migrations.AddField(
+            model_name='model',
+            name='s3',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='projects.s3'),
+        ),
+        migrations.AddField(
+            model_name='model',
+            name='tags',
+            field=tagulous.models.fields.TagField(_set_tag_meta=True, help_text='Enter a comma-separated tag string', to='models.Tagulous_Model_tags'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='metadata',
+            unique_together={('run_id', 'trained_model')},
+        ),
+        migrations.AlterUniqueTogether(
+            name='model',
+            unique_together={('name', 'version', 'project')},
+        ),
+    ]