automl: video beta - move beta samples out of branch and into master (#2750)

* automl: video beta - move beta samples out of branch and into master

* lint

* update error message on batch predict

Co-authored-by: Leah E. Cole <[email protected]>

Author: nnegrey

automl/beta/batch_predict.py

@@ -0,0 +1,52 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def batch_predict(project_id, model_id, input_uri, output_uri):
+    """Batch predict"""
+    # [START automl_batch_predict_beta]
+    from google.cloud import automl_v1beta1 as automl
+
+    # TODO(developer): Uncomment and set the following variables
+    # project_id = "YOUR_PROJECT_ID"
+    # model_id = "YOUR_MODEL_ID"
+    # input_uri = "gs://YOUR_BUCKET_ID/path/to/your/input/csv_or_jsonl"
+    # output_uri = "gs://YOUR_BUCKET_ID/path/to/save/results/"
+
+    prediction_client = automl.PredictionServiceClient()
+
+    # Get the full path of the model.
+    model_full_id = prediction_client.model_path(
+        project_id, "us-central1", model_id
+    )
+
+    gcs_source = automl.types.GcsSource(input_uris=[input_uri])
+
+    input_config = automl.types.BatchPredictInputConfig(gcs_source=gcs_source)
+    gcs_destination = automl.types.GcsDestination(output_uri_prefix=output_uri)
+    output_config = automl.types.BatchPredictOutputConfig(
+        gcs_destination=gcs_destination
+    )
+
+    response = prediction_client.batch_predict(
+        model_full_id, input_config, output_config
+    )
+
+    print("Waiting for operation to complete...")
+    print(
+        "Batch Prediction results saved to Cloud Storage bucket. {}".format(
+            response.result()
+        )
+    )
+    # [END automl_batch_predict_beta]

automl/beta/batch_predict_test.py

@@ -0,0 +1,47 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific ladnguage governing permissions and
+# limitations under the License.
+
+import datetime
+import os
+
+import batch_predict
+
+PROJECT_ID = os.environ["AUTOML_PROJECT_ID"]
+BUCKET_ID = "{}-lcm".format(PROJECT_ID)
+MODEL_ID = "TEN0000000000000000000"
+PREFIX = "TEST_EXPORT_OUTPUT_" + datetime.datetime.now().strftime(
+    "%Y%m%d%H%M%S"
+)
+
+
+def test_batch_predict(capsys):
+    # As batch prediction can take a long time. Try to batch predict on a model
+    # and confirm that the model was not found, but other elements of the
+    # request were valid.
+    try:
+        input_uri = "gs://{}/entity-extraction/input.jsonl".format(BUCKET_ID)
+        output_uri = "gs://{}/{}/".format(BUCKET_ID, PREFIX)
+        batch_predict.batch_predict(
+            PROJECT_ID, MODEL_ID, input_uri, output_uri
+        )
+        out, _ = capsys.readouterr()
+        assert (
+            "does not exist"
+            in out
+        )
+    except Exception as e:
+        assert (
+            "does not exist"
+            in e.message
+        )

automl/beta/video_classification_create_dataset.py

@@ -0,0 +1,45 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def create_dataset(project_id, display_name):
+    """Create a dataset."""
+    # [START automl_video_classification_create_dataset_beta]
+    from google.cloud import automl_v1beta1 as automl
+
+    # TODO(developer): Uncomment and set the following variables
+    # project_id = "YOUR_PROJECT_ID"
+    # display_name = "your_datasets_display_name"
+
+    client = automl.AutoMlClient()
+
+    # A resource that represents Google Cloud Platform location.
+    project_location = client.location_path(project_id, "us-central1")
+    metadata = automl.types.VideoClassificationDatasetMetadata()
+    dataset = automl.types.Dataset(
+        display_name=display_name,
+        video_classification_dataset_metadata=metadata,
+    )
+
+    # Create a dataset with the dataset metadata in the region.
+    created_dataset = client.create_dataset(project_location, dataset)
+
+    # Display the dataset information
+    print("Dataset name: {}".format(created_dataset.name))
+    # To get the dataset id, you have to parse it out of the `name` field.
+    # As dataset Ids are required for other methods.
+    # Name Form:
+    #    `projects/{project_id}/locations/{location_id}/datasets/{dataset_id}`
+    print("Dataset id: {}".format(created_dataset.name.split("/")[-1]))
+    # [END automl_video_classification_create_dataset_beta]

automl/beta/video_classification_create_dataset_test.py

@@ -0,0 +1,51 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import os
+
+from google.cloud import automl_v1beta1 as automl
+import pytest
+
+import video_classification_create_dataset
+
+
+PROJECT_ID = os.environ["AUTOML_PROJECT_ID"]
+pytest.DATASET_ID = None
+
+
+@pytest.fixture(scope="function", autouse=True)
+def teardown():
+    yield
+
+    # Delete the created dataset
+    client = automl.AutoMlClient()
+    dataset_full_id = client.dataset_path(
+        PROJECT_ID, "us-central1", pytest.DATASET_ID
+    )
+    response = client.delete_dataset(dataset_full_id)
+    response.result()
+
+
+def test_video_classification_create_dataset(capsys):
+    # create dataset
+    dataset_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+    video_classification_create_dataset.create_dataset(
+        PROJECT_ID, dataset_name
+    )
+    out, _ = capsys.readouterr()
+    assert "Dataset id: " in out
+
+    # Get the the created dataset id for deletion
+    pytest.DATASET_ID = out.splitlines()[1].split()[2]

automl/beta/video_classification_create_model.py

@@ -0,0 +1,42 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def create_model(project_id, dataset_id, display_name):
+    """Create a model."""
+    # [START automl_video_classification_create_model_beta]
+    from google.cloud import automl_v1beta1 as automl
+
+    # TODO(developer): Uncomment and set the following variables
+    # project_id = "YOUR_PROJECT_ID"
+    # dataset_id = "YOUR_DATASET_ID"
+    # display_name = "your_models_display_name"
+
+    client = automl.AutoMlClient()
+
+    # A resource that represents Google Cloud Platform location.
+    project_location = client.location_path(project_id, "us-central1")
+    metadata = automl.types.VideoClassificationModelMetadata()
+    model = automl.types.Model(
+        display_name=display_name,
+        dataset_id=dataset_id,
+        video_classification_model_metadata=metadata,
+    )
+
+    # Create a model with the model metadata in the region.
+    response = client.create_model(project_location, model)
+
+    print("Training operation name: {}".format(response.operation.name))
+    print("Training started...")
+    # [END automl_video_classification_create_model_beta]

automl/beta/video_classification_create_model_test.py

@@ -0,0 +1,46 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from google.cloud import automl_v1beta1 as automl
+import pytest
+
+import video_classification_create_model
+
+PROJECT_ID = os.environ["GCLOUD_PROJECT"]
+DATASET_ID = "VCN510437278078730240"
+pytest.OPERATION_ID = None
+
+
+@pytest.fixture(scope="function", autouse=True)
+def teardown():
+    yield
+
+    # Cancel the operation
+    client = automl.AutoMlClient()
+    client.transport._operations_client.cancel_operation(pytest.OPERATION_ID)
+
+
+def test_video_classification_create_model(capsys):
+    video_classification_create_model.create_model(
+        PROJECT_ID, DATASET_ID, "classification_test_create_model"
+    )
+    out, _ = capsys.readouterr()
+    assert "Training started" in out
+
+    # Get the the operation id for cancellation
+    pytest.OPERATION_ID = out.split("Training operation name: ")[1].split(
+        "\n"
+    )[0]