Skip to content

Commit 9d36026

Browse files
tetiana-karasovagcf-owl-bot[bot]partheakweinmeister
authored and
dandhlee
committed
docs(samples): add samples for events (#155)
* feat: Retail. Products importing code samples * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * lint fix * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * update copyright year * remove ClientOptions * update requirements * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add requirement for pytest-xdist * test samples on all py3.7+ versions * add EVENTS_BUCKET_NAME * importing trsts fix * importing trsts fix * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update import_products_gcs_test.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add google-cloud-testutils==1.3.1 to requirements-test.txt * rename setup->setup_events * fix tests * lint * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix flaky tests; address review feedback * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Anthonios Partheniou <[email protected]> Co-authored-by: Karl Weinmeister <[email protected]>
1 parent bc3bee3 commit 9d36026

21 files changed

+1171
-13
lines changed

‎generated_samples/interactive-tutorials/events/conftest.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2022 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pytest
16+
import test_utils.prefixer
17+
18+
prefixer = test_utils.prefixer.Prefixer(
19+
"python-retail", "samples/interactive-tutorials/product"
20+
)
21+
22+
23+
@pytest.fixture(scope="session")
24+
def table_id_prefix() -> str:
25+
return prefixer.create_prefix()
26+
27+
28+
@pytest.fixture(scope="session")
29+
def bucket_name_prefix() -> str:
30+
return prefixer.create_prefix()

‎generated_samples/interactive-tutorials/events/import_user_events_big_query.py

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Copyright 2022 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import argparse
16+
import os
17+
18+
project_id = os.environ["GOOGLE_CLOUD_PROJECT"]
19+
20+
21+
def main(project_id, dataset_id, table_id):
22+
# [START retail_import_user_events_from_big_query]
23+
# TODO: Set project_id to your Google Cloud Platform project ID.
24+
# project_id = "my-project"
25+
26+
# TODO: Set dataset_id
27+
# dataset_id = "user_events"
28+
29+
# TODO: Set table_id
30+
# table_id = "events"
31+
32+
# Import products into a catalog from big query table using Retail API
33+
import time
34+
35+
from google.cloud.retail import (
36+
BigQuerySource,
37+
ImportUserEventsRequest,
38+
UserEventInputConfig,
39+
UserEventServiceClient,
40+
)
41+
42+
default_catalog = f"projects/{project_id}/locations/global/catalogs/default_catalog"
43+
44+
# TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS:
45+
# table_id = "events_some_invalid"
46+
47+
# get import user events from big query request
48+
def get_import_events_big_query_request():
49+
# TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE:
50+
# default_catalog = "invalid_catalog_name"
51+
big_query_source = BigQuerySource()
52+
big_query_source.project_id = project_id
53+
big_query_source.dataset_id = dataset_id
54+
big_query_source.table_id = table_id
55+
big_query_source.data_schema = "user_event"
56+
57+
input_config = UserEventInputConfig()
58+
input_config.big_query_source = big_query_source
59+
60+
import_request = ImportUserEventsRequest()
61+
import_request.parent = default_catalog
62+
import_request.input_config = input_config
63+
64+
print("---import user events from BigQuery source request---")
65+
print(import_request)
66+
67+
return import_request
68+
69+
# call the Retail API to import user events
70+
def import_user_events_from_big_query():
71+
import_big_query_request = get_import_events_big_query_request()
72+
big_query_operation = UserEventServiceClient().import_user_events(
73+
import_big_query_request
74+
)
75+
76+
print("---the operation was started:----")
77+
print(big_query_operation.operation.name)
78+
79+
while not big_query_operation.done():
80+
print("---please wait till operation is done---")
81+
time.sleep(30)
82+
print("---import user events operation is done---")
83+
84+
if big_query_operation.metadata is not None:
85+
print("---number of successfully imported events---")
86+
print(big_query_operation.metadata.success_count)
87+
print("---number of failures during the importing---")
88+
print(big_query_operation.metadata.failure_count)
89+
else:
90+
print("---operation.metadata is empty---")
91+
92+
if big_query_operation.result is not None:
93+
print("---operation result:---")
94+
print(big_query_operation.result())
95+
else:
96+
print("---operation.result is empty---")
97+
98+
import_user_events_from_big_query()
99+
100+
# [END retail_import_user_events_from_big_query]
101+
102+
103+
if __name__ == "__main__":
104+
parser = argparse.ArgumentParser()
105+
parser.add_argument("dataset_id", nargs="?", default="user_events")
106+
parser.add_argument("table_id", nargs="?", default="events")
107+
args = parser.parse_args()
108+
main(project_id, args.dataset_id, args.table_id)

‎generated_samples/interactive-tutorials/events/import_user_events_bigquery_test.py

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Copyright 2022 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import re
16+
import subprocess
17+
18+
from setup_events.setup_cleanup import (
19+
create_bq_dataset,
20+
create_bq_table,
21+
delete_bq_table,
22+
upload_data_to_bq_table,
23+
)
24+
from setup_events.update_user_events_json import update_events_timestamp
25+
26+
27+
def test_import_products_bq(table_id_prefix):
28+
dataset = "user_events"
29+
valid_products_table = f"{table_id_prefix}events"
30+
product_schema = "../resources/events_schema.json"
31+
valid_products_source_file = "../resources/user_events.json"
32+
33+
try:
34+
update_events_timestamp("../resources/user_events.json")
35+
update_events_timestamp("../resources/user_events_some_invalid.json")
36+
create_bq_dataset(dataset)
37+
create_bq_table(dataset, valid_products_table, product_schema)
38+
upload_data_to_bq_table(
39+
dataset, valid_products_table, valid_products_source_file, product_schema
40+
)
41+
output = str(
42+
subprocess.check_output(
43+
f"python import_user_events_big_query.py {dataset} {valid_products_table}",
44+
shell=True,
45+
)
46+
)
47+
finally:
48+
delete_bq_table(dataset, valid_products_table)
49+
50+
assert re.match(
51+
'.*import user events from BigQuery source request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*',
52+
output,
53+
)
54+
assert re.match(
55+
".*import user events from BigQuery source request.*?input_config.*?big_query_source.*",
56+
output,
57+
)
58+
assert re.match(
59+
".*the operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/import-user-events.*",
60+
output,
61+
)
62+
assert re.match(".*import user events operation is done.*", output)
63+
assert re.match(".*number of successfully imported events.*", output)
64+
assert re.match(".*number of failures during the importing.*?0.*", output)
65+
assert re.match(".*operation result.*?errors_config.*", output)

‎generated_samples/interactive-tutorials/events/import_user_events_gcs.py

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Copyright 2022 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import argparse
16+
import os
17+
18+
19+
def main(bucket_name):
20+
# [START retail_import_user_events_from_gcs]
21+
# Import user events into a catalog from GCS using Retail API
22+
23+
import time
24+
25+
from google.cloud.retail import (
26+
GcsSource,
27+
ImportErrorsConfig,
28+
ImportUserEventsRequest,
29+
UserEventInputConfig,
30+
UserEventServiceClient,
31+
)
32+
33+
# Read the project number from the environment variable
34+
project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
35+
36+
# Read bucket name from the environment variable
37+
bucket_name = os.getenv("EVENTS_BUCKET_NAME")
38+
39+
# TODO: Developer set the bucket_name
40+
# bucket_name = 'user_events_bucket'
41+
42+
default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format(
43+
project_id
44+
)
45+
46+
gcs_bucket = "gs://{}".format(bucket_name)
47+
gcs_errors_bucket = "{}/error".format(gcs_bucket)
48+
gcs_events_object = "user_events.json"
49+
50+
# TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT
51+
# gcs_events_object = "user_events_some_invalid.json"
52+
53+
# get import user events from gcs request
54+
def get_import_events_gcs_request(gcs_object_name: str):
55+
# TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE:
56+
# default_catalog = "invalid_catalog_name"
57+
gcs_source = GcsSource()
58+
gcs_source.input_uris = [f"{gcs_bucket}/{gcs_object_name}"]
59+
60+
input_config = UserEventInputConfig()
61+
input_config.gcs_source = gcs_source
62+
63+
errors_config = ImportErrorsConfig()
64+
errors_config.gcs_prefix = gcs_errors_bucket
65+
66+
import_request = ImportUserEventsRequest()
67+
import_request.parent = default_catalog
68+
import_request.input_config = input_config
69+
import_request.errors_config = errors_config
70+
71+
print("---import user events from google cloud source request---")
72+
print(import_request)
73+
74+
return import_request
75+
76+
# call the Retail API to import user events
77+
def import_user_events_from_gcs():
78+
import_gcs_request = get_import_events_gcs_request(gcs_events_object)
79+
gcs_operation = UserEventServiceClient().import_user_events(import_gcs_request)
80+
81+
print("---the operation was started:----")
82+
print(gcs_operation.operation.name)
83+
84+
while not gcs_operation.done():
85+
print("---please wait till operation is done---")
86+
time.sleep(30)
87+
88+
print("---import user events operation is done---")
89+
90+
if gcs_operation.metadata is not None:
91+
print("---number of successfully imported events---")
92+
print(gcs_operation.metadata.success_count)
93+
print("---number of failures during the importing---")
94+
print(gcs_operation.metadata.failure_count)
95+
else:
96+
print("---operation.metadata is empty---")
97+
98+
if gcs_operation.result is not None:
99+
print("---operation result:---")
100+
print(gcs_operation.result())
101+
else:
102+
print("---operation.result is empty---")
103+
104+
import_user_events_from_gcs()
105+
106+
107+
# [END retail_import_user_events_from_gcs]
108+
109+
if __name__ == "__main__":
110+
parser = argparse.ArgumentParser()
111+
parser.add_argument(
112+
"bucket_name", nargs="?", default=os.environ["EVENTS_BUCKET_NAME"]
113+
)
114+
args = parser.parse_args()
115+
main(args.bucket_name)

‎generated_samples/interactive-tutorials/events/import_user_events_gcs_test.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright 2022 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import re
16+
import subprocess
17+
18+
from setup_events.setup_cleanup import create_bucket, delete_bucket, upload_blob
19+
from setup_events.update_user_events_json import update_events_timestamp
20+
21+
22+
def test_import_events_gcs(bucket_name_prefix):
23+
# gcs buckets have a limit of 63 characters. Get the last 60 characters
24+
bucket_name = bucket_name_prefix[63:]
25+
26+
try:
27+
update_events_timestamp("../resources/user_events.json")
28+
update_events_timestamp("../resources/user_events_some_invalid.json")
29+
create_bucket(bucket_name)
30+
upload_blob(bucket_name, "../resources/user_events.json")
31+
32+
output = str(
33+
subprocess.check_output("python import_user_events_gcs.py", shell=True)
34+
)
35+
finally:
36+
delete_bucket(bucket_name)
37+
38+
assert re.match(
39+
'.*import user events from google cloud source request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*',
40+
output,
41+
)
42+
assert re.match(
43+
".*import user events from google cloud source request.*?input_config.*?gcs_source.*",
44+
output,
45+
)
46+
assert re.match(
47+
".*the operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/import-user-events.*",
48+
output,
49+
)
50+
assert re.match(".*import user events operation is done.*", output)
51+
assert re.match(".*number of successfully imported events.*?4.*", output)
52+
assert re.match(".*number of failures during the importing.*?0.*", output)
53+
assert re.match(".*operation result.*?errors_config.*", output)

0 commit comments

Comments
 (0)