From b0a81733a9e90ed066c396a521e810fbafe560ba Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 10 Aug 2016 14:49:53 -0700 Subject: [PATCH 001/100] Add veneer samples for BigQuery. [(#444)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/444) Put them in their own directory, since they duplicate the content from the existing samples that use the generated libraries. --- samples/snippets/README.md | 5 ++ samples/snippets/async_query.py | 83 ++++++++++++++++++++++++++++ samples/snippets/async_query_test.py | 27 +++++++++ samples/snippets/requirements.txt | 1 + samples/snippets/sync_query.py | 66 ++++++++++++++++++++++ samples/snippets/sync_query_test.py | 27 +++++++++ 6 files changed, 209 insertions(+) create mode 100644 samples/snippets/README.md create mode 100755 samples/snippets/async_query.py create mode 100644 samples/snippets/async_query_test.py create mode 100644 samples/snippets/requirements.txt create mode 100755 samples/snippets/sync_query.py create mode 100644 samples/snippets/sync_query_test.py diff --git a/samples/snippets/README.md b/samples/snippets/README.md new file mode 100644 index 000000000..9c1f9b9fb --- /dev/null +++ b/samples/snippets/README.md @@ -0,0 +1,5 @@ +# BigQuery Google Cloud Client Library Samples + + + + diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py new file mode 100755 index 000000000..0ca324240 --- /dev/null +++ b/samples/snippets/async_query.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to perform asynchronous queries in BigQuery. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python async_query.py \ + 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' +""" + +import argparse +import time +import uuid + +from gcloud import bigquery + + +def async_query(query): + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + query_job.use_legacy_sql = False + query_job.begin() + + wait_for_job(query_job) + + # Manually construct the QueryResults. + # TODO: The client library will provide a helper method that does this. + # https://github.com/GoogleCloudPlatform/gcloud-python/issues/2083 + query_results = bigquery.query.QueryResults('', client) + query_results._properties['jobReference'] = { + 'jobId': query_job.name, + 'projectId': query_job.project + } + + # Drain the query results by requesting a page at a time. + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break + + +def wait_for_job(job): + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('query', help='BigQuery SQL Query.') + + args = parser.parse_args() + + async_query(args.query) diff --git a/samples/snippets/async_query_test.py b/samples/snippets/async_query_test.py new file mode 100644 index 000000000..810c538a6 --- /dev/null +++ b/samples/snippets/async_query_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from async_query import async_query + + +def test_async_query(cloud_config, capsys): + query = ( + 'SELECT corpus FROM `publicdata.samples.shakespeare` ' + 'GROUP BY corpus;') + + async_query(query) + + out, _ = capsys.readouterr() + + assert 'romeoandjuliet' in out diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 000000000..2beeafe63 --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1 @@ +gcloud==0.18.1 diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py new file mode 100755 index 000000000..59007b537 --- /dev/null +++ b/samples/snippets/sync_query.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to perform synchronous queries in BigQuery. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python sync_query.py \ + 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' +""" + +import argparse + +# [START sync_query] +from gcloud import bigquery + + +def sync_query(query): + client = bigquery.Client() + query_results = client.run_sync_query(query) + + # Use standard SQL syntax for queries. + # See: https://cloud.google.com/bigquery/sql-reference/ + query_results.use_legacy_sql = False + + query_results.run() + + # Drain the query results by requesting a page at a time. + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break +# [END sync_query] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('query', help='BigQuery SQL Query.') + + args = parser.parse_args() + + sync_query(args.query) diff --git a/samples/snippets/sync_query_test.py b/samples/snippets/sync_query_test.py new file mode 100644 index 000000000..6f6b4f5fa --- /dev/null +++ b/samples/snippets/sync_query_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sync_query import sync_query + + +def test_sync_query(cloud_config, capsys): + query = ( + 'SELECT corpus FROM `publicdata.samples.shakespeare` ' + 'GROUP BY corpus;') + + sync_query(query) + + out, _ = capsys.readouterr() + + assert 'romeoandjuliet' in out From 8ed04d1cfed7e5a74e31e7735d9675bf92bd4666 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 18 Aug 2016 16:19:00 -0700 Subject: [PATCH 002/100] Add data loading samples Change-Id: Ie78c06c7375f11dc2a98e2e8cf601f3820c6f0c1 --- samples/snippets/export_data_to_gcs.py | 77 +++++++++++++++++++ samples/snippets/export_data_to_gcs_test.py | 30 ++++++++ samples/snippets/load_data_from_file.py | 79 ++++++++++++++++++++ samples/snippets/load_data_from_file_test.py | 34 +++++++++ samples/snippets/load_data_from_gcs.py | 77 +++++++++++++++++++ samples/snippets/load_data_from_gcs_test.py | 31 ++++++++ samples/snippets/resources/data.csv | 1 + samples/snippets/resources/data.json | 1 + samples/snippets/resources/schema.json | 1 + samples/snippets/stream_data.py | 69 +++++++++++++++++ samples/snippets/stream_data_test.py | 29 +++++++ 11 files changed, 429 insertions(+) create mode 100644 samples/snippets/export_data_to_gcs.py create mode 100644 samples/snippets/export_data_to_gcs_test.py create mode 100644 samples/snippets/load_data_from_file.py create mode 100644 samples/snippets/load_data_from_file_test.py create mode 100644 samples/snippets/load_data_from_gcs.py create mode 100644 samples/snippets/load_data_from_gcs_test.py create mode 100644 samples/snippets/resources/data.csv create mode 100644 samples/snippets/resources/data.json create mode 100644 samples/snippets/resources/schema.json create mode 100644 samples/snippets/stream_data.py create mode 100644 samples/snippets/stream_data_test.py diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py new file mode 100644 index 000000000..c9771ea1b --- /dev/null +++ b/samples/snippets/export_data_to_gcs.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exports data from BigQuery to an object in Google Cloud Storage. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python export_data_to_gcs.py example_dataset example_table \ + gs://example-bucket/example-data.csv + +The dataset and table should already exist. +""" + +import argparse +import time +import uuid + +from gcloud import bigquery + + +def export_data_to_gcs(dataset_name, table_name, destination): + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + job_name = str(uuid.uuid4()) + + job = bigquery_client.extract_table_to_storage( + job_name, table, destination) + + job.begin() + + wait_for_job(job) + + print('Exported {}:{} to {}'.format( + dataset_name, table_name, destination)) + + +def wait_for_job(job): + while True: + job.reload() + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('dataset_name') + parser.add_argument('table_name') + parser.add_argument( + 'destination', help='The desintation Google Cloud Storage object.' + 'Must be in the format gs://bucket_name/object_name') + + args = parser.parse_args() + + export_data_to_gcs( + args.dataset_name, + args.table_name, + args.destination) diff --git a/samples/snippets/export_data_to_gcs_test.py b/samples/snippets/export_data_to_gcs_test.py new file mode 100644 index 000000000..e260e47b4 --- /dev/null +++ b/samples/snippets/export_data_to_gcs_test.py @@ -0,0 +1,30 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import export_data_to_gcs + + +DATASET_ID = 'test_dataset' +TABLE_ID = 'test_import_table' + + +def test_export_data_to_gcs(cloud_config, capsys): + export_data_to_gcs.export_data_to_gcs( + DATASET_ID, + TABLE_ID, + 'gs://{}/test-export-data-to-gcs.csv'.format( + cloud_config.storage_bucket)) + + out, _ = capsys.readouterr() + + assert 'Exported' in out diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py new file mode 100644 index 000000000..cbb015347 --- /dev/null +++ b/samples/snippets/load_data_from_file.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads data into BigQuery from a local file. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python load_data_from_file.py example_dataset example_table \ + example-data.csv + +The dataset and table should already exist. +""" + +import argparse +import time +from gcloud import bigquery + + +def load_data_from_file(dataset_name, table_name, source_file_name): + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + + # Reload the table to get the schema. + table.reload() + + with open(source_file_name, 'rb') as source_file: + # This example uses CSV, but you can use other formats. + # See https://cloud.google.com/bigquery/loading-data + job = table.upload_from_file( + source_file, source_format='text/csv') + + job.begin() + + wait_for_job(job) + + print('Loaded {} rows into {}:{}.'.format( + job.output_rows, dataset_name, table_name)) + + +def wait_for_job(job): + while True: + job.reload() + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('dataset_name') + parser.add_argument('table_name') + parser.add_argument( + 'source_file_name', help='Path to a .csv file to upload.') + + args = parser.parse_args() + + load_data_from_file( + args.dataset_name, + args.table_name, + args.source_file_name) diff --git a/samples/snippets/load_data_from_file_test.py b/samples/snippets/load_data_from_file_test.py new file mode 100644 index 000000000..9adc99dfe --- /dev/null +++ b/samples/snippets/load_data_from_file_test.py @@ -0,0 +1,34 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import load_data_from_file +import pytest + +DATASET_ID = 'test_dataset' +TABLE_ID = 'test_import_table' + + +@pytest.mark.xfail( + strict=True, + reason='https://github.com/GoogleCloudPlatform/gcloud-python/issues/2133') +def test_load_table(resource, capsys): + data_path = resource('data.csv') + + load_data_from_file.load_data_from_file( + DATASET_ID, + TABLE_ID, + data_path) + + out, _ = capsys.readouterr() + + assert 'Loaded 1 rows' in out diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py new file mode 100644 index 000000000..1a577be64 --- /dev/null +++ b/samples/snippets/load_data_from_gcs.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads data into BigQuery from an object in Google Cloud Storage. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python load_data_from_gcs.py example_dataset example_table \ + gs://example-bucket/example-data.csv + +The dataset and table should already exist. +""" + +import argparse +import time +import uuid + +from gcloud import bigquery + + +def load_data_from_gcs(dataset_name, table_name, source): + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + job_name = str(uuid.uuid4()) + + job = bigquery_client.load_table_from_storage( + job_name, table, source) + + job.begin() + + wait_for_job(job) + + print('Loaded {} rows into {}:{}.'.format( + job.output_rows, dataset_name, table_name)) + + +def wait_for_job(job): + while True: + job.reload() + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('dataset_name') + parser.add_argument('table_name') + parser.add_argument( + 'source', help='The Google Cloud Storage object to load. Must be in ' + 'the format gs://bucket_name/object_name') + + args = parser.parse_args() + + load_data_from_gcs( + args.dataset_name, + args.table_name, + args.source) diff --git a/samples/snippets/load_data_from_gcs_test.py b/samples/snippets/load_data_from_gcs_test.py new file mode 100644 index 000000000..2d1c66162 --- /dev/null +++ b/samples/snippets/load_data_from_gcs_test.py @@ -0,0 +1,31 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import load_data_from_gcs + +DATASET_ID = 'test_dataset' +TABLE_ID = 'test_import_table' + + +def test_load_table(cloud_config, capsys): + cloud_storage_input_uri = 'gs://{}/data.csv'.format( + cloud_config.storage_bucket) + + load_data_from_gcs.load_data_from_gcs( + DATASET_ID, + TABLE_ID, + cloud_storage_input_uri) + + out, _ = capsys.readouterr() + + assert 'Loaded 1 rows' in out diff --git a/samples/snippets/resources/data.csv b/samples/snippets/resources/data.csv new file mode 100644 index 000000000..230a96b55 --- /dev/null +++ b/samples/snippets/resources/data.csv @@ -0,0 +1 @@ +Gandalf, 2000, 140.0, 1 diff --git a/samples/snippets/resources/data.json b/samples/snippets/resources/data.json new file mode 100644 index 000000000..b8eef90c5 --- /dev/null +++ b/samples/snippets/resources/data.json @@ -0,0 +1 @@ +{"Name": "Gandalf", "Age": 2000, "Weight": 140.0, "IsMagic": true} diff --git a/samples/snippets/resources/schema.json b/samples/snippets/resources/schema.json new file mode 100644 index 000000000..a48971ef8 --- /dev/null +++ b/samples/snippets/resources/schema.json @@ -0,0 +1 @@ +[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}] \ No newline at end of file diff --git a/samples/snippets/stream_data.py b/samples/snippets/stream_data.py new file mode 100644 index 000000000..5df6be114 --- /dev/null +++ b/samples/snippets/stream_data.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads a single row of data directly into BigQuery. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python stream_data.py example_dataset example_table \ + '["Gandalf", 2000]' + +The dataset and table should already exist. +""" + +import argparse +import json +from pprint import pprint + +from gcloud import bigquery + + +def stream_data(dataset_name, table_name, json_data): + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + data = json.loads(json_data) + + # Reload the table to get the schema. + table.reload() + + rows = [data] + errors = table.insert_data(rows) + + if not errors: + print('Loaded 1 row into {}:{}'.format(dataset_name, table_name)) + else: + print('Errors:') + pprint(errors) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('dataset_name') + parser.add_argument('table_name') + parser.add_argument( + 'json_data', + help='The row to load into BigQuery as an array in JSON format.') + + args = parser.parse_args() + + stream_data( + args.dataset_name, + args.table_name, + args.json_data) diff --git a/samples/snippets/stream_data_test.py b/samples/snippets/stream_data_test.py new file mode 100644 index 000000000..f7d244b3a --- /dev/null +++ b/samples/snippets/stream_data_test.py @@ -0,0 +1,29 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import stream_data + + +DATASET_ID = 'test_dataset' +TABLE_ID = 'test_import_table' + + +def test_stream_data(resource, capsys): + stream_data.stream_data( + DATASET_ID, + TABLE_ID, + '["Gandalf", 2000]') + + out, _ = capsys.readouterr() + + assert 'Loaded 1 row' in out From baaa4a52687f35ca466742a5b5ef8d013303d039 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 19 Aug 2016 12:39:59 -0700 Subject: [PATCH 003/100] Add bigquery snippets Change-Id: I148b9f444ad5e481d4f091b53121873b51de191e --- samples/snippets/snippets.py | 172 ++++++++++++++++++++++++++++++ samples/snippets/snippets_test.py | 75 +++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 samples/snippets/snippets.py create mode 100644 samples/snippets/snippets_test.py diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py new file mode 100644 index 000000000..7c7a90c30 --- /dev/null +++ b/samples/snippets/snippets.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Samples that demonstrate basic operations in the BigQuery API. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python snippets.py list-datasets + +The dataset and table should already exist. +""" + +import argparse + +from gcloud import bigquery + + +def list_projects(): + raise NotImplementedError( + 'https://github.com/GoogleCloudPlatform/gcloud-python/issues/2143') + + +def list_datasets(project=None): + """Lists all datasets in a given project. + + If no project is specified, then the currently active project is used + """ + bigquery_client = bigquery.Client(project=project) + + datasets = [] + page_token = None + + while True: + results, page_token = bigquery_client.list_datasets( + page_token=page_token) + datasets.extend(results) + + if not page_token: + break + + for dataset in datasets: + print(dataset.name) + + +def list_tables(dataset_name, project=None): + """Lists all of the tables in a given dataset. + + If no project is specified, then the currently active project is used. + """ + bigquery_client = bigquery.Client(project=project) + dataset = bigquery_client.dataset(dataset_name) + + if not dataset.exists(): + print('Dataset {} does not exist.'.format(dataset_name)) + + tables = [] + page_token = None + + while True: + results, page_token = dataset.list_tables(page_token=page_token) + tables.extend(results) + + if not page_token: + break + + for table in tables: + print(table.name) + + +def list_rows(dataset_name, table_name, project=None): + """Prints rows in the given table. + + Will print 25 rows at most for brevity as tables can contain large amounts + of rows. + + If no project is specified, then the currently active project is used. + """ + bigquery_client = bigquery.Client(project=project) + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + + if not table.exists(): + print('Table {}:{} does not exist.'.format(dataset_name, table_name)) + + # Reload the table so that the schema is available. + table.reload() + + rows = [] + page_token = None + + while len(rows) < 25: + results, total_rows, page_token = table.fetch_data( + max_results=25, page_token=page_token) + rows.extend(results) + + if not page_token: + break + + # Use format to create a simple table. + format_string = '{:<16} ' * len(table.schema) + + # Print schema field names + field_names = [field.name for field in table.schema] + print(format_string.format(*field_names)) + + for row in rows: + print(format_string.format(*row)) + + +def delete_table(dataset_name, table_name, project=None): + """Deletes a table in a given dataset. + + If no project is specified, then the currently active project is used. + """ + bigquery_client = bigquery.Client(project=project) + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + + table.delete() + + print('Table {}:{} deleted.'.format(dataset_name, table_name)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--project', default=None) + + subparsers = parser.add_subparsers(dest='command') + + list_datasets_parser = subparsers.add_parser( + 'list-datasets', help=list_datasets.__doc__) + + list_tables_parser = subparsers.add_parser( + 'list-tables', help=list_tables.__doc__) + list_tables_parser.add_argument('dataset_name') + + list_rows_parser = subparsers.add_parser( + 'list-rows', help=list_rows.__doc__) + list_rows_parser.add_argument('dataset_name') + list_rows_parser.add_argument('table_name') + + delete_table_parser = subparsers.add_parser( + 'delete-table', help=delete_table.__doc__) + delete_table_parser.add_argument('dataset_name') + delete_table_parser.add_argument('table_name') + + args = parser.parse_args() + + if args.command == 'list-datasets': + list_datasets(args.project) + elif args.command == 'list-tables': + list_tables(args.dataset_name, args.project) + elif args.command == 'list-rows': + list_rows(args.dataset_name, args.table_name, args.project) + elif args.command == 'delete-table': + delete_table(args.dataset_name, args.table_name, args.project) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py new file mode 100644 index 000000000..e45484ef2 --- /dev/null +++ b/samples/snippets/snippets_test.py @@ -0,0 +1,75 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gcloud import bigquery +import pytest +import snippets + + +DATASET_ID = 'test_dataset' +TABLE_ID = 'test_import_table' + + +@pytest.mark.xfail( + strict=True, + reason='https://github.com/GoogleCloudPlatform/gcloud-python/issues/2143') +def test_list_projects(): + snippets.list_projects() + # No need to check the ouput, lack of exception is enough. + + +def test_list_datasets(capsys): + # Requires the dataset to have been created in the test project. + snippets.list_datasets() + + out, _ = capsys.readouterr() + + assert DATASET_ID in out + + +def test_list_tables(capsys): + # Requires teh dataset and table to have been created in the test project. + snippets.list_tables(DATASET_ID) + + out, _ = capsys.readouterr() + + assert TABLE_ID in out + + +def test_list_rows(capsys): + # Requires the dataset and table to have been created in the test project. + + # Check for the schema. It's okay if the table is empty as long as there + # aren't any errors. + + snippets.list_rows(DATASET_ID, TABLE_ID) + + out, _ = capsys.readouterr() + + assert 'Name' in out + assert 'Age' in out + + +def test_delete_table(capsys): + # Create a table to delete + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(DATASET_ID) + table = dataset.table('test_delete_table') + + if not table.exists(): + table.schema = [bigquery.SchemaField('id', 'INTEGER')] + table.create() + + snippets.delete_table(DATASET_ID, table.name) + + assert not table.exists() From 9e60f76bdad4ebf1eab44f713d8ab24a9951e470 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 19 Aug 2016 12:45:19 -0700 Subject: [PATCH 004/100] Remove unnecessary fixture Change-Id: I5e3f3faecfaf05749e7cd931d1d81758d3032651 --- samples/snippets/stream_data_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/stream_data_test.py b/samples/snippets/stream_data_test.py index f7d244b3a..25982b88a 100644 --- a/samples/snippets/stream_data_test.py +++ b/samples/snippets/stream_data_test.py @@ -18,7 +18,7 @@ TABLE_ID = 'test_import_table' -def test_stream_data(resource, capsys): +def test_stream_data(capsys): stream_data.stream_data( DATASET_ID, TABLE_ID, From 6f03bef56040320d6ae2ff90af90b707bf1f7855 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 19 Aug 2016 14:42:54 -0700 Subject: [PATCH 005/100] Address review comments Change-Id: I957fa992850ed3305dbcad5cd73ed61a7eedb63c --- samples/snippets/snippets.py | 2 ++ samples/snippets/snippets_test.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index 7c7a90c30..bcf534c8d 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -66,6 +66,7 @@ def list_tables(dataset_name, project=None): if not dataset.exists(): print('Dataset {} does not exist.'.format(dataset_name)) + return tables = [] page_token = None @@ -95,6 +96,7 @@ def list_rows(dataset_name, table_name, project=None): if not table.exists(): print('Table {}:{} does not exist.'.format(dataset_name, table_name)) + return # Reload the table so that the schema is available. table.reload() diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index e45484ef2..34faa277f 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -1,9 +1,10 @@ -# Copyright 2015, Google, Inc. +# Copyright 2016 Google Inc. All Rights Reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, From dbe847d9de16258db00040c7929f06eeb902558b Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 19 Aug 2016 14:55:30 -0700 Subject: [PATCH 006/100] Fix lint Change-Id: If7b90925c51b739a293fad737c8ab6919b3f7eae --- samples/snippets/load_data_from_file_test.py | 3 ++- samples/snippets/snippets_test.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/snippets/load_data_from_file_test.py b/samples/snippets/load_data_from_file_test.py index 9adc99dfe..eccefe038 100644 --- a/samples/snippets/load_data_from_file_test.py +++ b/samples/snippets/load_data_from_file_test.py @@ -11,9 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import load_data_from_file import pytest +import load_data_from_file + DATASET_ID = 'test_dataset' TABLE_ID = 'test_import_table' diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 34faa277f..0a52922b9 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -14,6 +14,7 @@ from gcloud import bigquery import pytest + import snippets From 09c139b31ec3b98acb3a50ac89b11ef9a2439a94 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 22 Aug 2016 10:34:15 -0700 Subject: [PATCH 007/100] Address post-review comments for BigQuery samples Change-Id: I74627bc2744d3b0946d34c1d2a04c424ed708e87 --- samples/snippets/snippets.py | 3 +++ samples/snippets/snippets_test.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index bcf534c8d..49272965b 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -104,6 +104,9 @@ def list_rows(dataset_name, table_name, project=None): rows = [] page_token = None + # Load at most 25 results. You can change this to `while True` and change + # the max_results argument to load more rows from BigQuery, but note + # that this can take some time. It's preferred to use a query. while len(rows) < 25: results, total_rows, page_token = table.fetch_data( max_results=25, page_token=page_token) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 0a52922b9..ed2f47639 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -40,7 +40,7 @@ def test_list_datasets(capsys): def test_list_tables(capsys): - # Requires teh dataset and table to have been created in the test project. + # Requires the dataset and table to have been created in the test project. snippets.list_tables(DATASET_ID) out, _ = capsys.readouterr() From 39e556ac5916719725a9869b70cc94005f5a6647 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 12 Sep 2016 12:59:26 -0700 Subject: [PATCH 008/100] Add bigquery create and copy table examples [(#514)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/514) * Add bigquery create table sample * Add copy table sample * Fix test table ids --- samples/snippets/export_data_to_gcs_test.py | 2 +- samples/snippets/snippets.py | 88 +++++++++++++++++++++ samples/snippets/snippets_test.py | 39 ++++++++- 3 files changed, 126 insertions(+), 3 deletions(-) diff --git a/samples/snippets/export_data_to_gcs_test.py b/samples/snippets/export_data_to_gcs_test.py index e260e47b4..acbbe50e5 100644 --- a/samples/snippets/export_data_to_gcs_test.py +++ b/samples/snippets/export_data_to_gcs_test.py @@ -15,7 +15,7 @@ DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' +TABLE_ID = 'test_table' def test_export_data_to_gcs(cloud_config, capsys): diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index 49272965b..d201a84cb 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -25,8 +25,11 @@ """ import argparse +import time +import uuid from gcloud import bigquery +import gcloud.bigquery.job def list_projects(): @@ -82,6 +85,32 @@ def list_tables(dataset_name, project=None): print(table.name) +def create_table(dataset_name, table_name, project=None): + """Creates a simple table in the given dataset. + + If no project is specified, then the currently active project is used. + """ + bigquery_client = bigquery.Client(project=project) + dataset = bigquery_client.dataset(dataset_name) + + if not dataset.exists(): + print('Dataset {} does not exist.'.format(dataset_name)) + return + + table = dataset.table(table_name) + + # Set the table schema + table.schema = ( + bigquery.SchemaField('Name', 'STRING'), + bigquery.SchemaField('Age', 'INTEGER'), + bigquery.SchemaField('Weight', 'FLOAT'), + ) + + table.create() + + print('Created table {} in dataset {}.'.format(table_name, dataset_name)) + + def list_rows(dataset_name, table_name, project=None): """Prints rows in the given table. @@ -126,6 +155,50 @@ def list_rows(dataset_name, table_name, project=None): print(format_string.format(*row)) +def copy_table(dataset_name, table_name, new_table_name, project=None): + """Copies a table. + + If no project is specified, then the currently active project is used. + """ + bigquery_client = bigquery.Client(project=project) + dataset = bigquery_client.dataset(dataset_name) + table = dataset.table(table_name) + + # This sample shows the destination table in the same dataset and project, + # however, it's possible to copy across datasets and projects. You can + # also copy muliple source tables into a single destination table by + # providing addtional arguments to `copy_table`. + destination_table = dataset.table(new_table_name) + + # Create a job to copy the table to the destination table. + job_id = str(uuid.uuid4()) + job = bigquery_client.copy_table( + job_id, destination_table, table) + + # Create the table if it doesn't exist. + job.create_disposition = ( + gcloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED) + + # Start the job. + job.begin() + + # Wait for the the job to finish. + print('Waiting for job to finish...') + wait_for_job(job) + + print('Table {} copied to {}.'.format(table_name, new_table_name)) + + +def wait_for_job(job): + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + def delete_table(dataset_name, table_name, project=None): """Deletes a table in a given dataset. @@ -155,11 +228,22 @@ def delete_table(dataset_name, table_name, project=None): 'list-tables', help=list_tables.__doc__) list_tables_parser.add_argument('dataset_name') + create_table_parser = subparsers.add_parser( + 'create-table', help=create_table.__doc__) + create_table_parser.add_argument('dataset_name') + create_table_parser.add_argument('table_name') + list_rows_parser = subparsers.add_parser( 'list-rows', help=list_rows.__doc__) list_rows_parser.add_argument('dataset_name') list_rows_parser.add_argument('table_name') + copy_table_parser = subparsers.add_parser( + 'copy-table', help=copy_table.__doc__) + copy_table_parser.add_argument('dataset_name') + copy_table_parser.add_argument('table_name') + copy_table_parser.add_argument('new_table_name') + delete_table_parser = subparsers.add_parser( 'delete-table', help=delete_table.__doc__) delete_table_parser.add_argument('dataset_name') @@ -171,7 +255,11 @@ def delete_table(dataset_name, table_name, project=None): list_datasets(args.project) elif args.command == 'list-tables': list_tables(args.dataset_name, args.project) + elif args.command == 'create-table': + create_table(args.dataset_name, args.table_name, args.project) elif args.command == 'list-rows': list_rows(args.dataset_name, args.table_name, args.project) + elif args.command == 'copy-table': + copy_table(args.dataset_name, args.table_name, args.new_table_name) elif args.command == 'delete-table': delete_table(args.dataset_name, args.table_name, args.project) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index ed2f47639..372cbc834 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -19,7 +19,7 @@ DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' +TABLE_ID = 'test_table' @pytest.mark.xfail( @@ -62,7 +62,42 @@ def test_list_rows(capsys): assert 'Age' in out -def test_delete_table(capsys): +@pytest.fixture +def temporary_table(): + """Fixture that returns a factory for tables that do not yet exist and + will be automatically deleted after the test.""" + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(DATASET_ID) + tables = [] + + def factory(table_name): + new_table = dataset.table(table_name) + if new_table.exists(): + new_table.delete() + tables.append(new_table) + return new_table + + yield factory + + for table in tables: + if table.exists(): + table.delete() + + +def test_create_table(temporary_table): + new_table = temporary_table('test_create_table') + snippets.create_table(DATASET_ID, new_table.name) + assert new_table.exists() + + +@pytest.mark.slow +def test_copy_table(temporary_table): + new_table = temporary_table('test_copy_table') + snippets.copy_table(DATASET_ID, TABLE_ID, new_table.name) + assert new_table.exists() + + +def test_delete_table(): # Create a table to delete bigquery_client = bigquery.Client() dataset = bigquery_client.dataset(DATASET_ID) From dd47c915ee26a3620ad355152516696a69630978 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 23 Sep 2016 09:48:46 -0700 Subject: [PATCH 009/100] Auto-update dependencies. [(#540)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/540) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 2beeafe63..dfb42aaaa 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -gcloud==0.18.1 +gcloud==0.18.2 From 72defef09d21b466369fc41112b870efc0a11669 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 26 Sep 2016 11:34:45 -0700 Subject: [PATCH 010/100] Auto-update dependencies. [(#542)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/542) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index dfb42aaaa..97a207d3a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -gcloud==0.18.2 +gcloud==0.18.3 From 9f01c6b63b004e2c9ade82bea267d9b80dbacd38 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 29 Sep 2016 20:51:47 -0700 Subject: [PATCH 011/100] Move to google-cloud [(#544)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/544) --- samples/snippets/async_query.py | 2 +- samples/snippets/export_data_to_gcs.py | 2 +- samples/snippets/load_data_from_file.py | 2 +- samples/snippets/load_data_from_gcs.py | 2 +- samples/snippets/requirements.txt | 2 +- samples/snippets/snippets.py | 8 ++++---- samples/snippets/snippets_test.py | 2 +- samples/snippets/stream_data.py | 2 +- samples/snippets/sync_query.py | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py index 0ca324240..37192d156 100755 --- a/samples/snippets/async_query.py +++ b/samples/snippets/async_query.py @@ -27,7 +27,7 @@ import time import uuid -from gcloud import bigquery +from google.cloud import bigquery def async_query(query): diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py index c9771ea1b..e9037ee01 100644 --- a/samples/snippets/export_data_to_gcs.py +++ b/samples/snippets/export_data_to_gcs.py @@ -29,7 +29,7 @@ import time import uuid -from gcloud import bigquery +from google.cloud import bigquery def export_data_to_gcs(dataset_name, table_name, destination): diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index cbb015347..b4f851f1a 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -27,7 +27,7 @@ import argparse import time -from gcloud import bigquery +from google.cloud import bigquery def load_data_from_file(dataset_name, table_name, source_file_name): diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py index 1a577be64..4aa435fad 100644 --- a/samples/snippets/load_data_from_gcs.py +++ b/samples/snippets/load_data_from_gcs.py @@ -29,7 +29,7 @@ import time import uuid -from gcloud import bigquery +from google.cloud import bigquery def load_data_from_gcs(dataset_name, table_name, source): diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 97a207d3a..11a303264 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -gcloud==0.18.3 +google-cloud-bigquery==0.20.0 diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index d201a84cb..0e1f5d4b5 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -28,8 +28,8 @@ import time import uuid -from gcloud import bigquery -import gcloud.bigquery.job +from google.cloud import bigquery +import google.cloud.bigquery.job def list_projects(): @@ -145,7 +145,7 @@ def list_rows(dataset_name, table_name, project=None): break # Use format to create a simple table. - format_string = '{:<16} ' * len(table.schema) + format_string = '{!s:<16} ' * len(table.schema) # Print schema field names field_names = [field.name for field in table.schema] @@ -177,7 +177,7 @@ def copy_table(dataset_name, table_name, new_table_name, project=None): # Create the table if it doesn't exist. job.create_disposition = ( - gcloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED) + google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED) # Start the job. job.begin() diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 372cbc834..35f79af7b 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcloud import bigquery +from google.cloud import bigquery import pytest import snippets diff --git a/samples/snippets/stream_data.py b/samples/snippets/stream_data.py index 5df6be114..a90d432bf 100644 --- a/samples/snippets/stream_data.py +++ b/samples/snippets/stream_data.py @@ -29,7 +29,7 @@ import json from pprint import pprint -from gcloud import bigquery +from google.cloud import bigquery def stream_data(dataset_name, table_name, json_data): diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py index 59007b537..f21270ed0 100755 --- a/samples/snippets/sync_query.py +++ b/samples/snippets/sync_query.py @@ -26,7 +26,7 @@ import argparse # [START sync_query] -from gcloud import bigquery +from google.cloud import bigquery def sync_query(query): From af0b251bf1d2e7c89a5e3f8f738efe0d1077a239 Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Wed, 5 Oct 2016 09:56:04 -0700 Subject: [PATCH 012/100] Add new "quickstart" samples [(#547)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/547) --- samples/snippets/quickstart.py | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 samples/snippets/quickstart.py diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py new file mode 100644 index 000000000..2c9923f6e --- /dev/null +++ b/samples/snippets/quickstart.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START bigquery_quickstart] + # Imports the Google Cloud client library + from google.cloud import bigquery + + # Instantiates a client + bigquery_client = bigquery.Client() + + # The name for the new dataset + dataset_name = 'my_new_dataset' + + # Prepares the new dataset + dataset = bigquery_client.dataset(dataset_name) + + # Creates the new dataset + dataset.create() + + print('Dataset {} created.'.format(dataset.name)) + # [END bigquery_quickstart] + + +if __name__ == '__main__': + run_quickstart() From b3b063a4ddb41ff7db3bd8c49bda95626179ec54 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 5 Oct 2016 11:25:06 -0700 Subject: [PATCH 013/100] BigQuery - use new QueryJob.results() method. [(#555)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/555) This method was added in https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2083. I can remove my hack now. --- samples/snippets/async_query.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py index 37192d156..2531c6152 100755 --- a/samples/snippets/async_query.py +++ b/samples/snippets/async_query.py @@ -30,6 +30,16 @@ from google.cloud import bigquery +def wait_for_job(job): + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + def async_query(query): client = bigquery.Client() query_job = client.run_async_query(str(uuid.uuid4()), query) @@ -38,16 +48,8 @@ def async_query(query): wait_for_job(query_job) - # Manually construct the QueryResults. - # TODO: The client library will provide a helper method that does this. - # https://github.com/GoogleCloudPlatform/gcloud-python/issues/2083 - query_results = bigquery.query.QueryResults('', client) - query_results._properties['jobReference'] = { - 'jobId': query_job.name, - 'projectId': query_job.project - } - # Drain the query results by requesting a page at a time. + query_results = query_job.results() page_token = None while True: @@ -62,16 +64,6 @@ def async_query(query): break -def wait_for_job(job): - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.error_result) - return - time.sleep(1) - - if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, From 5b2f9bfc856132f2ccca71fc316e6979dc81d9ea Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 12 Oct 2016 10:48:57 -0700 Subject: [PATCH 014/100] Quickstart tests [(#569)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/569) * Add tests for quickstarts * Update secrets --- samples/snippets/quickstart_test.py | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 samples/snippets/quickstart_test.py diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py new file mode 100644 index 000000000..f5842960c --- /dev/null +++ b/samples/snippets/quickstart_test.py @@ -0,0 +1,45 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + +import quickstart + + +# Must match the dataset listed in quickstart.py (there's no easy way to +# extract this). +DATASET_ID = 'my_new_dataset' + + +@pytest.fixture +def temporary_dataset(): + """Fixture that ensures the test dataset does not exist before or + after a test.""" + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(DATASET_ID) + + if dataset.exists(): + dataset.delete() + + yield + + if dataset.exists(): + dataset.delete() + + +def test_quickstart(capsys, temporary_dataset): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert DATASET_ID in out From 2140994d55fe85890e6a6b3e77ca111a1860ad8d Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 24 Oct 2016 11:03:17 -0700 Subject: [PATCH 015/100] Generate readmes for most service samples [(#599)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/599) --- samples/snippets/README.md | 5 - samples/snippets/README.rst | 332 ++++++++++++++++++++++++++++++++ samples/snippets/README.rst.in | 43 +++++ samples/snippets/async_query.py | 2 +- samples/snippets/sync_query.py | 2 +- 5 files changed, 377 insertions(+), 7 deletions(-) delete mode 100644 samples/snippets/README.md create mode 100644 samples/snippets/README.rst create mode 100644 samples/snippets/README.rst.in diff --git a/samples/snippets/README.md b/samples/snippets/README.md deleted file mode 100644 index 9c1f9b9fb..000000000 --- a/samples/snippets/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# BigQuery Google Cloud Client Library Samples - - - - diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst new file mode 100644 index 000000000..3fbb1716f --- /dev/null +++ b/samples/snippets/README.rst @@ -0,0 +1,332 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google BigQuery Python Samples +=============================================================================== + +This directory contains samples for Google BigQuery. `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on analyzing data to find meaningful insights, use familiar SQL, and take advantage of our pay-as-you-go model. + + + + +.. _Google BigQuery: https://cloud.google.com/bigquery/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +Authentication is typically done through `Application Default Credentials`_, +which means you do not have to change the code to authenticate as long as +your environment has credentials. You have a few options for setting up +authentication: + +#. When running locally, use the `Google Cloud SDK`_ + + .. code-block:: bash + + gcloud beta auth application-default login + + +#. When running on App Engine or Compute Engine, credentials are already + set-up. However, you may need to configure your Compute Engine instance + with `additional scopes`_. + +#. You can create a `Service Account key file`_. This file can be used to + authenticate to Google Cloud Platform services from any environment. To use + the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to + the path to the key file, for example: + + .. code-block:: bash + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json + +.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow +.. _additional scopes: https://cloud.google.com/compute/docs/authentication#using +.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount + +Install Dependencies +++++++++++++++++++++ + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Sync query ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python sync_query.py + + usage: sync_query.py [-h] query + + Command-line application to perform synchronous queries in BigQuery. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python sync_query.py \ + 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' + + positional arguments: + query BigQuery SQL Query. + + optional arguments: + -h, --help show this help message and exit + + +Async query ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python async_query.py + + usage: async_query.py [-h] query + + Command-line application to perform asynchronous queries in BigQuery. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python async_query.py 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' + + positional arguments: + query BigQuery SQL Query. + + optional arguments: + -h, --help show this help message and exit + + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] [--project PROJECT] + {list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} + ... + + Samples that demonstrate basic operations in the BigQuery API. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python snippets.py list-datasets + + The dataset and table should already exist. + + positional arguments: + {list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} + list-datasets Lists all datasets in a given project. If no project + is specified, then the currently active project is + used + list-tables Lists all of the tables in a given dataset. If no + project is specified, then the currently active + project is used. + create-table Creates a simple table in the given dataset. If no + project is specified, then the currently active + project is used. + list-rows Prints rows in the given table. Will print 25 rows at + most for brevity as tables can contain large amounts + of rows. If no project is specified, then the + currently active project is used. + copy-table Copies a table. If no project is specified, then the + currently active project is used. + delete-table Deletes a table in a given dataset. If no project is + specified, then the currently active project is used. + + optional arguments: + -h, --help show this help message and exit + --project PROJECT + + +Load data from a file ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python load_data_from_file.py + + usage: load_data_from_file.py [-h] dataset_name table_name source_file_name + + Loads data into BigQuery from a local file. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python load_data_from_file.py example_dataset example_table example-data.csv + + The dataset and table should already exist. + + positional arguments: + dataset_name + table_name + source_file_name Path to a .csv file to upload. + + optional arguments: + -h, --help show this help message and exit + + +Load data from Cloud Storage ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python load_data_from_gcs.py + + usage: load_data_from_gcs.py [-h] dataset_name table_name source + + Loads data into BigQuery from an object in Google Cloud Storage. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python load_data_from_gcs.py example_dataset example_table gs://example-bucket/example-data.csv + + The dataset and table should already exist. + + positional arguments: + dataset_name + table_name + source The Google Cloud Storage object to load. Must be in the format + gs://bucket_name/object_name + + optional arguments: + -h, --help show this help message and exit + + +Load streaming data ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python stream_data.py + + usage: stream_data.py [-h] dataset_name table_name json_data + + Loads a single row of data directly into BigQuery. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python stream_data.py example_dataset example_table '["Gandalf", 2000]' + + The dataset and table should already exist. + + positional arguments: + dataset_name + table_name + json_data The row to load into BigQuery as an array in JSON format. + + optional arguments: + -h, --help show this help message and exit + + +Export data to Cloud Storage ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python export_data_to_gcs.py + + usage: export_data_to_gcs.py [-h] dataset_name table_name destination + + Exports data from BigQuery to an object in Google Cloud Storage. + + For more information, see the README.md under /bigquery. + + Example invocation: + $ python export_data_to_gcs.py example_dataset example_table gs://example-bucket/example-data.csv + + The dataset and table should already exist. + + positional arguments: + dataset_name + table_name + destination The desintation Google Cloud Storage object.Must be in the + format gs://bucket_name/object_name + + optional arguments: + -h, --help show this help message and exit + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in new file mode 100644 index 000000000..49143f062 --- /dev/null +++ b/samples/snippets/README.rst.in @@ -0,0 +1,43 @@ +# This file is used to generate README.rst + +product: + name: Google BigQuery + short_name: BigQuery + url: https://cloud.google.com/bigquery/docs + description: > + `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost + analytics data warehouse. BigQuery is NoOps—there is no infrastructure to + manage and you don't need a database administrator—so you can focus on + analyzing data to find meaningful insights, use familiar SQL, and take + advantage of our pay-as-you-go model. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Sync query + file: sync_query.py + show_help: true +- name: Async query + file: async_query.py + show_help: true +- name: Snippets + file: snippets.py + show_help: true +- name: Load data from a file + file: load_data_from_file.py + show_help: true +- name: Load data from Cloud Storage + file: load_data_from_gcs.py + show_help: true +- name: Load streaming data + file: stream_data.py + show_help: true +- name: Export data to Cloud Storage + file: export_data_to_gcs.py + show_help: true + +cloud_client_library: true diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py index 2531c6152..aa3397351 100755 --- a/samples/snippets/async_query.py +++ b/samples/snippets/async_query.py @@ -19,7 +19,7 @@ For more information, see the README.md under /bigquery. Example invocation: - $ python async_query.py \ + $ python async_query.py \\ 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' """ diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py index f21270ed0..37c8fea8a 100755 --- a/samples/snippets/sync_query.py +++ b/samples/snippets/sync_query.py @@ -19,7 +19,7 @@ For more information, see the README.md under /bigquery. Example invocation: - $ python sync_query.py \ + $ python sync_query.py \\ 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' """ From 2ce6397506b51392e9e973d4b42a581869d77788 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 25 Oct 2016 10:54:45 -0700 Subject: [PATCH 016/100] Generate most non-appengine readmes Change-Id: I3779282126cdd05b047194d356932b9995484115 --- samples/snippets/README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 3fbb1716f..be037a3b8 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -128,7 +128,8 @@ To run this sample: For more information, see the README.md under /bigquery. Example invocation: - $ python async_query.py 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' + $ python async_query.py \ + 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' positional arguments: query BigQuery SQL Query. From f4aca0245bc1cdd1a7348e0677a911c969409560 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 12:16:03 -0700 Subject: [PATCH 017/100] Add missing bigquery samples [(#622)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/622) --- samples/snippets/snippets.py | 46 ++++++++++++++++++++++++++++--- samples/snippets/snippets_test.py | 26 +++++++++++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index 0e1f5d4b5..f7ab33731 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -33,14 +33,27 @@ def list_projects(): - raise NotImplementedError( - 'https://github.com/GoogleCloudPlatform/gcloud-python/issues/2143') + bigquery_client = bigquery.Client() + + projects = [] + page_token = None + + while True: + results, page_token = bigquery_client.list_projects( + page_token=page_token) + projects.extend(results) + + if not page_token: + break + + for project in projects: + print(project.project_id) def list_datasets(project=None): """Lists all datasets in a given project. - If no project is specified, then the currently active project is used + If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) @@ -59,6 +72,20 @@ def list_datasets(project=None): print(dataset.name) +def create_dataset(dataset_name, project=None): + """Craetes a dataset in a given project. + + If no project is specified, then the currently active project is used. + """ + bigquery_client = bigquery.Client(project=project) + + dataset = bigquery_client.dataset(dataset_name) + + dataset.create() + + print('Created dataset {}.'.format(dataset_name)) + + def list_tables(dataset_name, project=None): """Lists all of the tables in a given dataset. @@ -221,9 +248,16 @@ def delete_table(dataset_name, table_name, project=None): subparsers = parser.add_subparsers(dest='command') + list_projects_parser = subparsers.add_parser( + 'list-projects', help=list_projects.__doc__) + list_datasets_parser = subparsers.add_parser( 'list-datasets', help=list_datasets.__doc__) + create_dataset_parser = subparsers.add_parser( + 'list-datasets', help=list_datasets.__doc__) + create_dataset_parser.add_argument('dataset_name') + list_tables_parser = subparsers.add_parser( 'list-tables', help=list_tables.__doc__) list_tables_parser.add_argument('dataset_name') @@ -251,8 +285,12 @@ def delete_table(dataset_name, table_name, project=None): args = parser.parse_args() - if args.command == 'list-datasets': + if args.command == 'list-projects': + list_projects() + elif args.command == 'list-datasets': list_datasets(args.project) + elif args.command == 'create-dataset': + create_dataset(args.dataset_name, args.project) elif args.command == 'list-tables': list_tables(args.dataset_name, args.project) elif args.command == 'create-table': diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 35f79af7b..af368d9a4 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -22,9 +22,6 @@ TABLE_ID = 'test_table' -@pytest.mark.xfail( - strict=True, - reason='https://github.com/GoogleCloudPlatform/gcloud-python/issues/2143') def test_list_projects(): snippets.list_projects() # No need to check the ouput, lack of exception is enough. @@ -39,6 +36,29 @@ def test_list_datasets(capsys): assert DATASET_ID in out +@pytest.fixture +def cleanup_dataset(): + dataset_name = 'test_temporary_dataset' + bigquery_client = bigquery.Client() + dataset = bigquery_client.dataset(dataset_name) + + if dataset.exists(): + dataset.delete() + + yield dataset_name + + if dataset.exists(): + dataset.delete() + + +def test_create_dataset(capsys, cleanup_dataset): + snippets.create_dataset(cleanup_dataset) + + out, _ = capsys.readouterr() + + assert cleanup_dataset in out + + def test_list_tables(capsys): # Requires the dataset and table to have been created in the test project. snippets.list_tables(DATASET_ID) From ab4d7b74baa8369ce686e605c081ff7da1c03ba9 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 15 Nov 2016 14:58:27 -0800 Subject: [PATCH 018/100] Update samples to support latest Google Cloud Python [(#656)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/656) --- samples/snippets/requirements.txt | 2 +- samples/snippets/snippets.py | 55 ++++--------------------------- 2 files changed, 8 insertions(+), 49 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 11a303264..d9f269ca7 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-bigquery==0.20.0 +google-cloud-bigquery==0.21.0 diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index f7ab33731..b02ac8ce6 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -35,18 +35,7 @@ def list_projects(): bigquery_client = bigquery.Client() - projects = [] - page_token = None - - while True: - results, page_token = bigquery_client.list_projects( - page_token=page_token) - projects.extend(results) - - if not page_token: - break - - for project in projects: + for project in bigquery_client.list_projects(): print(project.project_id) @@ -57,18 +46,7 @@ def list_datasets(project=None): """ bigquery_client = bigquery.Client(project=project) - datasets = [] - page_token = None - - while True: - results, page_token = bigquery_client.list_datasets( - page_token=page_token) - datasets.extend(results) - - if not page_token: - break - - for dataset in datasets: + for dataset in bigquery_client.list_datasets(): print(dataset.name) @@ -98,17 +76,7 @@ def list_tables(dataset_name, project=None): print('Dataset {} does not exist.'.format(dataset_name)) return - tables = [] - page_token = None - - while True: - results, page_token = dataset.list_tables(page_token=page_token) - tables.extend(results) - - if not page_token: - break - - for table in tables: + for table in dataset.list_tables(): print(table.name) @@ -157,19 +125,10 @@ def list_rows(dataset_name, table_name, project=None): # Reload the table so that the schema is available. table.reload() - rows = [] - page_token = None - - # Load at most 25 results. You can change this to `while True` and change - # the max_results argument to load more rows from BigQuery, but note - # that this can take some time. It's preferred to use a query. - while len(rows) < 25: - results, total_rows, page_token = table.fetch_data( - max_results=25, page_token=page_token) - rows.extend(results) - - if not page_token: - break + # Load at most 25 results. You can change the max_results argument to load + # more rows from BigQuery, but note that this can take some time. It's + # preferred to use a query. + rows = list(table.fetch_data(max_results=25)) # Use format to create a simple table. format_string = '{!s:<16} ' * len(table.schema) From 884d136901d3a7d1656cf8c0ecb42c4d9ca60c98 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 15 Nov 2016 15:05:13 -0800 Subject: [PATCH 019/100] Update readmes Change-Id: Ie385fd8105325c6f2754b737e0f11c84254bcb47 --- samples/snippets/README.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index be037a3b8..1824609dd 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -150,7 +150,7 @@ To run this sample: $ python snippets.py usage: snippets.py [-h] [--project PROJECT] - {list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} + {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} ... Samples that demonstrate basic operations in the BigQuery API. @@ -163,10 +163,14 @@ To run this sample: The dataset and table should already exist. positional arguments: - {list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} + {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} + list-projects list-datasets Lists all datasets in a given project. If no project is specified, then the currently active project is - used + used. + list-datasets Lists all datasets in a given project. If no project + is specified, then the currently active project is + used. list-tables Lists all of the tables in a given dataset. If no project is specified, then the currently active project is used. From 59d5ce0022a165b129230185f33e0982f0878e46 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 30 Nov 2016 09:49:50 -0800 Subject: [PATCH 020/100] Fix bigquery load from file sample Change-Id: I5bcb3d41bed33f638e5dba13da3335adfbdc2ead --- samples/snippets/load_data_from_file.py | 2 -- samples/snippets/load_data_from_file_test.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index b4f851f1a..671752531 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -44,8 +44,6 @@ def load_data_from_file(dataset_name, table_name, source_file_name): job = table.upload_from_file( source_file, source_format='text/csv') - job.begin() - wait_for_job(job) print('Loaded {} rows into {}:{}.'.format( diff --git a/samples/snippets/load_data_from_file_test.py b/samples/snippets/load_data_from_file_test.py index eccefe038..434bbd1d8 100644 --- a/samples/snippets/load_data_from_file_test.py +++ b/samples/snippets/load_data_from_file_test.py @@ -11,17 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - import load_data_from_file DATASET_ID = 'test_dataset' TABLE_ID = 'test_import_table' -@pytest.mark.xfail( - strict=True, - reason='https://github.com/GoogleCloudPlatform/gcloud-python/issues/2133') def test_load_table(resource, capsys): data_path = resource('data.csv') From bd8f47cb3e9f210ded24d59e9605761e91187f54 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 30 Nov 2016 09:55:03 -0800 Subject: [PATCH 021/100] Fix lint Change-Id: If721ff085eb502b679a8decd3a39ac112425a114 --- samples/snippets/load_data_from_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index 671752531..80018f3fc 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -27,6 +27,7 @@ import argparse import time + from google.cloud import bigquery From 3d3475cf635312252408a23aa299e0003089713b Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Wed, 30 Nov 2016 13:18:30 -0800 Subject: [PATCH 022/100] Adjusted error handling based on Googler feedback [(#693)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/693) --- samples/snippets/load_data_from_gcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py index 4aa435fad..7c576e20a 100644 --- a/samples/snippets/load_data_from_gcs.py +++ b/samples/snippets/load_data_from_gcs.py @@ -54,7 +54,7 @@ def wait_for_job(job): job.reload() if job.state == 'DONE': if job.error_result: - raise RuntimeError(job.error_result) + raise RuntimeError(job.errors) return time.sleep(1) From cff9fb78ba80c04b386146b468f0f769d07b77fa Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Wed, 30 Nov 2016 14:14:36 -0800 Subject: [PATCH 023/100] Changed error handling for all other big query samples as well [(#694)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/694) --- samples/snippets/async_query.py | 2 +- samples/snippets/export_data_to_gcs.py | 2 +- samples/snippets/load_data_from_file.py | 2 +- samples/snippets/snippets.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py index aa3397351..f90c8f255 100755 --- a/samples/snippets/async_query.py +++ b/samples/snippets/async_query.py @@ -35,7 +35,7 @@ def wait_for_job(job): job.reload() # Refreshes the state via a GET request. if job.state == 'DONE': if job.error_result: - raise RuntimeError(job.error_result) + raise RuntimeError(job.errors) return time.sleep(1) diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py index e9037ee01..3aee442c1 100644 --- a/samples/snippets/export_data_to_gcs.py +++ b/samples/snippets/export_data_to_gcs.py @@ -54,7 +54,7 @@ def wait_for_job(job): job.reload() if job.state == 'DONE': if job.error_result: - raise RuntimeError(job.error_result) + raise RuntimeError(job.errors) return time.sleep(1) diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index 80018f3fc..0bbdd7ba7 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -56,7 +56,7 @@ def wait_for_job(job): job.reload() if job.state == 'DONE': if job.error_result: - raise RuntimeError(job.error_result) + raise RuntimeError(job.errors) return time.sleep(1) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index b02ac8ce6..f4294123d 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -180,7 +180,7 @@ def wait_for_job(job): job.reload() # Refreshes the state via a GET request. if job.state == 'DONE': if job.error_result: - raise RuntimeError(job.error_result) + raise RuntimeError(job.errors) return time.sleep(1) From f45c6bf2bc19969c350f840cffcb119d636af988 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 6 Dec 2016 13:56:45 -0800 Subject: [PATCH 024/100] Add bigquery shakespeare example [(#604)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/604) --- samples/snippets/requirements.txt | 2 +- samples/snippets/simple_app.py | 59 +++++++++++++++++++++++++++++ samples/snippets/simple_app_test.py | 21 ++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/simple_app.py create mode 100644 samples/snippets/simple_app_test.py diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index d9f269ca7..7523b0dac 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-bigquery==0.21.0 +google-cloud-bigquery==0.22.0 diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py new file mode 100644 index 000000000..7180c4fda --- /dev/null +++ b/samples/snippets/simple_app.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Simple application that performs a query with BigQuery.""" +# [START all] +# [START create_client] +from google.cloud import bigquery + + +def query_shakespeare(): + client = bigquery.Client() + # [END create_client] + # [START run_query] + query_results = client.run_sync_query(""" + SELECT + APPROX_TOP_COUNT(corpus, 10) as title, + COUNT(*) as unique_words + FROM `publicdata.samples.shakespeare`;""") + + # Use standard SQL syntax for queries. + # See: https://cloud.google.com/bigquery/sql-reference/ + query_results.use_legacy_sql = False + + query_results.run() + # [END run_query] + + # [START print_results] + # Drain the query results by requesting a page at a time. + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break + # [END print_results] + + +if __name__ == '__main__': + query_shakespeare() +# [END all] diff --git a/samples/snippets/simple_app_test.py b/samples/snippets/simple_app_test.py new file mode 100644 index 000000000..3733bf6ef --- /dev/null +++ b/samples/snippets/simple_app_test.py @@ -0,0 +1,21 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import simple_app + + +def test_query_shakespeare(capsys): + simple_app.query_shakespeare() + out, _ = capsys.readouterr() + assert 'hamlet' in out From 961f326d08c7dad35ce25b33bc90688a20a83053 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 6 Dec 2016 15:01:30 -0800 Subject: [PATCH 025/100] BigQuery parameterized query sample [(#699)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/699) Available in latest version of google-cloud-bigquery (0.22.0), so upgrades the requirement, too. --- samples/snippets/sync_query_params.py | 120 +++++++++++++++++++++ samples/snippets/sync_query_params_test.py | 31 ++++++ 2 files changed, 151 insertions(+) create mode 100644 samples/snippets/sync_query_params.py create mode 100644 samples/snippets/sync_query_params_test.py diff --git a/samples/snippets/sync_query_params.py b/samples/snippets/sync_query_params.py new file mode 100644 index 000000000..41ad94052 --- /dev/null +++ b/samples/snippets/sync_query_params.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line app to perform synchronous queries with parameters in BigQuery. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python sync_query_params.py --use-named-params 'romeoandjuliet' 100 + $ python sync_query_params.py --use-positional-params 'romeoandjuliet' 100 +""" + +import argparse + +from google.cloud import bigquery + + +def print_results(query_results): + """Print the query results by requesting a page at a time.""" + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break + + +def sync_query_positional_params(corpus, min_word_count): + client = bigquery.Client() + query_results = client.run_sync_query( + """SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = ? + AND word_count >= ? + ORDER BY word_count DESC; + """, + query_parameters=( + bigquery.ScalarQueryParameter( + # Set the name to None to use positional parameters (? symbol + # in the query). Note that you cannot mix named and positional + # parameters. + None, + 'STRING', + corpus), + bigquery.ScalarQueryParameter(None, 'INT64', min_word_count))) + + # Only standard SQL syntax supports parameters in queries. + # See: https://cloud.google.com/bigquery/sql-reference/ + query_results.use_legacy_sql = False + query_results.run() + print_results(query_results) + + +def sync_query_named_params(corpus, min_word_count): + client = bigquery.Client() + query_results = client.run_sync_query( + """SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus + AND word_count >= @min_word_count + ORDER BY word_count DESC; + """, + query_parameters=( + bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), + bigquery.ScalarQueryParameter( + 'min_word_count', + 'INT64', + min_word_count))) + query_results.use_legacy_sql = False + query_results.run() + print_results(query_results) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'corpus', + help='Corpus to search from Shakespeare dataset.') + parser.add_argument( + 'min_word_count', + help='Minimum count of words to query.', + type=int) + + params_type_parser = parser.add_mutually_exclusive_group(required=False) + params_type_parser.add_argument( + '--use-named-params', + dest='use_named_params', + action='store_true') + params_type_parser.add_argument( + '--use-positional-params', + dest='use_named_params', + action='store_false') + parser.set_defaults(use_named_params=False) + args = parser.parse_args() + + if args.use_named_params: + sync_query_named_params(args.corpus, args.min_word_count) + else: + sync_query_positional_params(args.corpus, args.min_word_count) diff --git a/samples/snippets/sync_query_params_test.py b/samples/snippets/sync_query_params_test.py new file mode 100644 index 000000000..270dfc62a --- /dev/null +++ b/samples/snippets/sync_query_params_test.py @@ -0,0 +1,31 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sync_query_params + + +def test_sync_query_named_params(cloud_config, capsys): + sync_query_params.sync_query_named_params( + corpus='romeoandjuliet', + min_word_count=100) + out, _ = capsys.readouterr() + assert 'love' in out + + +def test_sync_query_positional_params(cloud_config, capsys): + sync_query_params.sync_query_positional_params( + corpus='romeoandjuliet', + min_word_count=100) + out, _ = capsys.readouterr() + assert 'love' in out From a0837f979e4caeef94884211770be8241b2d39bd Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 13 Dec 2016 09:54:02 -0800 Subject: [PATCH 026/100] Auto-update dependencies. [(#715)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/715) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 7523b0dac..ba11329ac 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-bigquery==0.22.0 +google-cloud-bigquery==0.22.1 From b1b1f59fe947cff0435555f91b3d4073c1b91f30 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 4 Apr 2017 16:08:30 -0700 Subject: [PATCH 027/100] Remove cloud config fixture [(#887)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/887) * Remove cloud config fixture * Fix client secrets * Fix bigtable instance --- samples/snippets/async_query_test.py | 2 +- samples/snippets/export_data_to_gcs_test.py | 9 +++++---- samples/snippets/load_data_from_gcs_test.py | 8 +++++--- samples/snippets/sync_query_params_test.py | 4 ++-- samples/snippets/sync_query_test.py | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/samples/snippets/async_query_test.py b/samples/snippets/async_query_test.py index 810c538a6..85ce3fce9 100644 --- a/samples/snippets/async_query_test.py +++ b/samples/snippets/async_query_test.py @@ -15,7 +15,7 @@ from async_query import async_query -def test_async_query(cloud_config, capsys): +def test_async_query(capsys): query = ( 'SELECT corpus FROM `publicdata.samples.shakespeare` ' 'GROUP BY corpus;') diff --git a/samples/snippets/export_data_to_gcs_test.py b/samples/snippets/export_data_to_gcs_test.py index acbbe50e5..a41cfd226 100644 --- a/samples/snippets/export_data_to_gcs_test.py +++ b/samples/snippets/export_data_to_gcs_test.py @@ -11,19 +11,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import export_data_to_gcs +import os +import export_data_to_gcs +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] DATASET_ID = 'test_dataset' TABLE_ID = 'test_table' -def test_export_data_to_gcs(cloud_config, capsys): +def test_export_data_to_gcs(capsys): export_data_to_gcs.export_data_to_gcs( DATASET_ID, TABLE_ID, - 'gs://{}/test-export-data-to-gcs.csv'.format( - cloud_config.storage_bucket)) + 'gs://{}/test-export-data-to-gcs.csv'.format(BUCKET)) out, _ = capsys.readouterr() diff --git a/samples/snippets/load_data_from_gcs_test.py b/samples/snippets/load_data_from_gcs_test.py index 2d1c66162..dbd39fc5c 100644 --- a/samples/snippets/load_data_from_gcs_test.py +++ b/samples/snippets/load_data_from_gcs_test.py @@ -11,15 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + import load_data_from_gcs +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] DATASET_ID = 'test_dataset' TABLE_ID = 'test_import_table' -def test_load_table(cloud_config, capsys): - cloud_storage_input_uri = 'gs://{}/data.csv'.format( - cloud_config.storage_bucket) +def test_load_table(capsys): + cloud_storage_input_uri = 'gs://{}/data.csv'.format(BUCKET) load_data_from_gcs.load_data_from_gcs( DATASET_ID, diff --git a/samples/snippets/sync_query_params_test.py b/samples/snippets/sync_query_params_test.py index 270dfc62a..d87fe8231 100644 --- a/samples/snippets/sync_query_params_test.py +++ b/samples/snippets/sync_query_params_test.py @@ -15,7 +15,7 @@ import sync_query_params -def test_sync_query_named_params(cloud_config, capsys): +def test_sync_query_named_params(capsys): sync_query_params.sync_query_named_params( corpus='romeoandjuliet', min_word_count=100) @@ -23,7 +23,7 @@ def test_sync_query_named_params(cloud_config, capsys): assert 'love' in out -def test_sync_query_positional_params(cloud_config, capsys): +def test_sync_query_positional_params(capsys): sync_query_params.sync_query_positional_params( corpus='romeoandjuliet', min_word_count=100) diff --git a/samples/snippets/sync_query_test.py b/samples/snippets/sync_query_test.py index 6f6b4f5fa..26c8973e4 100644 --- a/samples/snippets/sync_query_test.py +++ b/samples/snippets/sync_query_test.py @@ -15,7 +15,7 @@ from sync_query import sync_query -def test_sync_query(cloud_config, capsys): +def test_sync_query(capsys): query = ( 'SELECT corpus FROM `publicdata.samples.shakespeare` ' 'GROUP BY corpus;') From 51fb449484571592d8f3fafac04a50bc9a7f1edd Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 5 Apr 2017 15:21:33 -0700 Subject: [PATCH 028/100] Remove resource [(#890)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/890) * Remove resource fixture * Remove remote resource --- samples/snippets/load_data_from_file_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/samples/snippets/load_data_from_file_test.py b/samples/snippets/load_data_from_file_test.py index 434bbd1d8..960fe62c9 100644 --- a/samples/snippets/load_data_from_file_test.py +++ b/samples/snippets/load_data_from_file_test.py @@ -11,14 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + import load_data_from_file +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') DATASET_ID = 'test_dataset' TABLE_ID = 'test_import_table' -def test_load_table(resource, capsys): - data_path = resource('data.csv') +def test_load_table(capsys): + data_path = os.path.join(RESOURCES, 'data.csv') load_data_from_file.load_data_from_file( DATASET_ID, From f7c3c4cdac9b9a802c2328e86b82d198750c4408 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 19 Dec 2016 12:25:25 -0800 Subject: [PATCH 029/100] BigQuery: named parameter query samples - query using an array value as a parameter. - query using timestamps in named parameters. See: https://cloud.google.com/bigquery/querying-data#using_timestamps_in_parameterized_queries - query using a struct in query parameters. --- samples/snippets/requirements.txt | 1 + samples/snippets/sync_query_params.py | 145 +++++++++++++++++---- samples/snippets/sync_query_params_test.py | 21 ++- 3 files changed, 136 insertions(+), 31 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index ba11329ac..c1f420a87 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1,2 @@ google-cloud-bigquery==0.22.1 +pytz==2016.10 diff --git a/samples/snippets/sync_query_params.py b/samples/snippets/sync_query_params.py index 41ad94052..1d0f3298a 100644 --- a/samples/snippets/sync_query_params.py +++ b/samples/snippets/sync_query_params.py @@ -24,8 +24,10 @@ """ import argparse +import datetime from google.cloud import bigquery +import pytz def print_results(query_results): @@ -46,13 +48,14 @@ def print_results(query_results): def sync_query_positional_params(corpus, min_word_count): client = bigquery.Client() + query = """SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = ? + AND word_count >= ? + ORDER BY word_count DESC; + """ query_results = client.run_sync_query( - """SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = ? - AND word_count >= ? - ORDER BY word_count DESC; - """, + query, query_parameters=( bigquery.ScalarQueryParameter( # Set the name to None to use positional parameters (? symbol @@ -72,13 +75,14 @@ def sync_query_positional_params(corpus, min_word_count): def sync_query_named_params(corpus, min_word_count): client = bigquery.Client() + query = """SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus + AND word_count >= @min_word_count + ORDER BY word_count DESC; + """ query_results = client.run_sync_query( - """SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = @corpus - AND word_count >= @min_word_count - ORDER BY word_count DESC; - """, + query, query_parameters=( bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), bigquery.ScalarQueryParameter( @@ -90,31 +94,118 @@ def sync_query_named_params(corpus, min_word_count): print_results(query_results) +def sync_query_array_params(gender, states): + client = bigquery.Client() + query = """SELECT name, sum(number) as count + FROM `bigquery-public-data.usa_names.usa_1910_2013` + WHERE gender = @gender + AND state IN UNNEST(@states) + GROUP BY name + ORDER BY count DESC + LIMIT 10; + """ + query_results = client.run_sync_query( + query, + query_parameters=( + bigquery.ScalarQueryParameter('gender', 'STRING', gender), + bigquery.ArrayQueryParameter('states', 'STRING', states))) + query_results.use_legacy_sql = False + query_results.run() + print_results(query_results) + + +def sync_query_timestamp_params(year, month, day, hour, minute): + client = bigquery.Client() + query = 'SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);' + query_results = client.run_sync_query( + query, + query_parameters=[ + bigquery.ScalarQueryParameter( + 'ts_value', + 'TIMESTAMP', + datetime.datetime( + year, month, day, hour, minute, tzinfo=pytz.UTC))]) + query_results.use_legacy_sql = False + query_results.run() + print_results(query_results) + + +def sync_query_struct_params(x, y): + client = bigquery.Client() + query = 'SELECT @struct_value AS s;' + query_results = client.run_sync_query( + query, + query_parameters=[ + bigquery.StructQueryParameter( + 'struct_value', + bigquery.ScalarQueryParameter('x', 'INT64', x), + bigquery.ScalarQueryParameter('y', 'STRING', y))]) + query_results.use_legacy_sql = False + query_results.run() + print_results(query_results) + + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( + subparsers = parser.add_subparsers(dest='sample', help='samples') + named_parser = subparsers.add_parser( + 'named', + help='Run a query with named parameters.') + named_parser.add_argument( 'corpus', help='Corpus to search from Shakespeare dataset.') - parser.add_argument( + named_parser.add_argument( 'min_word_count', help='Minimum count of words to query.', type=int) - - params_type_parser = parser.add_mutually_exclusive_group(required=False) - params_type_parser.add_argument( - '--use-named-params', - dest='use_named_params', - action='store_true') - params_type_parser.add_argument( - '--use-positional-params', - dest='use_named_params', - action='store_false') - parser.set_defaults(use_named_params=False) + positional_parser = subparsers.add_parser( + 'positional', + help='Run a query with positional parameters.') + positional_parser.add_argument( + 'corpus', + help='Corpus to search from Shakespeare dataset.') + positional_parser.add_argument( + 'min_word_count', + help='Minimum count of words to query.', + type=int) + array_parser = subparsers.add_parser( + 'array', + help='Run a query with an array parameter.') + array_parser.add_argument( + 'gender', + choices=['F', 'M'], + help='Gender of baby in the Social Security baby names database.') + array_parser.add_argument( + 'states', + help='U.S. States to consider for popular baby names.', + nargs='+') + timestamp_parser = subparsers.add_parser( + 'timestamp', + help='Run a query with a timestamp parameter.') + timestamp_parser.add_argument('year', type=int) + timestamp_parser.add_argument('month', type=int) + timestamp_parser.add_argument('day', type=int) + timestamp_parser.add_argument('hour', type=int) + timestamp_parser.add_argument('minute', type=int) + struct_parser = subparsers.add_parser( + 'struct', + help='Run a query with a struct parameter.') + struct_parser.add_argument('x', help='Integer for x', type=int) + struct_parser.add_argument('y', help='String for y') args = parser.parse_args() - if args.use_named_params: + if args.sample == 'named': sync_query_named_params(args.corpus, args.min_word_count) - else: + elif args.sample == 'positional': sync_query_positional_params(args.corpus, args.min_word_count) + elif args.sample == 'array': + sync_query_array_params(args.gender, args.states) + elif args.sample == 'timestamp': + sync_query_timestamp_params( + args.year, args.month, args.day, args.hour, args.minute) + elif args.sample == 'struct': + sync_query_struct_params(args.x, args.y) + else: + print('Unexpected value for sample') diff --git a/samples/snippets/sync_query_params_test.py b/samples/snippets/sync_query_params_test.py index d87fe8231..c20fe8300 100644 --- a/samples/snippets/sync_query_params_test.py +++ b/samples/snippets/sync_query_params_test.py @@ -16,11 +16,11 @@ def test_sync_query_named_params(capsys): - sync_query_params.sync_query_named_params( - corpus='romeoandjuliet', - min_word_count=100) + sync_query_params.sync_query_array_params( + gender='M', + states=['WA', 'WI', 'WV', 'WY']) out, _ = capsys.readouterr() - assert 'love' in out + assert 'James' in out def test_sync_query_positional_params(capsys): @@ -29,3 +29,16 @@ def test_sync_query_positional_params(capsys): min_word_count=100) out, _ = capsys.readouterr() assert 'love' in out + + +def test_sync_query_struct_params(capsys): + sync_query_params.sync_query_struct_params(765, "hello world") + out, _ = capsys.readouterr() + assert '765' in out + assert 'hello world' in out + + +def test_sync_query_timestamp_params(capsys): + sync_query_params.sync_query_timestamp_params(2016, 12, 7, 8, 0) + out, _ = capsys.readouterr() + assert '2016-12-07 09:00:00' in out From 34fd8e9272922e5918a660fa62dfb65158e990a6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 21 Apr 2017 16:51:55 -0700 Subject: [PATCH 030/100] Rename query file and use async method. Update to latest version of google-cloud-bigquery. --- .../{sync_query_params.py => query_params.py} | 98 ++++++++++++------- ...ry_params_test.py => query_params_test.py} | 28 ++++-- samples/snippets/requirements.txt | 2 +- 3 files changed, 84 insertions(+), 44 deletions(-) rename samples/snippets/{sync_query_params.py => query_params.py} (72%) rename samples/snippets/{sync_query_params_test.py => query_params_test.py} (60%) diff --git a/samples/snippets/sync_query_params.py b/samples/snippets/query_params.py similarity index 72% rename from samples/snippets/sync_query_params.py rename to samples/snippets/query_params.py index 1d0f3298a..891f0ef79 100644 --- a/samples/snippets/sync_query_params.py +++ b/samples/snippets/query_params.py @@ -14,22 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Command-line app to perform synchronous queries with parameters in BigQuery. +"""Command-line app to perform queries with parameters in BigQuery. For more information, see the README.md under /bigquery. Example invocation: - $ python sync_query_params.py --use-named-params 'romeoandjuliet' 100 - $ python sync_query_params.py --use-positional-params 'romeoandjuliet' 100 + $ python query_params.py --use-named-params 'romeoandjuliet' 100 + $ python query_params.py --use-positional-params 'romeoandjuliet' 100 """ import argparse import datetime +import time +import uuid from google.cloud import bigquery import pytz +def wait_for_job(job): + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.errors) + return + time.sleep(1) + + def print_results(query_results): """Print the query results by requesting a page at a time.""" page_token = None @@ -46,7 +58,7 @@ def print_results(query_results): break -def sync_query_positional_params(corpus, min_word_count): +def query_positional_params(corpus, min_word_count): client = bigquery.Client() query = """SELECT word, word_count FROM `bigquery-public-data.samples.shakespeare` @@ -54,7 +66,8 @@ def sync_query_positional_params(corpus, min_word_count): AND word_count >= ? ORDER BY word_count DESC; """ - query_results = client.run_sync_query( + query_job = client.run_async_query( + str(uuid.uuid4()), query, query_parameters=( bigquery.ScalarQueryParameter( @@ -68,12 +81,15 @@ def sync_query_positional_params(corpus, min_word_count): # Only standard SQL syntax supports parameters in queries. # See: https://cloud.google.com/bigquery/sql-reference/ - query_results.use_legacy_sql = False - query_results.run() - print_results(query_results) + query_job.use_legacy_sql = False + + # Start the query and wait for the job to complete. + query_job.begin() + wait_for_job(query_job) + print_results(query_job.results()) -def sync_query_named_params(corpus, min_word_count): +def query_named_params(corpus, min_word_count): client = bigquery.Client() query = """SELECT word, word_count FROM `bigquery-public-data.samples.shakespeare` @@ -81,7 +97,8 @@ def sync_query_named_params(corpus, min_word_count): AND word_count >= @min_word_count ORDER BY word_count DESC; """ - query_results = client.run_sync_query( + query_job = client.run_async_query( + str(uuid.uuid4()), query, query_parameters=( bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), @@ -89,12 +106,15 @@ def sync_query_named_params(corpus, min_word_count): 'min_word_count', 'INT64', min_word_count))) - query_results.use_legacy_sql = False - query_results.run() - print_results(query_results) + query_job.use_legacy_sql = False + # Start the query and wait for the job to complete. + query_job.begin() + wait_for_job(query_job) + print_results(query_job.results()) -def sync_query_array_params(gender, states): + +def query_array_params(gender, states): client = bigquery.Client() query = """SELECT name, sum(number) as count FROM `bigquery-public-data.usa_names.usa_1910_2013` @@ -104,20 +124,25 @@ def sync_query_array_params(gender, states): ORDER BY count DESC LIMIT 10; """ - query_results = client.run_sync_query( + query_job = client.run_async_query( + str(uuid.uuid4()), query, query_parameters=( bigquery.ScalarQueryParameter('gender', 'STRING', gender), bigquery.ArrayQueryParameter('states', 'STRING', states))) - query_results.use_legacy_sql = False - query_results.run() - print_results(query_results) + query_job.use_legacy_sql = False + + # Start the query and wait for the job to complete. + query_job.begin() + wait_for_job(query_job) + print_results(query_job.results()) -def sync_query_timestamp_params(year, month, day, hour, minute): +def query_timestamp_params(year, month, day, hour, minute): client = bigquery.Client() query = 'SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);' - query_results = client.run_sync_query( + query_job = client.run_async_query( + str(uuid.uuid4()), query, query_parameters=[ bigquery.ScalarQueryParameter( @@ -125,24 +150,31 @@ def sync_query_timestamp_params(year, month, day, hour, minute): 'TIMESTAMP', datetime.datetime( year, month, day, hour, minute, tzinfo=pytz.UTC))]) - query_results.use_legacy_sql = False - query_results.run() - print_results(query_results) + query_job.use_legacy_sql = False + # Start the query and wait for the job to complete. + query_job.begin() + wait_for_job(query_job) + print_results(query_job.results()) -def sync_query_struct_params(x, y): + +def query_struct_params(x, y): client = bigquery.Client() query = 'SELECT @struct_value AS s;' - query_results = client.run_sync_query( + query_job = client.run_async_query( + str(uuid.uuid4()), query, query_parameters=[ bigquery.StructQueryParameter( 'struct_value', bigquery.ScalarQueryParameter('x', 'INT64', x), bigquery.ScalarQueryParameter('y', 'STRING', y))]) - query_results.use_legacy_sql = False - query_results.run() - print_results(query_results) + query_job.use_legacy_sql = False + + # Start the query and wait for the job to complete. + query_job.begin() + wait_for_job(query_job) + print_results(query_job.results()) if __name__ == '__main__': @@ -197,15 +229,15 @@ def sync_query_struct_params(x, y): args = parser.parse_args() if args.sample == 'named': - sync_query_named_params(args.corpus, args.min_word_count) + query_named_params(args.corpus, args.min_word_count) elif args.sample == 'positional': - sync_query_positional_params(args.corpus, args.min_word_count) + query_positional_params(args.corpus, args.min_word_count) elif args.sample == 'array': - sync_query_array_params(args.gender, args.states) + query_array_params(args.gender, args.states) elif args.sample == 'timestamp': - sync_query_timestamp_params( + query_timestamp_params( args.year, args.month, args.day, args.hour, args.minute) elif args.sample == 'struct': - sync_query_struct_params(args.x, args.y) + query_struct_params(args.x, args.y) else: print('Unexpected value for sample') diff --git a/samples/snippets/sync_query_params_test.py b/samples/snippets/query_params_test.py similarity index 60% rename from samples/snippets/sync_query_params_test.py rename to samples/snippets/query_params_test.py index c20fe8300..66f4951dd 100644 --- a/samples/snippets/sync_query_params_test.py +++ b/samples/snippets/query_params_test.py @@ -12,33 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sync_query_params +import query_params -def test_sync_query_named_params(capsys): - sync_query_params.sync_query_array_params( +def test_query_array_params(capsys): + query_params.query_array_params( gender='M', states=['WA', 'WI', 'WV', 'WY']) out, _ = capsys.readouterr() assert 'James' in out -def test_sync_query_positional_params(capsys): - sync_query_params.sync_query_positional_params( +def test_query_named_params(capsys): + query_params.query_named_params( corpus='romeoandjuliet', min_word_count=100) out, _ = capsys.readouterr() assert 'love' in out -def test_sync_query_struct_params(capsys): - sync_query_params.sync_query_struct_params(765, "hello world") +def test_query_positional_params(capsys): + query_params.query_positional_params( + corpus='romeoandjuliet', + min_word_count=100) + out, _ = capsys.readouterr() + assert 'love' in out + + +def test_query_struct_params(capsys): + query_params.query_struct_params(765, "hello world") out, _ = capsys.readouterr() assert '765' in out assert 'hello world' in out -def test_sync_query_timestamp_params(capsys): - sync_query_params.sync_query_timestamp_params(2016, 12, 7, 8, 0) +def test_query_timestamp_params(capsys): + query_params.query_timestamp_params(2016, 12, 7, 8, 0) out, _ = capsys.readouterr() - assert '2016-12-07 09:00:00' in out + assert '2016, 12, 7, 9, 0' in out diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index c1f420a87..b969cd8cd 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-bigquery==0.22.1 +google-cloud-bigquery==0.24.0 pytz==2016.10 From 8d11523f2f96abfaa0fe3c90af6c5d9f9b5d1861 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 24 Apr 2017 13:12:09 -0700 Subject: [PATCH 031/100] Auto-update dependencies. [(#914)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/914) * Auto-update dependencies. * xfail the error reporting test * Fix lint --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index ba11329ac..2ca20a5bd 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-bigquery==0.22.1 +google-cloud-bigquery==0.24.0 From 47d3068845a8ff18ace1d759926a4350fb9e1269 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 24 Apr 2017 15:10:27 -0700 Subject: [PATCH 032/100] Indent queries. --- samples/snippets/query_params.py | 51 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/samples/snippets/query_params.py b/samples/snippets/query_params.py index 891f0ef79..1025fd26b 100644 --- a/samples/snippets/query_params.py +++ b/samples/snippets/query_params.py @@ -60,12 +60,13 @@ def print_results(query_results): def query_positional_params(corpus, min_word_count): client = bigquery.Client() - query = """SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = ? - AND word_count >= ? - ORDER BY word_count DESC; - """ + query = """ + SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = ? + AND word_count >= ? + ORDER BY word_count DESC; + """ query_job = client.run_async_query( str(uuid.uuid4()), query, @@ -74,9 +75,7 @@ def query_positional_params(corpus, min_word_count): # Set the name to None to use positional parameters (? symbol # in the query). Note that you cannot mix named and positional # parameters. - None, - 'STRING', - corpus), + None, 'STRING', corpus), bigquery.ScalarQueryParameter(None, 'INT64', min_word_count))) # Only standard SQL syntax supports parameters in queries. @@ -91,21 +90,20 @@ def query_positional_params(corpus, min_word_count): def query_named_params(corpus, min_word_count): client = bigquery.Client() - query = """SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = @corpus - AND word_count >= @min_word_count - ORDER BY word_count DESC; - """ + query = """ + SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus + AND word_count >= @min_word_count + ORDER BY word_count DESC; + """ query_job = client.run_async_query( str(uuid.uuid4()), query, query_parameters=( bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), bigquery.ScalarQueryParameter( - 'min_word_count', - 'INT64', - min_word_count))) + 'min_word_count', 'INT64', min_word_count))) query_job.use_legacy_sql = False # Start the query and wait for the job to complete. @@ -116,14 +114,15 @@ def query_named_params(corpus, min_word_count): def query_array_params(gender, states): client = bigquery.Client() - query = """SELECT name, sum(number) as count - FROM `bigquery-public-data.usa_names.usa_1910_2013` - WHERE gender = @gender - AND state IN UNNEST(@states) - GROUP BY name - ORDER BY count DESC - LIMIT 10; - """ + query = """ + SELECT name, sum(number) as count + FROM `bigquery-public-data.usa_names.usa_1910_2013` + WHERE gender = @gender + AND state IN UNNEST(@states) + GROUP BY name + ORDER BY count DESC + LIMIT 10; + """ query_job = client.run_async_query( str(uuid.uuid4()), query, From 9c3c2d8a0a5501f0bac3ca757b00917f77bf6933 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 25 Apr 2017 09:32:44 -0700 Subject: [PATCH 033/100] Auto-update dependencies. [(#916)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/916) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index b969cd8cd..393886fbe 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ google-cloud-bigquery==0.24.0 -pytz==2016.10 +pytz==2017.2 From f595798210097fb5fc093b6ea2ac165085a39a6f Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Apr 2017 09:54:41 -0700 Subject: [PATCH 034/100] Re-generate all readmes --- samples/snippets/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 1824609dd..4d108b1e5 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -26,7 +26,7 @@ authentication: .. code-block:: bash - gcloud beta auth application-default login + gcloud auth application-default login #. When running on App Engine or Compute Engine, credentials are already From 4d7ce418ca4de6220782110779555ef2fb8cedb6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 3 May 2017 10:52:35 -0700 Subject: [PATCH 035/100] BigQuery: user credentials to run a query. [(#925)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/925) * BigQuery: user credentials to run a query. * BigQuery user creds sample: add tests. Mocks out user credentials using the Application Default Credentials, but uses the same scopes. --- samples/snippets/requirements.txt | 1 + samples/snippets/user_credentials.py | 92 +++++++++++++++++++++++ samples/snippets/user_credentials_test.py | 41 ++++++++++ 3 files changed, 134 insertions(+) create mode 100644 samples/snippets/user_credentials.py create mode 100644 samples/snippets/user_credentials_test.py diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 393886fbe..224e1463a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,3 @@ google-cloud-bigquery==0.24.0 +google-auth-oauthlib==0.0.1 pytz==2017.2 diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py new file mode 100644 index 000000000..a239b741e --- /dev/null +++ b/samples/snippets/user_credentials.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to run a query using user credentials. + +You must supply a client secrets file, which would normally be bundled with +your application. +""" + +import argparse +import time +import uuid + +from google.cloud import bigquery +from google_auth_oauthlib import flow + + +def wait_for_job(job): + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.errors) + return + time.sleep(1) + + +def run_query(credentials, project, query): + client = bigquery.Client(project=project, credentials=credentials) + query_job = client.run_async_query(str(uuid.uuid4()), query) + query_job.use_legacy_sql = False + query_job.begin() + + wait_for_job(query_job) + + # Drain the query results by requesting a page at a time. + query_results = query_job.results() + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break + + +def authenticate_and_query(project, query, launch_browser=True): + appflow = flow.InstalledAppFlow.from_client_secrets_file( + 'client_secrets.json', + scopes=['https://www.googleapis.com/auth/bigquery']) + + if launch_browser: + appflow.run_local_server() + else: + appflow.run_console() + + run_query(appflow.credentials, project, query) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--launch-browser', + help='Use a local server flow to authenticate. ', + action='store_true') + parser.add_argument('project', help='Project to use for BigQuery billing.') + parser.add_argument('query', help='BigQuery SQL Query.') + + args = parser.parse_args() + + authenticate_and_query( + args.project, args.query, launch_browser=args.launch_browser) diff --git a/samples/snippets/user_credentials_test.py b/samples/snippets/user_credentials_test.py new file mode 100644 index 000000000..02acc19c3 --- /dev/null +++ b/samples/snippets/user_credentials_test.py @@ -0,0 +1,41 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.auth +import mock +import pytest + +from user_credentials import authenticate_and_query + + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@pytest.fixture +def mock_flow(): + flow_patch = mock.patch( + 'google_auth_oauthlib.flow.InstalledAppFlow', autospec=True) + + with flow_patch as flow_mock: + flow_mock.from_client_secrets_file.return_value = flow_mock + flow_mock.credentials = google.auth.default()[0] + yield flow_mock + + +def test_auth_query_console(mock_flow, capsys): + authenticate_and_query(PROJECT, 'SELECT 1+1;', launch_browser=False) + out, _ = capsys.readouterr() + assert '2' in out From 5c41eea66933aab91e8c5c372bef4a60155b394f Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 12 May 2017 09:22:05 -0700 Subject: [PATCH 036/100] Auto-update dependencies. [(#942)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/942) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 224e1463a..5874270b3 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-cloud-bigquery==0.24.0 -google-auth-oauthlib==0.0.1 +google-auth-oauthlib==0.1.0 pytz==2017.2 From 1f0ce24a919c25168326359e380e5e601ec5d785 Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 23 May 2017 17:01:25 -0700 Subject: [PATCH 037/100] Fix README rst links [(#962)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/962) * Fix README rst links * Update all READMEs --- samples/snippets/README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 4d108b1e5..1f5b08cab 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -326,11 +326,11 @@ This sample uses the `Google Cloud Client Library for Python`_. You can read the documentation for more details on API usage and use GitHub to `browse the source`_ and `report issues`_. -.. Google Cloud Client Library for Python: +.. _Google Cloud Client Library for Python: https://googlecloudplatform.github.io/google-cloud-python/ -.. browse the source: +.. _browse the source: https://github.com/GoogleCloudPlatform/google-cloud-python -.. report issues: +.. _report issues: https://github.com/GoogleCloudPlatform/google-cloud-python/issues From 21c080203e8f287e5f17613559ba6792202f588a Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 27 Jun 2017 12:41:15 -0700 Subject: [PATCH 038/100] Auto-update dependencies. [(#1004)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1004) * Auto-update dependencies. * Fix natural language samples * Fix pubsub iam samples * Fix language samples * Fix bigquery samples --- samples/snippets/async_query.py | 17 +++-------------- samples/snippets/query_params.py | 17 ++++------------- samples/snippets/query_params_test.py | 4 ++-- samples/snippets/requirements.txt | 2 +- samples/snippets/resources/data.csv | 2 +- samples/snippets/simple_app.py | 15 +++------------ samples/snippets/sync_query.py | 15 +++------------ samples/snippets/user_credentials.py | 15 +++------------ 8 files changed, 20 insertions(+), 67 deletions(-) diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py index f90c8f255..4f7b2330b 100755 --- a/samples/snippets/async_query.py +++ b/samples/snippets/async_query.py @@ -48,20 +48,9 @@ def async_query(query): wait_for_job(query_job) - # Drain the query results by requesting a page at a time. - query_results = query_job.results() - page_token = None - - while True: - rows, total_rows, page_token = query_results.fetch_data( - max_results=10, - page_token=page_token) - - for row in rows: - print(row) - - if not page_token: - break + rows = query_job.results().fetch_data(max_results=10) + for row in rows: + print(row) if __name__ == '__main__': diff --git a/samples/snippets/query_params.py b/samples/snippets/query_params.py index 1025fd26b..192558d91 100644 --- a/samples/snippets/query_params.py +++ b/samples/snippets/query_params.py @@ -43,19 +43,10 @@ def wait_for_job(job): def print_results(query_results): - """Print the query results by requesting a page at a time.""" - page_token = None - - while True: - rows, total_rows, page_token = query_results.fetch_data( - max_results=10, - page_token=page_token) - - for row in rows: - print(row) - - if not page_token: - break + """Print the rows in the query's results.""" + rows = query_results.fetch_data(max_results=10) + for row in rows: + print(row) def query_positional_params(corpus, min_word_count): diff --git a/samples/snippets/query_params_test.py b/samples/snippets/query_params_test.py index 66f4951dd..f4b493137 100644 --- a/samples/snippets/query_params_test.py +++ b/samples/snippets/query_params_test.py @@ -28,7 +28,7 @@ def test_query_named_params(capsys): corpus='romeoandjuliet', min_word_count=100) out, _ = capsys.readouterr() - assert 'love' in out + assert 'the' in out def test_query_positional_params(capsys): @@ -36,7 +36,7 @@ def test_query_positional_params(capsys): corpus='romeoandjuliet', min_word_count=100) out, _ = capsys.readouterr() - assert 'love' in out + assert 'the' in out def test_query_struct_params(capsys): diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 5874270b3..3e0afe76a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.24.0 +google-cloud-bigquery==0.25.0 google-auth-oauthlib==0.1.0 pytz==2017.2 diff --git a/samples/snippets/resources/data.csv b/samples/snippets/resources/data.csv index 230a96b55..affe39ef8 100644 --- a/samples/snippets/resources/data.csv +++ b/samples/snippets/resources/data.csv @@ -1 +1 @@ -Gandalf, 2000, 140.0, 1 +Gandalf,2000,140.0,1 diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 7180c4fda..31059c9f7 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -38,19 +38,10 @@ def query_shakespeare(): # [END run_query] # [START print_results] - # Drain the query results by requesting a page at a time. - page_token = None + rows = query_results.fetch_data(max_results=10) - while True: - rows, total_rows, page_token = query_results.fetch_data( - max_results=10, - page_token=page_token) - - for row in rows: - print(row) - - if not page_token: - break + for row in rows: + print(row) # [END print_results] diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py index 37c8fea8a..1f494f843 100755 --- a/samples/snippets/sync_query.py +++ b/samples/snippets/sync_query.py @@ -39,19 +39,10 @@ def sync_query(query): query_results.run() - # Drain the query results by requesting a page at a time. - page_token = None + rows = query_results.fetch_data(max_results=10) - while True: - rows, total_rows, page_token = query_results.fetch_data( - max_results=10, - page_token=page_token) - - for row in rows: - print(row) - - if not page_token: - break + for row in rows: + print(row) # [END sync_query] diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index a239b741e..017c87ffd 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -46,20 +46,11 @@ def run_query(credentials, project, query): wait_for_job(query_job) - # Drain the query results by requesting a page at a time. query_results = query_job.results() - page_token = None + rows = query_results.fetch_data(max_results=10) - while True: - rows, total_rows, page_token = query_results.fetch_data( - max_results=10, - page_token=page_token) - - for row in rows: - print(row) - - if not page_token: - break + for row in rows: + print(row) def authenticate_and_query(project, query, launch_browser=True): From 484cf15ba0c1573a1e6e9c6a065dfd44a9ce70bc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 1 May 2017 15:44:01 -0700 Subject: [PATCH 039/100] BigQuery: add auth samples for service accounts. --- samples/snippets/.gitignore | 2 + samples/snippets/auth_snippets.py | 62 ++++++++++++++++++++++++++ samples/snippets/auth_snippets_test.py | 33 ++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 samples/snippets/.gitignore create mode 100644 samples/snippets/auth_snippets.py create mode 100644 samples/snippets/auth_snippets_test.py diff --git a/samples/snippets/.gitignore b/samples/snippets/.gitignore new file mode 100644 index 000000000..0dc05ffad --- /dev/null +++ b/samples/snippets/.gitignore @@ -0,0 +1,2 @@ +client_secrets.json +service_account.json diff --git a/samples/snippets/auth_snippets.py b/samples/snippets/auth_snippets.py new file mode 100644 index 000000000..9a0c490d9 --- /dev/null +++ b/samples/snippets/auth_snippets.py @@ -0,0 +1,62 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to authenticate to Google BigQuery using the Google Cloud +Client Libraries.""" + +import argparse + + +def implicit(): + from google.cloud import bigquery + + # If you don't specify credentials when constructing the client, the + # client library will look for credentials in the environment. + bigquery_client = bigquery.Client() + + # Make an authenticated API request + datasets = list(bigquery_client.list_datasets()) + print(datasets) + + +def explicit(): + from google.cloud import bigquery + + # Explicitly use service account credentials by specifying the private key + # file. All clients in google-cloud-python have this helper, see + # https://google-cloud-python.readthedocs.io/en/latest/core/modules.html + # #google.cloud.client.Client.from_service_account_json + bigquery_client = bigquery.Client.from_service_account_json( + 'service_account.json') + + # Make an authenticated API request + buckets = list(bigquery_client.list_datasets()) + print(buckets) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('implicit', help=implicit.__doc__) + subparsers.add_parser('explicit', help=explicit.__doc__) + + args = parser.parse_args() + + if args.command == 'implicit': + implicit() + elif args.command == 'explicit': + explicit() diff --git a/samples/snippets/auth_snippets_test.py b/samples/snippets/auth_snippets_test.py new file mode 100644 index 000000000..5b5f2cac0 --- /dev/null +++ b/samples/snippets/auth_snippets_test.py @@ -0,0 +1,33 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mock + +import auth_snippets + + +def test_implicit(): + auth_snippets.implicit() + + +def test_explicit(): + with open(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) as creds_file: + creds_file_data = creds_file.read() + + open_mock = mock.mock_open(read_data=creds_file_data) + + with mock.patch('io.open', open_mock): + auth_snippets.explicit() From d50ad1dac24ef6b03efcef24f6d920f3a546b746 Mon Sep 17 00:00:00 2001 From: PicardParis Date: Mon, 31 Jul 2017 23:13:35 +0200 Subject: [PATCH 040/100] Fix SQL query to return 10 rows & simplify [(#1041)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1041) - Original query returned a single nested row (instead of 10 rows as apparently expected in section print_results) - Standard SQL specified directly in query - Removed parameter max_results to avoid redundancy w/ query --- samples/snippets/simple_app.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 31059c9f7..6db856839 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -24,21 +24,20 @@ def query_shakespeare(): client = bigquery.Client() # [END create_client] # [START run_query] - query_results = client.run_sync_query(""" - SELECT - APPROX_TOP_COUNT(corpus, 10) as title, - COUNT(*) as unique_words - FROM `publicdata.samples.shakespeare`;""") - - # Use standard SQL syntax for queries. # See: https://cloud.google.com/bigquery/sql-reference/ - query_results.use_legacy_sql = False + query_results = client.run_sync_query(""" + #standardSQL + SELECT corpus AS title, COUNT(*) AS unique_words + FROM `publicdata.samples.shakespeare` + GROUP BY title + ORDER BY unique_words DESC + LIMIT 10""") query_results.run() # [END run_query] # [START print_results] - rows = query_results.fetch_data(max_results=10) + rows = query_results.fetch_data() for row in rows: print(row) From a6e89e18dda9b5b0270c1cf388892c40baaeaece Mon Sep 17 00:00:00 2001 From: PicardParis Date: Tue, 1 Aug 2017 18:04:30 +0200 Subject: [PATCH 041/100] Fix test randomness [(#1043)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1043) --- samples/snippets/async_query_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/snippets/async_query_test.py b/samples/snippets/async_query_test.py index 85ce3fce9..10213e215 100644 --- a/samples/snippets/async_query_test.py +++ b/samples/snippets/async_query_test.py @@ -16,12 +16,13 @@ def test_async_query(capsys): + # Query only outputs the first 10 rows, sort results to avoid randomness query = ( 'SELECT corpus FROM `publicdata.samples.shakespeare` ' - 'GROUP BY corpus;') + 'GROUP BY corpus ORDER BY corpus') async_query(query) out, _ = capsys.readouterr() - assert 'romeoandjuliet' in out + assert 'antonyandcleopatra' in out From 944a6dc73e60ed74787d52ce7d525e78e080f5b4 Mon Sep 17 00:00:00 2001 From: PicardParis Date: Tue, 1 Aug 2017 18:04:41 +0200 Subject: [PATCH 042/100] Fix test randomness [(#1044)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1044) --- samples/snippets/sync_query_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/snippets/sync_query_test.py b/samples/snippets/sync_query_test.py index 26c8973e4..566994477 100644 --- a/samples/snippets/sync_query_test.py +++ b/samples/snippets/sync_query_test.py @@ -16,12 +16,13 @@ def test_sync_query(capsys): + # Query only outputs the first 10 rows, sort results to avoid randomness query = ( 'SELECT corpus FROM `publicdata.samples.shakespeare` ' - 'GROUP BY corpus;') + 'GROUP BY corpus ORDER BY corpus') sync_query(query) out, _ = capsys.readouterr() - assert 'romeoandjuliet' in out + assert 'antonyandcleopatra' in out From ab214bab5bb28d02ea68bc9cd03483a42dab5336 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 7 Aug 2017 10:04:55 -0700 Subject: [PATCH 043/100] Auto-update dependencies. [(#1055)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1055) * Auto-update dependencies. * Explicitly use latest bigtable client Change-Id: Id71e9e768f020730e4ca9514a0d7ebaa794e7d9e * Revert language update for now Change-Id: I8867f154e9a5aae00d0047c9caf880e5e8f50c53 * Remove pdb. smh Change-Id: I5ff905fadc026eebbcd45512d4e76e003e3b2b43 --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 3e0afe76a..fe1ea7ea2 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.25.0 +google-cloud-bigquery==0.26.0 google-auth-oauthlib==0.1.0 pytz==2017.2 From 1326a4d4378fdf0751430dc9b440d6ecf9ee2e82 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 8 Aug 2017 09:40:00 -0700 Subject: [PATCH 044/100] Update readme references. Resolves 1059 Change-Id: I05013e56fae4e801cd6682ec2ec12459baea9dc8 --- samples/snippets/README.rst | 14 +++++++------- samples/snippets/async_query.py | 2 +- samples/snippets/export_data_to_gcs.py | 2 +- samples/snippets/load_data_from_file.py | 2 +- samples/snippets/load_data_from_gcs.py | 2 +- samples/snippets/query_params.py | 2 +- samples/snippets/snippets.py | 2 +- samples/snippets/stream_data.py | 2 +- samples/snippets/sync_query.py | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 1f5b08cab..a68ff1f76 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -97,7 +97,7 @@ To run this sample: Command-line application to perform synchronous queries in BigQuery. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python sync_query.py \ @@ -125,7 +125,7 @@ To run this sample: Command-line application to perform asynchronous queries in BigQuery. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python async_query.py \ @@ -155,7 +155,7 @@ To run this sample: Samples that demonstrate basic operations in the BigQuery API. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python snippets.py list-datasets @@ -206,7 +206,7 @@ To run this sample: Loads data into BigQuery from a local file. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python load_data_from_file.py example_dataset example_table example-data.csv @@ -237,7 +237,7 @@ To run this sample: Loads data into BigQuery from an object in Google Cloud Storage. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python load_data_from_gcs.py example_dataset example_table gs://example-bucket/example-data.csv @@ -269,7 +269,7 @@ To run this sample: Loads a single row of data directly into BigQuery. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python stream_data.py example_dataset example_table '["Gandalf", 2000]' @@ -300,7 +300,7 @@ To run this sample: Exports data from BigQuery to an object in Google Cloud Storage. - For more information, see the README.md under /bigquery. + For more information, see the README.rst. Example invocation: $ python export_data_to_gcs.py example_dataset example_table gs://example-bucket/example-data.csv diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py index 4f7b2330b..895f294bd 100755 --- a/samples/snippets/async_query.py +++ b/samples/snippets/async_query.py @@ -16,7 +16,7 @@ """Command-line application to perform asynchronous queries in BigQuery. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python async_query.py \\ diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py index 3aee442c1..b93ea71ef 100644 --- a/samples/snippets/export_data_to_gcs.py +++ b/samples/snippets/export_data_to_gcs.py @@ -16,7 +16,7 @@ """Exports data from BigQuery to an object in Google Cloud Storage. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python export_data_to_gcs.py example_dataset example_table \ diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index 0bbdd7ba7..5823003a1 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -16,7 +16,7 @@ """Loads data into BigQuery from a local file. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python load_data_from_file.py example_dataset example_table \ diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py index 7c576e20a..21d96169f 100644 --- a/samples/snippets/load_data_from_gcs.py +++ b/samples/snippets/load_data_from_gcs.py @@ -16,7 +16,7 @@ """Loads data into BigQuery from an object in Google Cloud Storage. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python load_data_from_gcs.py example_dataset example_table \ diff --git a/samples/snippets/query_params.py b/samples/snippets/query_params.py index 192558d91..96dbb59b9 100644 --- a/samples/snippets/query_params.py +++ b/samples/snippets/query_params.py @@ -16,7 +16,7 @@ """Command-line app to perform queries with parameters in BigQuery. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python query_params.py --use-named-params 'romeoandjuliet' 100 diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index f4294123d..f62c074a7 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -16,7 +16,7 @@ """Samples that demonstrate basic operations in the BigQuery API. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python snippets.py list-datasets diff --git a/samples/snippets/stream_data.py b/samples/snippets/stream_data.py index a90d432bf..7d9970c3a 100644 --- a/samples/snippets/stream_data.py +++ b/samples/snippets/stream_data.py @@ -16,7 +16,7 @@ """Loads a single row of data directly into BigQuery. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python stream_data.py example_dataset example_table \ diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py index 1f494f843..34d3fbc8a 100755 --- a/samples/snippets/sync_query.py +++ b/samples/snippets/sync_query.py @@ -16,7 +16,7 @@ """Command-line application to perform synchronous queries in BigQuery. -For more information, see the README.md under /bigquery. +For more information, see the README.rst. Example invocation: $ python sync_query.py \\ From 8d19f9052931e336d09a71c669015f2044d44c6e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 8 Aug 2017 09:46:29 -0700 Subject: [PATCH 045/100] Use futures API to wait for jobs to finish. Removes unused sync_query.py sample. --- samples/snippets/async_query.py | 64 --------------- samples/snippets/export_data_to_gcs.py | 14 +--- samples/snippets/load_data_from_file.py | 13 +-- samples/snippets/load_data_from_gcs.py | 14 +--- samples/snippets/query.py | 81 +++++++++++++++++++ samples/snippets/query_params.py | 68 ++++++++-------- .../{async_query_test.py => query_test.py} | 30 +++++-- samples/snippets/snippets.py | 18 +---- samples/snippets/sync_query.py | 57 ------------- samples/snippets/sync_query_test.py | 28 ------- samples/snippets/user_credentials.py | 24 ++---- 11 files changed, 151 insertions(+), 260 deletions(-) delete mode 100755 samples/snippets/async_query.py create mode 100755 samples/snippets/query.py rename samples/snippets/{async_query_test.py => query_test.py} (53%) delete mode 100755 samples/snippets/sync_query.py delete mode 100644 samples/snippets/sync_query_test.py diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py deleted file mode 100755 index 895f294bd..000000000 --- a/samples/snippets/async_query.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to perform asynchronous queries in BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python async_query.py \\ - 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' -""" - -import argparse -import time -import uuid - -from google.cloud import bigquery - - -def wait_for_job(job): - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - -def async_query(query): - client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) - query_job.use_legacy_sql = False - query_job.begin() - - wait_for_job(query_job) - - rows = query_job.results().fetch_data(max_results=10) - for row in rows: - print(row) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('query', help='BigQuery SQL Query.') - - args = parser.parse_args() - - async_query(args.query) diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py index b93ea71ef..41b011ca0 100644 --- a/samples/snippets/export_data_to_gcs.py +++ b/samples/snippets/export_data_to_gcs.py @@ -26,7 +26,6 @@ """ import argparse -import time import uuid from google.cloud import bigquery @@ -42,23 +41,12 @@ def export_data_to_gcs(dataset_name, table_name, destination): job_name, table, destination) job.begin() - - wait_for_job(job) + job.result() # Wait for job to complete print('Exported {}:{} to {}'.format( dataset_name, table_name, destination)) -def wait_for_job(job): - while True: - job.reload() - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index 5823003a1..9e0bf9f4d 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -26,7 +26,6 @@ """ import argparse -import time from google.cloud import bigquery @@ -45,22 +44,12 @@ def load_data_from_file(dataset_name, table_name, source_file_name): job = table.upload_from_file( source_file, source_format='text/csv') - wait_for_job(job) + job.result() # Wait for job to complete print('Loaded {} rows into {}:{}.'.format( job.output_rows, dataset_name, table_name)) -def wait_for_job(job): - while True: - job.reload() - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py index 21d96169f..b0db3a011 100644 --- a/samples/snippets/load_data_from_gcs.py +++ b/samples/snippets/load_data_from_gcs.py @@ -26,7 +26,6 @@ """ import argparse -import time import uuid from google.cloud import bigquery @@ -42,23 +41,12 @@ def load_data_from_gcs(dataset_name, table_name, source): job_name, table, source) job.begin() - - wait_for_job(job) + job.result() # Wait for job to complete print('Loaded {} rows into {}:{}.'.format( job.output_rows, dataset_name, table_name)) -def wait_for_job(job): - while True: - job.reload() - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, diff --git a/samples/snippets/query.py b/samples/snippets/query.py new file mode 100755 index 000000000..f01f912cc --- /dev/null +++ b/samples/snippets/query.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to perform queries in BigQuery. + +For more information, see the README.rst. + +Example invocation: + $ python query.py '#standardSQL + SELECT corpus + FROM `publicdata.samples.shakespeare` + GROUP BY corpus + ORDER BY corpus' +""" + +import argparse +import uuid + +from google.cloud import bigquery + + +def query(query): + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + + query_job.begin() + query_job.result() # Wait for job to complete. + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) + + +def query_standard_sql(query): + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + # Set use_legacy_sql to False to use standard SQL syntax. See: + # https://cloud.google.com/bigquery/docs/reference/standard-sql/enabling-standard-sql + query_job.use_legacy_sql = False + + query_job.begin() + query_job.result() # Wait for job to complete. + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('query', help='BigQuery SQL Query.') + parser.add_argument( + '--use_standard_sql', + action='store_true', + help='Use standard SQL syntax.') + + args = parser.parse_args() + + if args.use_standard_sql: + query_standard_sql(args.query) + else: + query(args.query) diff --git a/samples/snippets/query_params.py b/samples/snippets/query_params.py index 96dbb59b9..435af29d3 100644 --- a/samples/snippets/query_params.py +++ b/samples/snippets/query_params.py @@ -25,30 +25,12 @@ import argparse import datetime -import time import uuid from google.cloud import bigquery import pytz -def wait_for_job(job): - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - -def print_results(query_results): - """Print the rows in the query's results.""" - rows = query_results.fetch_data(max_results=10) - for row in rows: - print(row) - - def query_positional_params(corpus, min_word_count): client = bigquery.Client() query = """ @@ -73,10 +55,14 @@ def query_positional_params(corpus, min_word_count): # See: https://cloud.google.com/bigquery/sql-reference/ query_job.use_legacy_sql = False - # Start the query and wait for the job to complete. query_job.begin() - wait_for_job(query_job) - print_results(query_job.results()) + query_job.result() # Wait for job to complete + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) def query_named_params(corpus, min_word_count): @@ -97,10 +83,14 @@ def query_named_params(corpus, min_word_count): 'min_word_count', 'INT64', min_word_count))) query_job.use_legacy_sql = False - # Start the query and wait for the job to complete. query_job.begin() - wait_for_job(query_job) - print_results(query_job.results()) + query_job.result() # Wait for job to complete + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) def query_array_params(gender, states): @@ -122,10 +112,14 @@ def query_array_params(gender, states): bigquery.ArrayQueryParameter('states', 'STRING', states))) query_job.use_legacy_sql = False - # Start the query and wait for the job to complete. query_job.begin() - wait_for_job(query_job) - print_results(query_job.results()) + query_job.result() # Wait for job to complete + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) def query_timestamp_params(year, month, day, hour, minute): @@ -142,10 +136,14 @@ def query_timestamp_params(year, month, day, hour, minute): year, month, day, hour, minute, tzinfo=pytz.UTC))]) query_job.use_legacy_sql = False - # Start the query and wait for the job to complete. query_job.begin() - wait_for_job(query_job) - print_results(query_job.results()) + query_job.result() # Wait for job to complete + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) def query_struct_params(x, y): @@ -161,10 +159,14 @@ def query_struct_params(x, y): bigquery.ScalarQueryParameter('y', 'STRING', y))]) query_job.use_legacy_sql = False - # Start the query and wait for the job to complete. query_job.begin() - wait_for_job(query_job) - print_results(query_job.results()) + query_job.result() # Wait for job to complete + + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): + print(row) if __name__ == '__main__': diff --git a/samples/snippets/async_query_test.py b/samples/snippets/query_test.py similarity index 53% rename from samples/snippets/async_query_test.py rename to samples/snippets/query_test.py index 10213e215..fa698e146 100644 --- a/samples/snippets/async_query_test.py +++ b/samples/snippets/query_test.py @@ -12,16 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from async_query import async_query +import query -def test_async_query(capsys): +def test_query(capsys): # Query only outputs the first 10 rows, sort results to avoid randomness - query = ( - 'SELECT corpus FROM `publicdata.samples.shakespeare` ' - 'GROUP BY corpus ORDER BY corpus') + query_string = '''#standardSQL + SELECT corpus + FROM `publicdata.samples.shakespeare` + GROUP BY corpus + ORDER BY corpus + LIMIT 10;''' - async_query(query) + query.query(query_string) + + out, _ = capsys.readouterr() + + assert 'antonyandcleopatra' in out + + +def test_query_standard_sql(capsys): + # Query only outputs the first 10 rows, sort results to avoid randomness + query_string = '''SELECT corpus + FROM `publicdata.samples.shakespeare` + GROUP BY corpus + ORDER BY corpus + LIMIT 10;''' + + query.query_standard_sql(query_string) out, _ = capsys.readouterr() diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index f62c074a7..ef6af0abe 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -25,7 +25,6 @@ """ import argparse -import time import uuid from google.cloud import bigquery @@ -165,26 +164,13 @@ def copy_table(dataset_name, table_name, new_table_name, project=None): job.create_disposition = ( google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED) - # Start the job. - job.begin() - - # Wait for the the job to finish. + job.begin() # Start the job. print('Waiting for job to finish...') - wait_for_job(job) + job.result() print('Table {} copied to {}.'.format(table_name, new_table_name)) -def wait_for_job(job): - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - def delete_table(dataset_name, table_name, project=None): """Deletes a table in a given dataset. diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py deleted file mode 100755 index 34d3fbc8a..000000000 --- a/samples/snippets/sync_query.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to perform synchronous queries in BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python sync_query.py \\ - 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' -""" - -import argparse - -# [START sync_query] -from google.cloud import bigquery - - -def sync_query(query): - client = bigquery.Client() - query_results = client.run_sync_query(query) - - # Use standard SQL syntax for queries. - # See: https://cloud.google.com/bigquery/sql-reference/ - query_results.use_legacy_sql = False - - query_results.run() - - rows = query_results.fetch_data(max_results=10) - - for row in rows: - print(row) -# [END sync_query] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('query', help='BigQuery SQL Query.') - - args = parser.parse_args() - - sync_query(args.query) diff --git a/samples/snippets/sync_query_test.py b/samples/snippets/sync_query_test.py deleted file mode 100644 index 566994477..000000000 --- a/samples/snippets/sync_query_test.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sync_query import sync_query - - -def test_sync_query(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query = ( - 'SELECT corpus FROM `publicdata.samples.shakespeare` ' - 'GROUP BY corpus ORDER BY corpus') - - sync_query(query) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index 017c87ffd..ca585c0a1 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -21,35 +21,23 @@ """ import argparse -import time import uuid from google.cloud import bigquery from google_auth_oauthlib import flow -def wait_for_job(job): - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - def run_query(credentials, project, query): client = bigquery.Client(project=project, credentials=credentials) query_job = client.run_async_query(str(uuid.uuid4()), query) - query_job.use_legacy_sql = False - query_job.begin() - wait_for_job(query_job) - - query_results = query_job.results() - rows = query_results.fetch_data(max_results=10) + query_job.begin() + query_job.result() # Wait for the job to complete. - for row in rows: + # Print the results. + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): print(row) From 4bfaa9884cd320547d053583d4be8ba27ab15d55 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 8 Aug 2017 12:11:16 -0700 Subject: [PATCH 046/100] BQ: Use futures API for quickstart. --- samples/snippets/simple_app.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 6db856839..9bca432ef 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -17,6 +17,8 @@ """Simple application that performs a query with BigQuery.""" # [START all] # [START create_client] +import uuid + from google.cloud import bigquery @@ -24,8 +26,7 @@ def query_shakespeare(): client = bigquery.Client() # [END create_client] # [START run_query] - # See: https://cloud.google.com/bigquery/sql-reference/ - query_results = client.run_sync_query(""" + query_job = client.run_async_query(str(uuid.uuid4()), """ #standardSQL SELECT corpus AS title, COUNT(*) AS unique_words FROM `publicdata.samples.shakespeare` @@ -33,13 +34,14 @@ def query_shakespeare(): ORDER BY unique_words DESC LIMIT 10""") - query_results.run() + query_job.begin() + query_job.result() # Wait for job to complete. # [END run_query] # [START print_results] - rows = query_results.fetch_data() - - for row in rows: + destination_table = query_job.destination + destination_table.reload() + for row in destination_table.fetch_data(): print(row) # [END print_results] From b0e9ee1a86528cb2439ae3e4f50e6ac30e4ec2c5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 21 Aug 2017 15:24:48 -0700 Subject: [PATCH 047/100] BigQuery : max_results changes page size not full list size Also, running a query is not "preferred" if you do want all the data. If you do run a query, you end up reading data from the destination table anyway. --- samples/snippets/snippets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index ef6af0abe..bb6b1c01f 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -25,6 +25,7 @@ """ import argparse +import itertools import uuid from google.cloud import bigquery @@ -124,10 +125,9 @@ def list_rows(dataset_name, table_name, project=None): # Reload the table so that the schema is available. table.reload() - # Load at most 25 results. You can change the max_results argument to load - # more rows from BigQuery, but note that this can take some time. It's - # preferred to use a query. - rows = list(table.fetch_data(max_results=25)) + # Load at most 25 results per page. You can change the max_results + # argument to load more rows from BigQuery at a time. + rows = list(itertools.islice(table.fetch_data(max_results=25), 25)) # Use format to create a simple table. format_string = '{!s:<16} ' * len(table.schema) From 74d08713228eb1cb766479986331767ce9bcccc7 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 22 Aug 2017 09:50:12 -0700 Subject: [PATCH 048/100] Remove max_results argument from table.fetch_data() call. The sample uses islice instead, which demonstrates that fetch_data returns an iterable. --- samples/snippets/snippets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index bb6b1c01f..33494cfaa 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -125,9 +125,8 @@ def list_rows(dataset_name, table_name, project=None): # Reload the table so that the schema is available. table.reload() - # Load at most 25 results per page. You can change the max_results - # argument to load more rows from BigQuery at a time. - rows = list(itertools.islice(table.fetch_data(max_results=25), 25)) + # Load at most 25 results. + rows = list(itertools.islice(table.fetch_data(), 25)) # Use format to create a simple table. format_string = '{!s:<16} ' * len(table.schema) From d8be82a99280e03c93decc94fc5d098cd89e8ddc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 22 Aug 2017 10:08:26 -0700 Subject: [PATCH 049/100] Remove unnecessary list() call --- samples/snippets/snippets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index 33494cfaa..401d18bbd 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -126,7 +126,7 @@ def list_rows(dataset_name, table_name, project=None): table.reload() # Load at most 25 results. - rows = list(itertools.islice(table.fetch_data(), 25)) + rows = itertools.islice(table.fetch_data(), 25) # Use format to create a simple table. format_string = '{!s:<16} ' * len(table.schema) From 8ca914ab141c878bd09f1d96a1b15b3d0fa777d1 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 29 Aug 2017 16:53:02 -0700 Subject: [PATCH 050/100] Auto-update dependencies. [(#1093)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1093) * Auto-update dependencies. * Fix storage notification poll sample Change-Id: I6afbc79d15e050531555e4c8e51066996717a0f3 * Fix spanner samples Change-Id: I40069222c60d57e8f3d3878167591af9130895cb * Drop coverage because it's not useful Change-Id: Iae399a7083d7866c3c7b9162d0de244fbff8b522 * Try again to fix flaky logging test Change-Id: I6225c074701970c17c426677ef1935bb6d7e36b4 --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index fe1ea7ea2..e72d04ef4 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.26.0 +google-cloud-bigquery==0.27.0 google-auth-oauthlib==0.1.0 pytz==2017.2 From e65d62f7b575a0365633da73bb5d92f0715f7596 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 19 Sep 2017 09:33:49 -0700 Subject: [PATCH 051/100] BigQuery : add sample for writing query results to a destination table. [(#1101)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1101) See: https://cloud.google.com/bigquery/docs/writing-results --- samples/snippets/query.py | 33 +++++++++++++++++++++++++++++++++ samples/snippets/query_test.py | 20 ++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/samples/snippets/query.py b/samples/snippets/query.py index f01f912cc..93b13b848 100755 --- a/samples/snippets/query.py +++ b/samples/snippets/query.py @@ -63,6 +63,30 @@ def query_standard_sql(query): print(row) +def query_destination_table(query, dest_dataset_id, dest_table_id): + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + + # Allow for query results larger than the maximum response size. + query_job.allow_large_results = True + + # When large results are allowed, a destination table must be set. + dest_dataset = client.dataset(dest_dataset_id) + dest_table = dest_dataset.table(dest_table_id) + query_job.destination = dest_table + + # Allow the results table to be overwritten. + query_job.write_disposition = 'WRITE_TRUNCATE' + + query_job.begin() + query_job.result() # Wait for job to complete. + + # Verify that the results were written to the destination table. + dest_table.reload() # Get the table metadata, such as the schema. + for row in dest_table.fetch_data(): + print(row) + + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, @@ -72,10 +96,19 @@ def query_standard_sql(query): '--use_standard_sql', action='store_true', help='Use standard SQL syntax.') + parser.add_argument( + '--destination_table', + type=str, + help=( + 'Destination table to use for results. ' + 'Example: my_dataset.my_table')) args = parser.parse_args() if args.use_standard_sql: query_standard_sql(args.query) + elif args.destination_table: + dataset, table = args.destination_table.split('.') + query_destination_table(args.query, dataset, table) else: query(args.query) diff --git a/samples/snippets/query_test.py b/samples/snippets/query_test.py index fa698e146..9d6c912b4 100644 --- a/samples/snippets/query_test.py +++ b/samples/snippets/query_test.py @@ -15,6 +15,10 @@ import query +DATASET_ID = 'test_dataset' +TABLE_ID = 'test_destination_table' + + def test_query(capsys): # Query only outputs the first 10 rows, sort results to avoid randomness query_string = '''#standardSQL @@ -44,3 +48,19 @@ def test_query_standard_sql(capsys): out, _ = capsys.readouterr() assert 'antonyandcleopatra' in out + + +def test_query_destination_table(capsys): + # Query only outputs the first 10 rows, sort results to avoid randomness + query_string = '''#standardSQL + SELECT corpus + FROM `publicdata.samples.shakespeare` + GROUP BY corpus + ORDER BY corpus + LIMIT 10;''' + + query.query_destination_table(query_string, DATASET_ID, TABLE_ID) + + out, _ = capsys.readouterr() + + assert 'antonyandcleopatra' in out From 9f92234d755a6ef50e2dcf12ce0f0fce11817476 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 20 Sep 2017 09:09:43 -0700 Subject: [PATCH 052/100] Auto-update dependencies. [(#1128)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1128) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index e72d04ef4..e90122cb6 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-cloud-bigquery==0.27.0 -google-auth-oauthlib==0.1.0 +google-auth-oauthlib==0.1.1 pytz==2017.2 From aecfab400218268cde2de0db00311abf3a5893d3 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 31 Oct 2017 09:58:29 -0700 Subject: [PATCH 053/100] BigQuery: Updates samples for BigQuery Beta 2 (do not merge until release) [(#1178)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1178) * updates client.extract_table() sample * updates client.load_table_from_file() sample * updates client.load_table_from_uri() sample * updates parameterized query samples * updates query samples * updates dataset quickstart * fixes typo in sample * updates simple app sample * updates snippets * updates stream data sample * updates user credentials sample * removes job_id generation * Update BQ lib to 0.28.0. Use max_results in list_rows. * updates public dataset queries to use 'bigquery-public-data' instead of outdated 'publicdata' * updates readme and help sections --- samples/snippets/README.rst | 111 +++++++++++------- samples/snippets/README.rst.in | 10 +- samples/snippets/export_data_to_gcs.py | 28 ++--- samples/snippets/load_data_from_file.py | 29 +++-- samples/snippets/load_data_from_gcs.py | 26 ++--- samples/snippets/query.py | 47 +++----- samples/snippets/query_params.py | 146 +++++++++++------------- samples/snippets/query_test.py | 6 +- samples/snippets/quickstart.py | 11 +- samples/snippets/quickstart_test.py | 19 ++- samples/snippets/requirements.txt | 2 +- samples/snippets/simple_app.py | 15 +-- samples/snippets/snippets.py | 129 ++++++++++----------- samples/snippets/snippets_test.py | 72 +++++++----- samples/snippets/stream_data.py | 24 ++-- samples/snippets/user_credentials.py | 10 +- 16 files changed, 347 insertions(+), 338 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index a68ff1f76..1a3b889a3 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -70,6 +70,18 @@ Install Dependencies Samples ------------------------------------------------------------------------------- +Simple Application ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python simple_app.py + + Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -82,7 +94,7 @@ To run this sample: $ python quickstart.py -Sync query +Query +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -91,26 +103,35 @@ To run this sample: .. code-block:: bash - $ python sync_query.py + $ python query.py - usage: sync_query.py [-h] query + usage: query.py [-h] [--use_standard_sql] + [--destination_table DESTINATION_TABLE] + query - Command-line application to perform synchronous queries in BigQuery. + Command-line application to perform queries in BigQuery. For more information, see the README.rst. Example invocation: - $ python sync_query.py \ - 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' + $ python query.py '#standardSQL + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus + ORDER BY corpus' positional arguments: - query BigQuery SQL Query. + query BigQuery SQL Query. optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit + --use_standard_sql Use standard SQL syntax. + --destination_table DESTINATION_TABLE + Destination table to use for results. Example: + my_dataset.my_table -Async query +Parameterized Query +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -119,23 +140,29 @@ To run this sample: .. code-block:: bash - $ python async_query.py + $ python query_params.py - usage: async_query.py [-h] query + usage: query_params.py [-h] {named,positional,array,timestamp,struct} ... - Command-line application to perform asynchronous queries in BigQuery. + Command-line app to perform queries with parameters in BigQuery. For more information, see the README.rst. Example invocation: - $ python async_query.py \ - 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' + $ python query_params.py named 'romeoandjuliet' 100 + $ python query_params.py positional 'romeoandjuliet' 100 positional arguments: - query BigQuery SQL Query. + {named,positional,array,timestamp,struct} + samples + named Run a query with named parameters. + positional Run a query with positional parameters. + array Run a query with an array parameter. + timestamp Run a query with a timestamp parameter. + struct Run a query with a struct parameter. optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit Snippets @@ -202,20 +229,21 @@ To run this sample: $ python load_data_from_file.py - usage: load_data_from_file.py [-h] dataset_name table_name source_file_name + usage: load_data_from_file.py [-h] dataset_id table_id source_file_name Loads data into BigQuery from a local file. For more information, see the README.rst. Example invocation: - $ python load_data_from_file.py example_dataset example_table example-data.csv + $ python load_data_from_file.py example_dataset example_table \ + example-data.csv The dataset and table should already exist. positional arguments: - dataset_name - table_name + dataset_id + table_id source_file_name Path to a .csv file to upload. optional arguments: @@ -233,25 +261,26 @@ To run this sample: $ python load_data_from_gcs.py - usage: load_data_from_gcs.py [-h] dataset_name table_name source + usage: load_data_from_gcs.py [-h] dataset_id table_id source Loads data into BigQuery from an object in Google Cloud Storage. For more information, see the README.rst. Example invocation: - $ python load_data_from_gcs.py example_dataset example_table gs://example-bucket/example-data.csv + $ python load_data_from_gcs.py example_dataset example_table \ + gs://example-bucket/example-data.csv The dataset and table should already exist. positional arguments: - dataset_name - table_name - source The Google Cloud Storage object to load. Must be in the format - gs://bucket_name/object_name + dataset_id + table_id + source The Google Cloud Storage object to load. Must be in the format + gs://bucket_name/object_name optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit Load streaming data @@ -265,24 +294,25 @@ To run this sample: $ python stream_data.py - usage: stream_data.py [-h] dataset_name table_name json_data + usage: stream_data.py [-h] dataset_id table_id json_data Loads a single row of data directly into BigQuery. For more information, see the README.rst. Example invocation: - $ python stream_data.py example_dataset example_table '["Gandalf", 2000]' + $ python stream_data.py example_dataset example_table \ + '["Gandalf", 2000]' The dataset and table should already exist. positional arguments: - dataset_name - table_name - json_data The row to load into BigQuery as an array in JSON format. + dataset_id + table_id + json_data The row to load into BigQuery as an array in JSON format. optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit Export data to Cloud Storage @@ -296,25 +326,26 @@ To run this sample: $ python export_data_to_gcs.py - usage: export_data_to_gcs.py [-h] dataset_name table_name destination + usage: export_data_to_gcs.py [-h] dataset_id table_id destination Exports data from BigQuery to an object in Google Cloud Storage. For more information, see the README.rst. Example invocation: - $ python export_data_to_gcs.py example_dataset example_table gs://example-bucket/example-data.csv + $ python export_data_to_gcs.py example_dataset example_table \ + gs://example-bucket/example-data.csv The dataset and table should already exist. positional arguments: - dataset_name - table_name - destination The desintation Google Cloud Storage object.Must be in the - format gs://bucket_name/object_name + dataset_id + table_id + destination The destination Google Cloud Storage object. Must be in the + format gs://bucket_name/object_name optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in index 49143f062..61c66ab43 100644 --- a/samples/snippets/README.rst.in +++ b/samples/snippets/README.rst.in @@ -16,13 +16,15 @@ setup: - install_deps samples: +- name: Simple Application + file: simple_app.py - name: Quickstart file: quickstart.py -- name: Sync query - file: sync_query.py +- name: Query + file: query.py show_help: true -- name: Async query - file: async_query.py +- name: Parameterized Query + file: query_params.py show_help: true - name: Snippets file: snippets.py diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py index 41b011ca0..5993ef0f6 100644 --- a/samples/snippets/export_data_to_gcs.py +++ b/samples/snippets/export_data_to_gcs.py @@ -19,47 +19,43 @@ For more information, see the README.rst. Example invocation: - $ python export_data_to_gcs.py example_dataset example_table \ + $ python export_data_to_gcs.py example_dataset example_table \\ gs://example-bucket/example-data.csv The dataset and table should already exist. """ import argparse -import uuid from google.cloud import bigquery -def export_data_to_gcs(dataset_name, table_name, destination): +def export_data_to_gcs(dataset_id, table_id, destination): bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) - job_name = str(uuid.uuid4()) + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) - job = bigquery_client.extract_table_to_storage( - job_name, table, destination) + job = bigquery_client.extract_table(table_ref, destination) - job.begin() - job.result() # Wait for job to complete + job.result() # Waits for job to complete print('Exported {}:{} to {}'.format( - dataset_name, table_name, destination)) + dataset_id, table_id, destination)) if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_name') - parser.add_argument('table_name') + parser.add_argument('dataset_id') + parser.add_argument('table_id') parser.add_argument( - 'destination', help='The desintation Google Cloud Storage object.' + 'destination', help='The destination Google Cloud Storage object. ' 'Must be in the format gs://bucket_name/object_name') args = parser.parse_args() export_data_to_gcs( - args.dataset_name, - args.table_name, + args.dataset_id, + args.table_id, args.destination) diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py index 9e0bf9f4d..e311daa1e 100644 --- a/samples/snippets/load_data_from_file.py +++ b/samples/snippets/load_data_from_file.py @@ -19,7 +19,7 @@ For more information, see the README.rst. Example invocation: - $ python load_data_from_file.py example_dataset example_table \ + $ python load_data_from_file.py example_dataset example_table \\ example-data.csv The dataset and table should already exist. @@ -30,38 +30,37 @@ from google.cloud import bigquery -def load_data_from_file(dataset_name, table_name, source_file_name): +def load_data_from_file(dataset_id, table_id, source_file_name): bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) - - # Reload the table to get the schema. - table.reload() + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) with open(source_file_name, 'rb') as source_file: # This example uses CSV, but you can use other formats. # See https://cloud.google.com/bigquery/loading-data - job = table.upload_from_file( - source_file, source_format='text/csv') + job_config = bigquery.LoadJobConfig() + job_config.source_format = 'text/csv' + job = bigquery_client.load_table_from_file( + source_file, table_ref, job_config=job_config) - job.result() # Wait for job to complete + job.result() # Waits for job to complete print('Loaded {} rows into {}:{}.'.format( - job.output_rows, dataset_name, table_name)) + job.output_rows, dataset_id, table_id)) if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_name') - parser.add_argument('table_name') + parser.add_argument('dataset_id') + parser.add_argument('table_id') parser.add_argument( 'source_file_name', help='Path to a .csv file to upload.') args = parser.parse_args() load_data_from_file( - args.dataset_name, - args.table_name, + args.dataset_id, + args.table_id, args.source_file_name) diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py index b0db3a011..285e6d1b2 100644 --- a/samples/snippets/load_data_from_gcs.py +++ b/samples/snippets/load_data_from_gcs.py @@ -19,40 +19,36 @@ For more information, see the README.rst. Example invocation: - $ python load_data_from_gcs.py example_dataset example_table \ + $ python load_data_from_gcs.py example_dataset example_table \\ gs://example-bucket/example-data.csv The dataset and table should already exist. """ import argparse -import uuid from google.cloud import bigquery -def load_data_from_gcs(dataset_name, table_name, source): +def load_data_from_gcs(dataset_id, table_id, source): bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) - job_name = str(uuid.uuid4()) + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) - job = bigquery_client.load_table_from_storage( - job_name, table, source) + job = bigquery_client.load_table_from_uri(source, table_ref) - job.begin() - job.result() # Wait for job to complete + job.result() # Waits for job to complete print('Loaded {} rows into {}:{}.'.format( - job.output_rows, dataset_name, table_name)) + job.output_rows, dataset_id, table_id)) if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_name') - parser.add_argument('table_name') + parser.add_argument('dataset_id') + parser.add_argument('table_id') parser.add_argument( 'source', help='The Google Cloud Storage object to load. Must be in ' 'the format gs://bucket_name/object_name') @@ -60,6 +56,6 @@ def load_data_from_gcs(dataset_name, table_name, source): args = parser.parse_args() load_data_from_gcs( - args.dataset_name, - args.table_name, + args.dataset_id, + args.table_id, args.source) diff --git a/samples/snippets/query.py b/samples/snippets/query.py index 93b13b848..19605bbad 100755 --- a/samples/snippets/query.py +++ b/samples/snippets/query.py @@ -21,69 +21,58 @@ Example invocation: $ python query.py '#standardSQL SELECT corpus - FROM `publicdata.samples.shakespeare` + FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus ORDER BY corpus' """ import argparse -import uuid from google.cloud import bigquery def query(query): client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) - - query_job.begin() - query_job.result() # Wait for job to complete. + query_job = client.query(query) # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + for row in query_job.result(): # Waits for job to complete. print(row) def query_standard_sql(query): client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) - # Set use_legacy_sql to False to use standard SQL syntax. See: - # https://cloud.google.com/bigquery/docs/reference/standard-sql/enabling-standard-sql - query_job.use_legacy_sql = False + job_config = bigquery.QueryJobConfig() - query_job.begin() - query_job.result() # Wait for job to complete. + # Set use_legacy_sql to False to use standard SQL syntax. + # Note that queries are treated as standard SQL by default. + job_config.use_legacy_sql = False + query_job = client.query(query, job_config=job_config) # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + for row in query_job.result(): # Waits for job to complete. print(row) def query_destination_table(query, dest_dataset_id, dest_table_id): client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) + job_config = bigquery.QueryJobConfig() # Allow for query results larger than the maximum response size. - query_job.allow_large_results = True + job_config.allow_large_results = True # When large results are allowed, a destination table must be set. - dest_dataset = client.dataset(dest_dataset_id) - dest_table = dest_dataset.table(dest_table_id) - query_job.destination = dest_table + dest_dataset_ref = client.dataset(dest_dataset_id) + dest_table_ref = dest_dataset_ref.table(dest_table_id) + job_config.destination = dest_table_ref # Allow the results table to be overwritten. - query_job.write_disposition = 'WRITE_TRUNCATE' + job_config.write_disposition = 'WRITE_TRUNCATE' - query_job.begin() - query_job.result() # Wait for job to complete. + query_job = client.query(query, job_config=job_config) - # Verify that the results were written to the destination table. - dest_table.reload() # Get the table metadata, such as the schema. - for row in dest_table.fetch_data(): + # Print the results. + for row in query_job.result(): # Waits for job to complete. print(row) diff --git a/samples/snippets/query_params.py b/samples/snippets/query_params.py index 435af29d3..594f51c75 100644 --- a/samples/snippets/query_params.py +++ b/samples/snippets/query_params.py @@ -19,13 +19,12 @@ For more information, see the README.rst. Example invocation: - $ python query_params.py --use-named-params 'romeoandjuliet' 100 - $ python query_params.py --use-positional-params 'romeoandjuliet' 100 + $ python query_params.py named 'romeoandjuliet' 100 + $ python query_params.py positional 'romeoandjuliet' 100 """ import argparse import datetime -import uuid from google.cloud import bigquery import pytz @@ -40,28 +39,23 @@ def query_positional_params(corpus, min_word_count): AND word_count >= ? ORDER BY word_count DESC; """ - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=( - bigquery.ScalarQueryParameter( - # Set the name to None to use positional parameters (? symbol - # in the query). Note that you cannot mix named and positional - # parameters. - None, 'STRING', corpus), - bigquery.ScalarQueryParameter(None, 'INT64', min_word_count))) - - # Only standard SQL syntax supports parameters in queries. - # See: https://cloud.google.com/bigquery/sql-reference/ - query_job.use_legacy_sql = False - - query_job.begin() + # Set the name to None to use positional parameters (? symbol in the + # query). Note that you cannot mix named and positional parameters. + # See: https://cloud.google.com/bigquery/docs/parameterized-queries/ + query_params = [ + bigquery.ScalarQueryParameter(None, 'STRING', corpus), + bigquery.ScalarQueryParameter(None, 'INT64', min_word_count) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query(query, job_config=job_config) + query_job.result() # Wait for job to complete # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + destination_table_ref = query_job.destination + table = client.get_table(destination_table_ref) + for row in client.list_rows(table): print(row) @@ -74,22 +68,21 @@ def query_named_params(corpus, min_word_count): AND word_count >= @min_word_count ORDER BY word_count DESC; """ - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=( - bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), - bigquery.ScalarQueryParameter( - 'min_word_count', 'INT64', min_word_count))) - query_job.use_legacy_sql = False - - query_job.begin() + query_params = [ + bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), + bigquery.ScalarQueryParameter( + 'min_word_count', 'INT64', min_word_count) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query(query, job_config=job_config) + query_job.result() # Wait for job to complete # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + destination_table_ref = query_job.destination + table = client.get_table(destination_table_ref) + for row in client.list_rows(table): print(row) @@ -104,68 +97,65 @@ def query_array_params(gender, states): ORDER BY count DESC LIMIT 10; """ - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=( - bigquery.ScalarQueryParameter('gender', 'STRING', gender), - bigquery.ArrayQueryParameter('states', 'STRING', states))) - query_job.use_legacy_sql = False - - query_job.begin() + query_params = [ + bigquery.ScalarQueryParameter('gender', 'STRING', gender), + bigquery.ArrayQueryParameter('states', 'STRING', states) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query(query, job_config=job_config) + query_job.result() # Wait for job to complete # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + destination_table_ref = query_job.destination + table = client.get_table(destination_table_ref) + for row in client.list_rows(table): print(row) def query_timestamp_params(year, month, day, hour, minute): client = bigquery.Client() query = 'SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);' - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=[ - bigquery.ScalarQueryParameter( - 'ts_value', - 'TIMESTAMP', - datetime.datetime( - year, month, day, hour, minute, tzinfo=pytz.UTC))]) - query_job.use_legacy_sql = False - - query_job.begin() - query_job.result() # Wait for job to complete + query_params = [ + bigquery.ScalarQueryParameter( + 'ts_value', + 'TIMESTAMP', + datetime.datetime(year, month, day, hour, minute, tzinfo=pytz.UTC)) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query(query, job_config=job_config) + + query_job.result() # Waits for job to complete # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + destination_table_ref = query_job.destination + table = client.get_table(destination_table_ref) + for row in client.list_rows(table): print(row) def query_struct_params(x, y): client = bigquery.Client() query = 'SELECT @struct_value AS s;' - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=[ - bigquery.StructQueryParameter( - 'struct_value', - bigquery.ScalarQueryParameter('x', 'INT64', x), - bigquery.ScalarQueryParameter('y', 'STRING', y))]) - query_job.use_legacy_sql = False - - query_job.begin() - query_job.result() # Wait for job to complete + query_params = [ + bigquery.StructQueryParameter( + 'struct_value', + bigquery.ScalarQueryParameter('x', 'INT64', x), + bigquery.ScalarQueryParameter('y', 'STRING', y) + ) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query(query, job_config=job_config) + + query_job.result() # Waits for job to complete # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + destination_table_ref = query_job.destination + table = client.get_table(destination_table_ref) + for row in client.list_rows(table): print(row) diff --git a/samples/snippets/query_test.py b/samples/snippets/query_test.py index 9d6c912b4..3d456cb59 100644 --- a/samples/snippets/query_test.py +++ b/samples/snippets/query_test.py @@ -23,7 +23,7 @@ def test_query(capsys): # Query only outputs the first 10 rows, sort results to avoid randomness query_string = '''#standardSQL SELECT corpus - FROM `publicdata.samples.shakespeare` + FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus ORDER BY corpus LIMIT 10;''' @@ -38,7 +38,7 @@ def test_query(capsys): def test_query_standard_sql(capsys): # Query only outputs the first 10 rows, sort results to avoid randomness query_string = '''SELECT corpus - FROM `publicdata.samples.shakespeare` + FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus ORDER BY corpus LIMIT 10;''' @@ -54,7 +54,7 @@ def test_query_destination_table(capsys): # Query only outputs the first 10 rows, sort results to avoid randomness query_string = '''#standardSQL SELECT corpus - FROM `publicdata.samples.shakespeare` + FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus ORDER BY corpus LIMIT 10;''' diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 2c9923f6e..10ae58e84 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -24,15 +24,16 @@ def run_quickstart(): bigquery_client = bigquery.Client() # The name for the new dataset - dataset_name = 'my_new_dataset' + dataset_id = 'my_new_dataset' - # Prepares the new dataset - dataset = bigquery_client.dataset(dataset_name) + # Prepares a reference to the new dataset + dataset_ref = bigquery_client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) # Creates the new dataset - dataset.create() + dataset = bigquery_client.create_dataset(dataset) - print('Dataset {} created.'.format(dataset.name)) + print('Dataset {} created.'.format(dataset.dataset_id)) # [END bigquery_quickstart] diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index f5842960c..02931086a 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -13,6 +13,7 @@ # limitations under the License. from google.cloud import bigquery +from google.cloud.exceptions import NotFound import pytest import quickstart @@ -28,15 +29,23 @@ def temporary_dataset(): """Fixture that ensures the test dataset does not exist before or after a test.""" bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(DATASET_ID) + dataset_ref = bigquery_client.dataset(DATASET_ID) - if dataset.exists(): - dataset.delete() + if dataset_exists(dataset_ref, bigquery_client): + bigquery_client.delete_dataset(dataset_ref) yield - if dataset.exists(): - dataset.delete() + if dataset_exists(dataset_ref, bigquery_client): + bigquery_client.delete_dataset(dataset_ref) + + +def dataset_exists(dataset, client): + try: + client.get_dataset(dataset) + return True + except NotFound: + return False def test_quickstart(capsys, temporary_dataset): diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index e90122cb6..0401c4221 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.27.0 +google-cloud-bigquery==0.28.0 google-auth-oauthlib==0.1.1 pytz==2017.2 diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 9bca432ef..5d0d04e66 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -17,8 +17,6 @@ """Simple application that performs a query with BigQuery.""" # [START all] # [START create_client] -import uuid - from google.cloud import bigquery @@ -26,23 +24,20 @@ def query_shakespeare(): client = bigquery.Client() # [END create_client] # [START run_query] - query_job = client.run_async_query(str(uuid.uuid4()), """ + query_job = client.query(""" #standardSQL SELECT corpus AS title, COUNT(*) AS unique_words - FROM `publicdata.samples.shakespeare` + FROM `bigquery-public-data.samples.shakespeare` GROUP BY title ORDER BY unique_words DESC LIMIT 10""") - query_job.begin() - query_job.result() # Wait for job to complete. + results = query_job.result() # Waits for job to complete. # [END run_query] # [START print_results] - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): - print(row) + for row in results: + print("{}: {}".format(row.title, row.unique_words)) # [END print_results] diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py index 401d18bbd..ee75f7fc2 100644 --- a/samples/snippets/snippets.py +++ b/samples/snippets/snippets.py @@ -25,11 +25,8 @@ """ import argparse -import itertools -import uuid from google.cloud import bigquery -import google.cloud.bigquery.job def list_projects(): @@ -47,52 +44,45 @@ def list_datasets(project=None): bigquery_client = bigquery.Client(project=project) for dataset in bigquery_client.list_datasets(): - print(dataset.name) + print(dataset.dataset_id) -def create_dataset(dataset_name, project=None): +def create_dataset(dataset_id, project=None): """Craetes a dataset in a given project. If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) - dataset = bigquery_client.dataset(dataset_name) + dataset_ref = bigquery_client.dataset(dataset_id) - dataset.create() + dataset = bigquery_client.create_dataset(bigquery.Dataset(dataset_ref)) - print('Created dataset {}.'.format(dataset_name)) + print('Created dataset {}.'.format(dataset.dataset_id)) -def list_tables(dataset_name, project=None): +def list_tables(dataset_id, project=None): """Lists all of the tables in a given dataset. If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) - dataset = bigquery_client.dataset(dataset_name) + dataset_ref = bigquery_client.dataset(dataset_id) - if not dataset.exists(): - print('Dataset {} does not exist.'.format(dataset_name)) - return + for table in bigquery_client.list_dataset_tables(dataset_ref): + print(table.table_id) - for table in dataset.list_tables(): - print(table.name) - -def create_table(dataset_name, table_name, project=None): +def create_table(dataset_id, table_id, project=None): """Creates a simple table in the given dataset. If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) - dataset = bigquery_client.dataset(dataset_name) - - if not dataset.exists(): - print('Dataset {} does not exist.'.format(dataset_name)) - return + dataset_ref = bigquery_client.dataset(dataset_id) - table = dataset.table(table_name) + table_ref = dataset_ref.table(table_id) + table = bigquery.Table(table_ref) # Set the table schema table.schema = ( @@ -101,12 +91,12 @@ def create_table(dataset_name, table_name, project=None): bigquery.SchemaField('Weight', 'FLOAT'), ) - table.create() + table = bigquery_client.create_table(table) - print('Created table {} in dataset {}.'.format(table_name, dataset_name)) + print('Created table {} in dataset {}.'.format(table_id, dataset_id)) -def list_rows(dataset_name, table_name, project=None): +def list_rows(dataset_id, table_id, project=None): """Prints rows in the given table. Will print 25 rows at most for brevity as tables can contain large amounts @@ -115,18 +105,14 @@ def list_rows(dataset_name, table_name, project=None): If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) - if not table.exists(): - print('Table {}:{} does not exist.'.format(dataset_name, table_name)) - return - - # Reload the table so that the schema is available. - table.reload() + # Get the table from the API so that the schema is available. + table = bigquery_client.get_table(table_ref) # Load at most 25 results. - rows = itertools.islice(table.fetch_data(), 25) + rows = bigquery_client.list_rows(table, max_results=25) # Use format to create a simple table. format_string = '{!s:<16} ' * len(table.schema) @@ -139,49 +125,50 @@ def list_rows(dataset_name, table_name, project=None): print(format_string.format(*row)) -def copy_table(dataset_name, table_name, new_table_name, project=None): +def copy_table(dataset_id, table_id, new_table_id, project=None): """Copies a table. If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) # This sample shows the destination table in the same dataset and project, # however, it's possible to copy across datasets and projects. You can - # also copy muliple source tables into a single destination table by + # also copy multiple source tables into a single destination table by # providing addtional arguments to `copy_table`. - destination_table = dataset.table(new_table_name) + destination_table_ref = dataset_ref.table(new_table_id) # Create a job to copy the table to the destination table. - job_id = str(uuid.uuid4()) - job = bigquery_client.copy_table( - job_id, destination_table, table) + # Start by creating a job configuration + job_config = bigquery.CopyJobConfig() + + # Configure the job to create the table if it doesn't exist. + job_config.create_disposition = ( + bigquery.job.CreateDisposition.CREATE_IF_NEEDED) - # Create the table if it doesn't exist. - job.create_disposition = ( - google.cloud.bigquery.job.CreateDisposition.CREATE_IF_NEEDED) + copy_job = bigquery_client.copy_table( + table_ref, destination_table_ref, job_config=job_config) - job.begin() # Start the job. print('Waiting for job to finish...') - job.result() + copy_job.result() - print('Table {} copied to {}.'.format(table_name, new_table_name)) + print('Table {} copied to {}.'.format(table_id, new_table_id)) -def delete_table(dataset_name, table_name, project=None): +def delete_table(dataset_id, table_id, project=None): """Deletes a table in a given dataset. If no project is specified, then the currently active project is used. """ bigquery_client = bigquery.Client(project=project) - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) - table.delete() + bigquery_client.delete_table(table_ref) - print('Table {}:{} deleted.'.format(dataset_name, table_name)) + print('Table {}:{} deleted.'.format(dataset_id, table_id)) if __name__ == '__main__': @@ -200,32 +187,32 @@ def delete_table(dataset_name, table_name, project=None): create_dataset_parser = subparsers.add_parser( 'list-datasets', help=list_datasets.__doc__) - create_dataset_parser.add_argument('dataset_name') + create_dataset_parser.add_argument('dataset_id') list_tables_parser = subparsers.add_parser( 'list-tables', help=list_tables.__doc__) - list_tables_parser.add_argument('dataset_name') + list_tables_parser.add_argument('dataset_id') create_table_parser = subparsers.add_parser( 'create-table', help=create_table.__doc__) - create_table_parser.add_argument('dataset_name') - create_table_parser.add_argument('table_name') + create_table_parser.add_argument('dataset_id') + create_table_parser.add_argument('table_id') list_rows_parser = subparsers.add_parser( 'list-rows', help=list_rows.__doc__) - list_rows_parser.add_argument('dataset_name') - list_rows_parser.add_argument('table_name') + list_rows_parser.add_argument('dataset_id') + list_rows_parser.add_argument('table_id') copy_table_parser = subparsers.add_parser( 'copy-table', help=copy_table.__doc__) - copy_table_parser.add_argument('dataset_name') - copy_table_parser.add_argument('table_name') - copy_table_parser.add_argument('new_table_name') + copy_table_parser.add_argument('dataset_id') + copy_table_parser.add_argument('table_id') + copy_table_parser.add_argument('new_table_id') delete_table_parser = subparsers.add_parser( 'delete-table', help=delete_table.__doc__) - delete_table_parser.add_argument('dataset_name') - delete_table_parser.add_argument('table_name') + delete_table_parser.add_argument('dataset_id') + delete_table_parser.add_argument('table_id') args = parser.parse_args() @@ -234,14 +221,14 @@ def delete_table(dataset_name, table_name, project=None): elif args.command == 'list-datasets': list_datasets(args.project) elif args.command == 'create-dataset': - create_dataset(args.dataset_name, args.project) + create_dataset(args.dataset_id, args.project) elif args.command == 'list-tables': - list_tables(args.dataset_name, args.project) + list_tables(args.dataset_id, args.project) elif args.command == 'create-table': - create_table(args.dataset_name, args.table_name, args.project) + create_table(args.dataset_id, args.table_id, args.project) elif args.command == 'list-rows': - list_rows(args.dataset_name, args.table_name, args.project) + list_rows(args.dataset_id, args.table_id, args.project) elif args.command == 'copy-table': - copy_table(args.dataset_name, args.table_name, args.new_table_name) + copy_table(args.dataset_id, args.table_id, args.new_table_id) elif args.command == 'delete-table': - delete_table(args.dataset_name, args.table_name, args.project) + delete_table(args.dataset_id, args.table_id, args.project) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index af368d9a4..5f666ccc6 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -13,6 +13,7 @@ # limitations under the License. from google.cloud import bigquery +from google.cloud.exceptions import NotFound import pytest import snippets @@ -38,17 +39,25 @@ def test_list_datasets(capsys): @pytest.fixture def cleanup_dataset(): - dataset_name = 'test_temporary_dataset' + dataset_id = 'test_temporary_dataset' bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(dataset_name) + dataset_ref = bigquery_client.dataset(dataset_id) - if dataset.exists(): - dataset.delete() + if dataset_exists(dataset_ref, bigquery_client): + bigquery_client.delete_dataset(dataset_ref) - yield dataset_name + yield dataset_id - if dataset.exists(): - dataset.delete() + if dataset_exists(dataset_ref, bigquery_client): + bigquery_client.delete_dataset(dataset_ref) + + +def dataset_exists(dataset, client): + try: + client.get_dataset(dataset) + return True + except NotFound: + return False def test_create_dataset(capsys, cleanup_dataset): @@ -87,46 +96,57 @@ def temporary_table(): """Fixture that returns a factory for tables that do not yet exist and will be automatically deleted after the test.""" bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(DATASET_ID) + dataset_ref = bigquery_client.dataset(DATASET_ID) tables = [] - def factory(table_name): - new_table = dataset.table(table_name) - if new_table.exists(): - new_table.delete() - tables.append(new_table) - return new_table + def factory(table_id): + new_table_ref = dataset_ref.table(table_id) + if table_exists(new_table_ref, bigquery_client): + bigquery_client.delete_table(new_table_ref) + tables.append(new_table_ref) + return new_table_ref yield factory for table in tables: - if table.exists(): - table.delete() + if table_exists(table, bigquery_client): + bigquery_client.delete_table(table) + + +def table_exists(table, client): + try: + client.get_table(table) + return True + except NotFound: + return False def test_create_table(temporary_table): + bigquery_client = bigquery.Client() new_table = temporary_table('test_create_table') - snippets.create_table(DATASET_ID, new_table.name) - assert new_table.exists() + snippets.create_table(DATASET_ID, new_table.table_id) + assert table_exists(new_table, bigquery_client) @pytest.mark.slow def test_copy_table(temporary_table): + bigquery_client = bigquery.Client() new_table = temporary_table('test_copy_table') - snippets.copy_table(DATASET_ID, TABLE_ID, new_table.name) - assert new_table.exists() + snippets.copy_table(DATASET_ID, TABLE_ID, new_table.table_id) + assert table_exists(new_table, bigquery_client) def test_delete_table(): # Create a table to delete bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(DATASET_ID) - table = dataset.table('test_delete_table') + dataset_ref = bigquery_client.dataset(DATASET_ID) + table_ref = dataset_ref.table('test_delete_table') + table = bigquery.Table(table_ref) - if not table.exists(): + if not table_exists(table, bigquery_client): table.schema = [bigquery.SchemaField('id', 'INTEGER')] - table.create() + table = bigquery_client.create_table(table) - snippets.delete_table(DATASET_ID, table.name) + snippets.delete_table(DATASET_ID, table.table_id) - assert not table.exists() + assert not table_exists(table, bigquery_client) diff --git a/samples/snippets/stream_data.py b/samples/snippets/stream_data.py index 7d9970c3a..c54960048 100644 --- a/samples/snippets/stream_data.py +++ b/samples/snippets/stream_data.py @@ -19,7 +19,7 @@ For more information, see the README.rst. Example invocation: - $ python stream_data.py example_dataset example_table \ + $ python stream_data.py example_dataset example_table \\ '["Gandalf", 2000]' The dataset and table should already exist. @@ -32,20 +32,20 @@ from google.cloud import bigquery -def stream_data(dataset_name, table_name, json_data): +def stream_data(dataset_id, table_id, json_data): bigquery_client = bigquery.Client() - dataset = bigquery_client.dataset(dataset_name) - table = dataset.table(table_name) + dataset_ref = bigquery_client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) data = json.loads(json_data) - # Reload the table to get the schema. - table.reload() + # Get the table from the API so that the schema is available. + table = bigquery_client.get_table(table_ref) rows = [data] - errors = table.insert_data(rows) + errors = bigquery_client.create_rows(table, rows) if not errors: - print('Loaded 1 row into {}:{}'.format(dataset_name, table_name)) + print('Loaded 1 row into {}:{}'.format(dataset_id, table_id)) else: print('Errors:') pprint(errors) @@ -55,8 +55,8 @@ def stream_data(dataset_name, table_name, json_data): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_name') - parser.add_argument('table_name') + parser.add_argument('dataset_id') + parser.add_argument('table_id') parser.add_argument( 'json_data', help='The row to load into BigQuery as an array in JSON format.') @@ -64,6 +64,6 @@ def stream_data(dataset_name, table_name, json_data): args = parser.parse_args() stream_data( - args.dataset_name, - args.table_name, + args.dataset_id, + args.table_id, args.json_data) diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index ca585c0a1..a170b6629 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -21,7 +21,6 @@ """ import argparse -import uuid from google.cloud import bigquery from google_auth_oauthlib import flow @@ -29,15 +28,10 @@ def run_query(credentials, project, query): client = bigquery.Client(project=project, credentials=credentials) - query_job = client.run_async_query(str(uuid.uuid4()), query) - - query_job.begin() - query_job.result() # Wait for the job to complete. + query_job = client.query(query) # Print the results. - destination_table = query_job.destination - destination_table.reload() - for row in destination_table.fetch_data(): + for row in query_job.result(): # Wait for the job to complete. print(row) From c009f25c140b914ec5e19d1283a67fd757acdd34 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 1 Nov 2017 12:30:10 -0700 Subject: [PATCH 054/100] Auto-update dependencies. [(#1186)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1186) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 0401c4221..d73dc929d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-cloud-bigquery==0.28.0 google-auth-oauthlib==0.1.1 -pytz==2017.2 +pytz==2017.3 From c504546e9c0f0a1fc6bc252d9cc414e8019ff3dd Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 16 Nov 2017 09:28:12 -0800 Subject: [PATCH 055/100] Auto-update dependencies. [(#1227)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1227) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index d73dc929d..c2b4ac2d5 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-cloud-bigquery==0.28.0 -google-auth-oauthlib==0.1.1 +google-auth-oauthlib==0.2.0 pytz==2017.3 From 0a1858e483fb4d5585ccfc90d6e5300df27912c7 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 1 Dec 2017 09:35:09 -0800 Subject: [PATCH 056/100] /s/buckets/datasets in BigQuery auth sample [(#1242)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1242) --- samples/snippets/auth_snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/auth_snippets.py b/samples/snippets/auth_snippets.py index 9a0c490d9..670c6a1ec 100644 --- a/samples/snippets/auth_snippets.py +++ b/samples/snippets/auth_snippets.py @@ -41,8 +41,8 @@ def explicit(): 'service_account.json') # Make an authenticated API request - buckets = list(bigquery_client.list_datasets()) - print(buckets) + datasets = list(bigquery_client.list_datasets()) + print(datasets) if __name__ == '__main__': From 8780a155391ba4a5bedb664f8b2ab792d5769266 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 1 Dec 2017 17:07:38 -0800 Subject: [PATCH 057/100] BigQuery: rewrite simple app tutorial. - Add region tags for needed dependencies. - Use more relevant query from public datasets. --- samples/snippets/requirements.txt | 2 ++ samples/snippets/simple_app.py | 23 ++++++++++++++--------- samples/snippets/simple_app_test.py | 6 +++--- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index c2b4ac2d5..618a85dfd 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,5 @@ +# [START bigquery_simple_app_pkgs] google-cloud-bigquery==0.28.0 +# [END bigquery_simple_app_pkgs] google-auth-oauthlib==0.2.0 pytz==2017.3 diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 5d0d04e66..9f266ed7a 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -16,20 +16,25 @@ """Simple application that performs a query with BigQuery.""" # [START all] -# [START create_client] +# [START bigquery_simple_app_deps] from google.cloud import bigquery +# [END bigquery_simple_app_deps] -def query_shakespeare(): +def query_stackoverflow(): + # [START create_client] client = bigquery.Client() # [END create_client] # [START run_query] query_job = client.query(""" - #standardSQL - SELECT corpus AS title, COUNT(*) AS unique_words - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY title - ORDER BY unique_words DESC + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC LIMIT 10""") results = query_job.result() # Waits for job to complete. @@ -37,10 +42,10 @@ def query_shakespeare(): # [START print_results] for row in results: - print("{}: {}".format(row.title, row.unique_words)) + print("{} : {} views".format(row.url, row.view_count)) # [END print_results] if __name__ == '__main__': - query_shakespeare() + query_stackoverflow() # [END all] diff --git a/samples/snippets/simple_app_test.py b/samples/snippets/simple_app_test.py index 3733bf6ef..33f9f1adf 100644 --- a/samples/snippets/simple_app_test.py +++ b/samples/snippets/simple_app_test.py @@ -15,7 +15,7 @@ import simple_app -def test_query_shakespeare(capsys): - simple_app.query_shakespeare() +def test_query_stackoverflow(capsys): + simple_app.query_stackoverflow() out, _ = capsys.readouterr() - assert 'hamlet' in out + assert 'views' in out From eaafc3c0232f2e398cc501e1132111e65b97e4fe Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 7 Dec 2017 09:46:43 -0800 Subject: [PATCH 058/100] Fix broken link to core Client service account helper. [(#1256)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1256) --- samples/snippets/auth_snippets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/snippets/auth_snippets.py b/samples/snippets/auth_snippets.py index 670c6a1ec..1dc6fddd2 100644 --- a/samples/snippets/auth_snippets.py +++ b/samples/snippets/auth_snippets.py @@ -35,8 +35,7 @@ def explicit(): # Explicitly use service account credentials by specifying the private key # file. All clients in google-cloud-python have this helper, see - # https://google-cloud-python.readthedocs.io/en/latest/core/modules.html - # #google.cloud.client.Client.from_service_account_json + # https://googlecloudplatform.github.io/google-cloud-python/latest/core/auth.html#service-accounts bigquery_client = bigquery.Client.from_service_account_json( 'service_account.json') From a83001f167f449b23bce1cc591fcde1a02db737a Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Thu, 7 Dec 2017 10:34:29 -0800 Subject: [PATCH 059/100] Added "Open in Cloud Shell" buttons to README files [(#1254)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1254) --- samples/snippets/README.rst | 168 +++++++++++++++++++-------------- samples/snippets/README.rst.in | 2 + 2 files changed, 100 insertions(+), 70 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 1a3b889a3..b0e5810a3 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -3,6 +3,10 @@ Google BigQuery Python Samples =============================================================================== +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/README.rst + + This directory contains samples for Google BigQuery. `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on analyzing data to find meaningful insights, use familiar SQL, and take advantage of our pay-as-you-go model. @@ -17,39 +21,20 @@ Setup Authentication ++++++++++++++ -Authentication is typically done through `Application Default Credentials`_, -which means you do not have to change the code to authenticate as long as -your environment has credentials. You have a few options for setting up -authentication: - -#. When running locally, use the `Google Cloud SDK`_ - - .. code-block:: bash - - gcloud auth application-default login - - -#. When running on App Engine or Compute Engine, credentials are already - set-up. However, you may need to configure your Compute Engine instance - with `additional scopes`_. - -#. You can create a `Service Account key file`_. This file can be used to - authenticate to Google Cloud Platform services from any environment. To use - the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to - the path to the key file, for example: - - .. code-block:: bash - - export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. -.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow -.. _additional scopes: https://cloud.google.com/compute/docs/authentication#using -.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started Install Dependencies ++++++++++++++++++++ -#. Install `pip`_ and `virtualenv`_ if you do not already have them. +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup #. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. @@ -73,6 +58,10 @@ Samples Simple Application +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -85,6 +74,10 @@ To run this sample: Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/quickstart.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -97,6 +90,10 @@ To run this sample: Query +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -106,23 +103,23 @@ To run this sample: $ python query.py usage: query.py [-h] [--use_standard_sql] - [--destination_table DESTINATION_TABLE] - query - + [--destination_table DESTINATION_TABLE] + query + Command-line application to perform queries in BigQuery. - + For more information, see the README.rst. - + Example invocation: $ python query.py '#standardSQL SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus ORDER BY corpus' - + positional arguments: query BigQuery SQL Query. - + optional arguments: -h, --help show this help message and exit --use_standard_sql Use standard SQL syntax. @@ -131,9 +128,14 @@ To run this sample: my_dataset.my_table + Parameterized Query +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query_params.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -143,15 +145,15 @@ To run this sample: $ python query_params.py usage: query_params.py [-h] {named,positional,array,timestamp,struct} ... - + Command-line app to perform queries with parameters in BigQuery. - + For more information, see the README.rst. - + Example invocation: $ python query_params.py named 'romeoandjuliet' 100 $ python query_params.py positional 'romeoandjuliet' 100 - + positional arguments: {named,positional,array,timestamp,struct} samples @@ -160,14 +162,19 @@ To run this sample: array Run a query with an array parameter. timestamp Run a query with a timestamp parameter. struct Run a query with a struct parameter. - + optional arguments: -h, --help show this help message and exit + Snippets +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/snippets.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -179,16 +186,16 @@ To run this sample: usage: snippets.py [-h] [--project PROJECT] {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} ... - + Samples that demonstrate basic operations in the BigQuery API. - + For more information, see the README.rst. - + Example invocation: $ python snippets.py list-datasets - + The dataset and table should already exist. - + positional arguments: {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} list-projects @@ -212,15 +219,20 @@ To run this sample: currently active project is used. delete-table Deletes a table in a given dataset. If no project is specified, then the currently active project is used. - + optional arguments: -h, --help show this help message and exit --project PROJECT + Load data from a file +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_file.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -230,29 +242,34 @@ To run this sample: $ python load_data_from_file.py usage: load_data_from_file.py [-h] dataset_id table_id source_file_name - + Loads data into BigQuery from a local file. - + For more information, see the README.rst. - + Example invocation: $ python load_data_from_file.py example_dataset example_table \ example-data.csv - + The dataset and table should already exist. - + positional arguments: dataset_id table_id source_file_name Path to a .csv file to upload. - + optional arguments: -h, --help show this help message and exit + Load data from Cloud Storage +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_gcs.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -262,30 +279,35 @@ To run this sample: $ python load_data_from_gcs.py usage: load_data_from_gcs.py [-h] dataset_id table_id source - + Loads data into BigQuery from an object in Google Cloud Storage. - + For more information, see the README.rst. - + Example invocation: $ python load_data_from_gcs.py example_dataset example_table \ gs://example-bucket/example-data.csv - + The dataset and table should already exist. - + positional arguments: dataset_id table_id source The Google Cloud Storage object to load. Must be in the format gs://bucket_name/object_name - + optional arguments: -h, --help show this help message and exit + Load streaming data +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/stream_data.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -295,29 +317,34 @@ To run this sample: $ python stream_data.py usage: stream_data.py [-h] dataset_id table_id json_data - + Loads a single row of data directly into BigQuery. - + For more information, see the README.rst. - + Example invocation: $ python stream_data.py example_dataset example_table \ '["Gandalf", 2000]' - + The dataset and table should already exist. - + positional arguments: dataset_id table_id json_data The row to load into BigQuery as an array in JSON format. - + optional arguments: -h, --help show this help message and exit + Export data to Cloud Storage +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/export_data_to_gcs.py;bigquery/cloud-client/README.rst + + To run this sample: @@ -327,29 +354,30 @@ To run this sample: $ python export_data_to_gcs.py usage: export_data_to_gcs.py [-h] dataset_id table_id destination - + Exports data from BigQuery to an object in Google Cloud Storage. - + For more information, see the README.rst. - + Example invocation: $ python export_data_to_gcs.py example_dataset example_table \ gs://example-bucket/example-data.csv - + The dataset and table should already exist. - + positional arguments: dataset_id table_id destination The destination Google Cloud Storage object. Must be in the format gs://bucket_name/object_name - + optional arguments: -h, --help show this help message and exit + The client library ------------------------------------------------------------------------------- diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in index 61c66ab43..55572c070 100644 --- a/samples/snippets/README.rst.in +++ b/samples/snippets/README.rst.in @@ -43,3 +43,5 @@ samples: show_help: true cloud_client_library: true + +folder: bigquery/cloud-client \ No newline at end of file From a55e274ca2cfcdd164490cd04cb2ff729b016680 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 11 Dec 2017 10:43:12 -0800 Subject: [PATCH 060/100] Move imports into bigquery samples Change-Id: I0be8d3e0778352a8b814258f83d13d131cb5054e --- samples/snippets/user_credentials.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index a170b6629..9ab062fb3 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -22,11 +22,10 @@ import argparse -from google.cloud import bigquery -from google_auth_oauthlib import flow - def run_query(credentials, project, query): + from google.cloud import bigquery + client = bigquery.Client(project=project, credentials=credentials) query_job = client.query(query) @@ -36,6 +35,8 @@ def run_query(credentials, project, query): def authenticate_and_query(project, query, launch_browser=True): + from google_auth_oauthlib import flow + appflow = flow.InstalledAppFlow.from_client_secrets_file( 'client_secrets.json', scopes=['https://www.googleapis.com/auth/bigquery']) From 7d80661f9ba4ea5c11f22e9d5c7e26d4f4560dca Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 12 Dec 2017 09:26:42 -0800 Subject: [PATCH 061/100] Auto-update dependencies. [(#1272)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1272) * Auto-update dependencies. * Update requirements.txt --- samples/snippets/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 618a85dfd..c2b4ac2d5 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,3 @@ -# [START bigquery_simple_app_pkgs] google-cloud-bigquery==0.28.0 -# [END bigquery_simple_app_pkgs] google-auth-oauthlib==0.2.0 pytz==2017.3 From db715f3490ce58ad0943fac53417eb23b582b68a Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 9 Jan 2018 09:06:58 -0800 Subject: [PATCH 062/100] Auto-update dependencies. [(#1307)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1307) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index c2b4ac2d5..659797618 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.28.0 +google-cloud-bigquery==0.29.0 google-auth-oauthlib==0.2.0 pytz==2017.3 From 69f2aff20b9d4d3225a821866b1d4197c9d6df81 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 9 Feb 2018 10:46:48 -0800 Subject: [PATCH 063/100] Auto-update dependencies. [(#1355)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1355) --- samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 659797618..8ec538738 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.29.0 +google-cloud-bigquery==0.30.0 google-auth-oauthlib==0.2.0 -pytz==2017.3 +pytz==2018.3 From b1a430f6eebd2ab118ce1e08bc480e611697c4ea Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 5 Mar 2018 12:28:55 -0800 Subject: [PATCH 064/100] Auto-update dependencies. [(#1377)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1377) * Auto-update dependencies. * Update requirements.txt --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 8ec538738..62490b384 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.30.0 +google-cloud-bigquery==0.31.0 google-auth-oauthlib==0.2.0 pytz==2018.3 From 5d143406e607c81fbc1e9c68bf3570362424a7ef Mon Sep 17 00:00:00 2001 From: chenyumic Date: Fri, 6 Apr 2018 22:57:36 -0700 Subject: [PATCH 065/100] Regenerate the README files and fix the Open in Cloud Shell link for some samples [(#1441)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1441) --- samples/snippets/README.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index b0e5810a3..60b51a7ca 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -12,7 +12,7 @@ This directory contains samples for Google BigQuery. `Google BigQuery`_ is Googl -.. _Google BigQuery: https://cloud.google.com/bigquery/docs +.. _Google BigQuery: https://cloud.google.com/bigquery/docs Setup ------------------------------------------------------------------------------- @@ -59,7 +59,7 @@ Simple Application +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst @@ -75,7 +75,7 @@ Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/quickstart.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/quickstart.py,bigquery/cloud-client/README.rst @@ -91,7 +91,7 @@ Query +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query.py,bigquery/cloud-client/README.rst @@ -133,7 +133,7 @@ Parameterized Query +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query_params.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query_params.py,bigquery/cloud-client/README.rst @@ -172,7 +172,7 @@ Snippets +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/snippets.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/snippets.py,bigquery/cloud-client/README.rst @@ -230,7 +230,7 @@ Load data from a file +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_file.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_file.py,bigquery/cloud-client/README.rst @@ -267,7 +267,7 @@ Load data from Cloud Storage +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_gcs.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_gcs.py,bigquery/cloud-client/README.rst @@ -305,7 +305,7 @@ Load streaming data +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/stream_data.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/stream_data.py,bigquery/cloud-client/README.rst @@ -342,7 +342,7 @@ Export data to Cloud Storage +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/export_data_to_gcs.py;bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/export_data_to_gcs.py,bigquery/cloud-client/README.rst From f5dd3b2067a6ae0ff18b18079b5f281299affdcf Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 17 Apr 2018 15:03:48 -0700 Subject: [PATCH 066/100] BigQuery: Remove unused samples [(#1455)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1455) * updates simple app region tags to standard * removes samples redundant with google-cloud-python * adds user_credentials.py to readme generator * shows command help in user_credentials section --- samples/snippets/README.rst | 283 +------------------ samples/snippets/README.rst.in | 28 +- samples/snippets/auth_snippets.py | 61 ---- samples/snippets/auth_snippets_test.py | 33 --- samples/snippets/export_data_to_gcs.py | 61 ---- samples/snippets/export_data_to_gcs_test.py | 31 -- samples/snippets/load_data_from_file.py | 66 ----- samples/snippets/load_data_from_file_test.py | 33 --- samples/snippets/load_data_from_gcs.py | 61 ---- samples/snippets/load_data_from_gcs_test.py | 33 --- samples/snippets/query.py | 103 ------- samples/snippets/query_params.py | 225 --------------- samples/snippets/query_params_test.py | 52 ---- samples/snippets/query_test.py | 66 ----- samples/snippets/simple_app.py | 16 +- samples/snippets/snippets.py | 234 --------------- samples/snippets/snippets_test.py | 152 ---------- samples/snippets/stream_data.py | 69 ----- samples/snippets/stream_data_test.py | 29 -- 19 files changed, 26 insertions(+), 1610 deletions(-) delete mode 100644 samples/snippets/auth_snippets.py delete mode 100644 samples/snippets/auth_snippets_test.py delete mode 100644 samples/snippets/export_data_to_gcs.py delete mode 100644 samples/snippets/export_data_to_gcs_test.py delete mode 100644 samples/snippets/load_data_from_file.py delete mode 100644 samples/snippets/load_data_from_file_test.py delete mode 100644 samples/snippets/load_data_from_gcs.py delete mode 100644 samples/snippets/load_data_from_gcs_test.py delete mode 100755 samples/snippets/query.py delete mode 100644 samples/snippets/query_params.py delete mode 100644 samples/snippets/query_params_test.py delete mode 100644 samples/snippets/query_test.py delete mode 100644 samples/snippets/snippets.py delete mode 100644 samples/snippets/snippets_test.py delete mode 100644 samples/snippets/stream_data.py delete mode 100644 samples/snippets/stream_data_test.py diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 60b51a7ca..cca0ff3e5 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -55,22 +55,6 @@ Install Dependencies Samples ------------------------------------------------------------------------------- -Simple Application -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python simple_app.py - - Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -87,92 +71,11 @@ To run this sample: $ python quickstart.py -Query -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python query.py - - usage: query.py [-h] [--use_standard_sql] - [--destination_table DESTINATION_TABLE] - query - - Command-line application to perform queries in BigQuery. - - For more information, see the README.rst. - - Example invocation: - $ python query.py '#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus' - - positional arguments: - query BigQuery SQL Query. - - optional arguments: - -h, --help show this help message and exit - --use_standard_sql Use standard SQL syntax. - --destination_table DESTINATION_TABLE - Destination table to use for results. Example: - my_dataset.my_table - - - -Parameterized Query -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query_params.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python query_params.py - - usage: query_params.py [-h] {named,positional,array,timestamp,struct} ... - - Command-line app to perform queries with parameters in BigQuery. - - For more information, see the README.rst. - - Example invocation: - $ python query_params.py named 'romeoandjuliet' 100 - $ python query_params.py positional 'romeoandjuliet' 100 - - positional arguments: - {named,positional,array,timestamp,struct} - samples - named Run a query with named parameters. - positional Run a query with positional parameters. - array Run a query with an array parameter. - timestamp Run a query with a timestamp parameter. - struct Run a query with a struct parameter. - - optional arguments: - -h, --help show this help message and exit - - - -Snippets +Simple Application +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/snippets.py,bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst @@ -181,56 +84,14 @@ To run this sample: .. code-block:: bash - $ python snippets.py - - usage: snippets.py [-h] [--project PROJECT] - {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} - ... - - Samples that demonstrate basic operations in the BigQuery API. - - For more information, see the README.rst. - - Example invocation: - $ python snippets.py list-datasets - - The dataset and table should already exist. - - positional arguments: - {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} - list-projects - list-datasets Lists all datasets in a given project. If no project - is specified, then the currently active project is - used. - list-datasets Lists all datasets in a given project. If no project - is specified, then the currently active project is - used. - list-tables Lists all of the tables in a given dataset. If no - project is specified, then the currently active - project is used. - create-table Creates a simple table in the given dataset. If no - project is specified, then the currently active - project is used. - list-rows Prints rows in the given table. Will print 25 rows at - most for brevity as tables can contain large amounts - of rows. If no project is specified, then the - currently active project is used. - copy-table Copies a table. If no project is specified, then the - currently active project is used. - delete-table Deletes a table in a given dataset. If no project is - specified, then the currently active project is used. - - optional arguments: - -h, --help show this help message and exit - --project PROJECT - + $ python simple_app.py -Load data from a file +User Credentials +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_file.py,bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/user_credentials.py,bigquery/cloud-client/README.rst @@ -239,140 +100,22 @@ To run this sample: .. code-block:: bash - $ python load_data_from_file.py - - usage: load_data_from_file.py [-h] dataset_id table_id source_file_name - - Loads data into BigQuery from a local file. + $ python user_credentials.py - For more information, see the README.rst. + usage: user_credentials.py [-h] [--launch-browser] project query - Example invocation: - $ python load_data_from_file.py example_dataset example_table \ - example-data.csv + Command-line application to run a query using user credentials. - The dataset and table should already exist. + You must supply a client secrets file, which would normally be bundled with + your application. positional arguments: - dataset_id - table_id - source_file_name Path to a .csv file to upload. + project Project to use for BigQuery billing. + query BigQuery SQL Query. optional arguments: -h, --help show this help message and exit - - - -Load data from Cloud Storage -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_gcs.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python load_data_from_gcs.py - - usage: load_data_from_gcs.py [-h] dataset_id table_id source - - Loads data into BigQuery from an object in Google Cloud Storage. - - For more information, see the README.rst. - - Example invocation: - $ python load_data_from_gcs.py example_dataset example_table \ - gs://example-bucket/example-data.csv - - The dataset and table should already exist. - - positional arguments: - dataset_id - table_id - source The Google Cloud Storage object to load. Must be in the format - gs://bucket_name/object_name - - optional arguments: - -h, --help show this help message and exit - - - -Load streaming data -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/stream_data.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python stream_data.py - - usage: stream_data.py [-h] dataset_id table_id json_data - - Loads a single row of data directly into BigQuery. - - For more information, see the README.rst. - - Example invocation: - $ python stream_data.py example_dataset example_table \ - '["Gandalf", 2000]' - - The dataset and table should already exist. - - positional arguments: - dataset_id - table_id - json_data The row to load into BigQuery as an array in JSON format. - - optional arguments: - -h, --help show this help message and exit - - - -Export data to Cloud Storage -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/export_data_to_gcs.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python export_data_to_gcs.py - - usage: export_data_to_gcs.py [-h] dataset_id table_id destination - - Exports data from BigQuery to an object in Google Cloud Storage. - - For more information, see the README.rst. - - Example invocation: - $ python export_data_to_gcs.py example_dataset example_table \ - gs://example-bucket/example-data.csv - - The dataset and table should already exist. - - positional arguments: - dataset_id - table_id - destination The destination Google Cloud Storage object. Must be in the - format gs://bucket_name/object_name - - optional arguments: - -h, --help show this help message and exit + --launch-browser Use a local server flow to authenticate. diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in index 55572c070..008b51795 100644 --- a/samples/snippets/README.rst.in +++ b/samples/snippets/README.rst.in @@ -4,7 +4,7 @@ product: name: Google BigQuery short_name: BigQuery url: https://cloud.google.com/bigquery/docs - description: > + description: > `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on @@ -16,30 +16,12 @@ setup: - install_deps samples: -- name: Simple Application - file: simple_app.py - name: Quickstart file: quickstart.py -- name: Query - file: query.py - show_help: true -- name: Parameterized Query - file: query_params.py - show_help: true -- name: Snippets - file: snippets.py - show_help: true -- name: Load data from a file - file: load_data_from_file.py - show_help: true -- name: Load data from Cloud Storage - file: load_data_from_gcs.py - show_help: true -- name: Load streaming data - file: stream_data.py - show_help: true -- name: Export data to Cloud Storage - file: export_data_to_gcs.py +- name: Simple Application + file: simple_app.py +- name: User Credentials + file: user_credentials.py show_help: true cloud_client_library: true diff --git a/samples/snippets/auth_snippets.py b/samples/snippets/auth_snippets.py deleted file mode 100644 index 1dc6fddd2..000000000 --- a/samples/snippets/auth_snippets.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Demonstrates how to authenticate to Google BigQuery using the Google Cloud -Client Libraries.""" - -import argparse - - -def implicit(): - from google.cloud import bigquery - - # If you don't specify credentials when constructing the client, the - # client library will look for credentials in the environment. - bigquery_client = bigquery.Client() - - # Make an authenticated API request - datasets = list(bigquery_client.list_datasets()) - print(datasets) - - -def explicit(): - from google.cloud import bigquery - - # Explicitly use service account credentials by specifying the private key - # file. All clients in google-cloud-python have this helper, see - # https://googlecloudplatform.github.io/google-cloud-python/latest/core/auth.html#service-accounts - bigquery_client = bigquery.Client.from_service_account_json( - 'service_account.json') - - # Make an authenticated API request - datasets = list(bigquery_client.list_datasets()) - print(datasets) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - subparsers = parser.add_subparsers(dest='command') - subparsers.add_parser('implicit', help=implicit.__doc__) - subparsers.add_parser('explicit', help=explicit.__doc__) - - args = parser.parse_args() - - if args.command == 'implicit': - implicit() - elif args.command == 'explicit': - explicit() diff --git a/samples/snippets/auth_snippets_test.py b/samples/snippets/auth_snippets_test.py deleted file mode 100644 index 5b5f2cac0..000000000 --- a/samples/snippets/auth_snippets_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import mock - -import auth_snippets - - -def test_implicit(): - auth_snippets.implicit() - - -def test_explicit(): - with open(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) as creds_file: - creds_file_data = creds_file.read() - - open_mock = mock.mock_open(read_data=creds_file_data) - - with mock.patch('io.open', open_mock): - auth_snippets.explicit() diff --git a/samples/snippets/export_data_to_gcs.py b/samples/snippets/export_data_to_gcs.py deleted file mode 100644 index 5993ef0f6..000000000 --- a/samples/snippets/export_data_to_gcs.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Exports data from BigQuery to an object in Google Cloud Storage. - -For more information, see the README.rst. - -Example invocation: - $ python export_data_to_gcs.py example_dataset example_table \\ - gs://example-bucket/example-data.csv - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def export_data_to_gcs(dataset_id, table_id, destination): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - job = bigquery_client.extract_table(table_ref, destination) - - job.result() # Waits for job to complete - - print('Exported {}:{} to {}'.format( - dataset_id, table_id, destination)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'destination', help='The destination Google Cloud Storage object. ' - 'Must be in the format gs://bucket_name/object_name') - - args = parser.parse_args() - - export_data_to_gcs( - args.dataset_id, - args.table_id, - args.destination) diff --git a/samples/snippets/export_data_to_gcs_test.py b/samples/snippets/export_data_to_gcs_test.py deleted file mode 100644 index a41cfd226..000000000 --- a/samples/snippets/export_data_to_gcs_test.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import export_data_to_gcs - -BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_table' - - -def test_export_data_to_gcs(capsys): - export_data_to_gcs.export_data_to_gcs( - DATASET_ID, - TABLE_ID, - 'gs://{}/test-export-data-to-gcs.csv'.format(BUCKET)) - - out, _ = capsys.readouterr() - - assert 'Exported' in out diff --git a/samples/snippets/load_data_from_file.py b/samples/snippets/load_data_from_file.py deleted file mode 100644 index e311daa1e..000000000 --- a/samples/snippets/load_data_from_file.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Loads data into BigQuery from a local file. - -For more information, see the README.rst. - -Example invocation: - $ python load_data_from_file.py example_dataset example_table \\ - example-data.csv - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def load_data_from_file(dataset_id, table_id, source_file_name): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - with open(source_file_name, 'rb') as source_file: - # This example uses CSV, but you can use other formats. - # See https://cloud.google.com/bigquery/loading-data - job_config = bigquery.LoadJobConfig() - job_config.source_format = 'text/csv' - job = bigquery_client.load_table_from_file( - source_file, table_ref, job_config=job_config) - - job.result() # Waits for job to complete - - print('Loaded {} rows into {}:{}.'.format( - job.output_rows, dataset_id, table_id)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'source_file_name', help='Path to a .csv file to upload.') - - args = parser.parse_args() - - load_data_from_file( - args.dataset_id, - args.table_id, - args.source_file_name) diff --git a/samples/snippets/load_data_from_file_test.py b/samples/snippets/load_data_from_file_test.py deleted file mode 100644 index 960fe62c9..000000000 --- a/samples/snippets/load_data_from_file_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import load_data_from_file - -RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -def test_load_table(capsys): - data_path = os.path.join(RESOURCES, 'data.csv') - - load_data_from_file.load_data_from_file( - DATASET_ID, - TABLE_ID, - data_path) - - out, _ = capsys.readouterr() - - assert 'Loaded 1 rows' in out diff --git a/samples/snippets/load_data_from_gcs.py b/samples/snippets/load_data_from_gcs.py deleted file mode 100644 index 285e6d1b2..000000000 --- a/samples/snippets/load_data_from_gcs.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Loads data into BigQuery from an object in Google Cloud Storage. - -For more information, see the README.rst. - -Example invocation: - $ python load_data_from_gcs.py example_dataset example_table \\ - gs://example-bucket/example-data.csv - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def load_data_from_gcs(dataset_id, table_id, source): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - job = bigquery_client.load_table_from_uri(source, table_ref) - - job.result() # Waits for job to complete - - print('Loaded {} rows into {}:{}.'.format( - job.output_rows, dataset_id, table_id)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'source', help='The Google Cloud Storage object to load. Must be in ' - 'the format gs://bucket_name/object_name') - - args = parser.parse_args() - - load_data_from_gcs( - args.dataset_id, - args.table_id, - args.source) diff --git a/samples/snippets/load_data_from_gcs_test.py b/samples/snippets/load_data_from_gcs_test.py deleted file mode 100644 index dbd39fc5c..000000000 --- a/samples/snippets/load_data_from_gcs_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import load_data_from_gcs - -BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -def test_load_table(capsys): - cloud_storage_input_uri = 'gs://{}/data.csv'.format(BUCKET) - - load_data_from_gcs.load_data_from_gcs( - DATASET_ID, - TABLE_ID, - cloud_storage_input_uri) - - out, _ = capsys.readouterr() - - assert 'Loaded 1 rows' in out diff --git a/samples/snippets/query.py b/samples/snippets/query.py deleted file mode 100755 index 19605bbad..000000000 --- a/samples/snippets/query.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to perform queries in BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python query.py '#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus' -""" - -import argparse - -from google.cloud import bigquery - - -def query(query): - client = bigquery.Client() - query_job = client.query(query) - - # Print the results. - for row in query_job.result(): # Waits for job to complete. - print(row) - - -def query_standard_sql(query): - client = bigquery.Client() - job_config = bigquery.QueryJobConfig() - - # Set use_legacy_sql to False to use standard SQL syntax. - # Note that queries are treated as standard SQL by default. - job_config.use_legacy_sql = False - query_job = client.query(query, job_config=job_config) - - # Print the results. - for row in query_job.result(): # Waits for job to complete. - print(row) - - -def query_destination_table(query, dest_dataset_id, dest_table_id): - client = bigquery.Client() - job_config = bigquery.QueryJobConfig() - - # Allow for query results larger than the maximum response size. - job_config.allow_large_results = True - - # When large results are allowed, a destination table must be set. - dest_dataset_ref = client.dataset(dest_dataset_id) - dest_table_ref = dest_dataset_ref.table(dest_table_id) - job_config.destination = dest_table_ref - - # Allow the results table to be overwritten. - job_config.write_disposition = 'WRITE_TRUNCATE' - - query_job = client.query(query, job_config=job_config) - - # Print the results. - for row in query_job.result(): # Waits for job to complete. - print(row) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('query', help='BigQuery SQL Query.') - parser.add_argument( - '--use_standard_sql', - action='store_true', - help='Use standard SQL syntax.') - parser.add_argument( - '--destination_table', - type=str, - help=( - 'Destination table to use for results. ' - 'Example: my_dataset.my_table')) - - args = parser.parse_args() - - if args.use_standard_sql: - query_standard_sql(args.query) - elif args.destination_table: - dataset, table = args.destination_table.split('.') - query_destination_table(args.query, dataset, table) - else: - query(args.query) diff --git a/samples/snippets/query_params.py b/samples/snippets/query_params.py deleted file mode 100644 index 594f51c75..000000000 --- a/samples/snippets/query_params.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line app to perform queries with parameters in BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python query_params.py named 'romeoandjuliet' 100 - $ python query_params.py positional 'romeoandjuliet' 100 -""" - -import argparse -import datetime - -from google.cloud import bigquery -import pytz - - -def query_positional_params(corpus, min_word_count): - client = bigquery.Client() - query = """ - SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = ? - AND word_count >= ? - ORDER BY word_count DESC; - """ - # Set the name to None to use positional parameters (? symbol in the - # query). Note that you cannot mix named and positional parameters. - # See: https://cloud.google.com/bigquery/docs/parameterized-queries/ - query_params = [ - bigquery.ScalarQueryParameter(None, 'STRING', corpus), - bigquery.ScalarQueryParameter(None, 'INT64', min_word_count) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Wait for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_named_params(corpus, min_word_count): - client = bigquery.Client() - query = """ - SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = @corpus - AND word_count >= @min_word_count - ORDER BY word_count DESC; - """ - query_params = [ - bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), - bigquery.ScalarQueryParameter( - 'min_word_count', 'INT64', min_word_count) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Wait for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_array_params(gender, states): - client = bigquery.Client() - query = """ - SELECT name, sum(number) as count - FROM `bigquery-public-data.usa_names.usa_1910_2013` - WHERE gender = @gender - AND state IN UNNEST(@states) - GROUP BY name - ORDER BY count DESC - LIMIT 10; - """ - query_params = [ - bigquery.ScalarQueryParameter('gender', 'STRING', gender), - bigquery.ArrayQueryParameter('states', 'STRING', states) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Wait for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_timestamp_params(year, month, day, hour, minute): - client = bigquery.Client() - query = 'SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);' - query_params = [ - bigquery.ScalarQueryParameter( - 'ts_value', - 'TIMESTAMP', - datetime.datetime(year, month, day, hour, minute, tzinfo=pytz.UTC)) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Waits for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_struct_params(x, y): - client = bigquery.Client() - query = 'SELECT @struct_value AS s;' - query_params = [ - bigquery.StructQueryParameter( - 'struct_value', - bigquery.ScalarQueryParameter('x', 'INT64', x), - bigquery.ScalarQueryParameter('y', 'STRING', y) - ) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Waits for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - subparsers = parser.add_subparsers(dest='sample', help='samples') - named_parser = subparsers.add_parser( - 'named', - help='Run a query with named parameters.') - named_parser.add_argument( - 'corpus', - help='Corpus to search from Shakespeare dataset.') - named_parser.add_argument( - 'min_word_count', - help='Minimum count of words to query.', - type=int) - positional_parser = subparsers.add_parser( - 'positional', - help='Run a query with positional parameters.') - positional_parser.add_argument( - 'corpus', - help='Corpus to search from Shakespeare dataset.') - positional_parser.add_argument( - 'min_word_count', - help='Minimum count of words to query.', - type=int) - array_parser = subparsers.add_parser( - 'array', - help='Run a query with an array parameter.') - array_parser.add_argument( - 'gender', - choices=['F', 'M'], - help='Gender of baby in the Social Security baby names database.') - array_parser.add_argument( - 'states', - help='U.S. States to consider for popular baby names.', - nargs='+') - timestamp_parser = subparsers.add_parser( - 'timestamp', - help='Run a query with a timestamp parameter.') - timestamp_parser.add_argument('year', type=int) - timestamp_parser.add_argument('month', type=int) - timestamp_parser.add_argument('day', type=int) - timestamp_parser.add_argument('hour', type=int) - timestamp_parser.add_argument('minute', type=int) - struct_parser = subparsers.add_parser( - 'struct', - help='Run a query with a struct parameter.') - struct_parser.add_argument('x', help='Integer for x', type=int) - struct_parser.add_argument('y', help='String for y') - args = parser.parse_args() - - if args.sample == 'named': - query_named_params(args.corpus, args.min_word_count) - elif args.sample == 'positional': - query_positional_params(args.corpus, args.min_word_count) - elif args.sample == 'array': - query_array_params(args.gender, args.states) - elif args.sample == 'timestamp': - query_timestamp_params( - args.year, args.month, args.day, args.hour, args.minute) - elif args.sample == 'struct': - query_struct_params(args.x, args.y) - else: - print('Unexpected value for sample') diff --git a/samples/snippets/query_params_test.py b/samples/snippets/query_params_test.py deleted file mode 100644 index f4b493137..000000000 --- a/samples/snippets/query_params_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import query_params - - -def test_query_array_params(capsys): - query_params.query_array_params( - gender='M', - states=['WA', 'WI', 'WV', 'WY']) - out, _ = capsys.readouterr() - assert 'James' in out - - -def test_query_named_params(capsys): - query_params.query_named_params( - corpus='romeoandjuliet', - min_word_count=100) - out, _ = capsys.readouterr() - assert 'the' in out - - -def test_query_positional_params(capsys): - query_params.query_positional_params( - corpus='romeoandjuliet', - min_word_count=100) - out, _ = capsys.readouterr() - assert 'the' in out - - -def test_query_struct_params(capsys): - query_params.query_struct_params(765, "hello world") - out, _ = capsys.readouterr() - assert '765' in out - assert 'hello world' in out - - -def test_query_timestamp_params(capsys): - query_params.query_timestamp_params(2016, 12, 7, 8, 0) - out, _ = capsys.readouterr() - assert '2016, 12, 7, 9, 0' in out diff --git a/samples/snippets/query_test.py b/samples/snippets/query_test.py deleted file mode 100644 index 3d456cb59..000000000 --- a/samples/snippets/query_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import query - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_destination_table' - - -def test_query(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query_string = '''#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus - LIMIT 10;''' - - query.query(query_string) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out - - -def test_query_standard_sql(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query_string = '''SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus - LIMIT 10;''' - - query.query_standard_sql(query_string) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out - - -def test_query_destination_table(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query_string = '''#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus - LIMIT 10;''' - - query.query_destination_table(query_string, DATASET_ID, TABLE_ID) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 9f266ed7a..a09e97f12 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -15,17 +15,17 @@ # limitations under the License. """Simple application that performs a query with BigQuery.""" -# [START all] +# [START bigquery_simple_app_all] # [START bigquery_simple_app_deps] from google.cloud import bigquery # [END bigquery_simple_app_deps] def query_stackoverflow(): - # [START create_client] + # [START bigquery_simple_app_client] client = bigquery.Client() - # [END create_client] - # [START run_query] + # [END bigquery_simple_app_client] + # [START bigquery_simple_app_query] query_job = client.query(""" SELECT CONCAT( @@ -38,14 +38,14 @@ def query_stackoverflow(): LIMIT 10""") results = query_job.result() # Waits for job to complete. - # [END run_query] + # [END bigquery_simple_app_query] - # [START print_results] + # [START bigquery_simple_app_print] for row in results: print("{} : {} views".format(row.url, row.view_count)) - # [END print_results] + # [END bigquery_simple_app_print] if __name__ == '__main__': query_stackoverflow() -# [END all] +# [END bigquery_simple_app_all] diff --git a/samples/snippets/snippets.py b/samples/snippets/snippets.py deleted file mode 100644 index ee75f7fc2..000000000 --- a/samples/snippets/snippets.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Samples that demonstrate basic operations in the BigQuery API. - -For more information, see the README.rst. - -Example invocation: - $ python snippets.py list-datasets - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def list_projects(): - bigquery_client = bigquery.Client() - - for project in bigquery_client.list_projects(): - print(project.project_id) - - -def list_datasets(project=None): - """Lists all datasets in a given project. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - - for dataset in bigquery_client.list_datasets(): - print(dataset.dataset_id) - - -def create_dataset(dataset_id, project=None): - """Craetes a dataset in a given project. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - - dataset_ref = bigquery_client.dataset(dataset_id) - - dataset = bigquery_client.create_dataset(bigquery.Dataset(dataset_ref)) - - print('Created dataset {}.'.format(dataset.dataset_id)) - - -def list_tables(dataset_id, project=None): - """Lists all of the tables in a given dataset. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - - for table in bigquery_client.list_dataset_tables(dataset_ref): - print(table.table_id) - - -def create_table(dataset_id, table_id, project=None): - """Creates a simple table in the given dataset. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - - table_ref = dataset_ref.table(table_id) - table = bigquery.Table(table_ref) - - # Set the table schema - table.schema = ( - bigquery.SchemaField('Name', 'STRING'), - bigquery.SchemaField('Age', 'INTEGER'), - bigquery.SchemaField('Weight', 'FLOAT'), - ) - - table = bigquery_client.create_table(table) - - print('Created table {} in dataset {}.'.format(table_id, dataset_id)) - - -def list_rows(dataset_id, table_id, project=None): - """Prints rows in the given table. - - Will print 25 rows at most for brevity as tables can contain large amounts - of rows. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - # Get the table from the API so that the schema is available. - table = bigquery_client.get_table(table_ref) - - # Load at most 25 results. - rows = bigquery_client.list_rows(table, max_results=25) - - # Use format to create a simple table. - format_string = '{!s:<16} ' * len(table.schema) - - # Print schema field names - field_names = [field.name for field in table.schema] - print(format_string.format(*field_names)) - - for row in rows: - print(format_string.format(*row)) - - -def copy_table(dataset_id, table_id, new_table_id, project=None): - """Copies a table. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - # This sample shows the destination table in the same dataset and project, - # however, it's possible to copy across datasets and projects. You can - # also copy multiple source tables into a single destination table by - # providing addtional arguments to `copy_table`. - destination_table_ref = dataset_ref.table(new_table_id) - - # Create a job to copy the table to the destination table. - # Start by creating a job configuration - job_config = bigquery.CopyJobConfig() - - # Configure the job to create the table if it doesn't exist. - job_config.create_disposition = ( - bigquery.job.CreateDisposition.CREATE_IF_NEEDED) - - copy_job = bigquery_client.copy_table( - table_ref, destination_table_ref, job_config=job_config) - - print('Waiting for job to finish...') - copy_job.result() - - print('Table {} copied to {}.'.format(table_id, new_table_id)) - - -def delete_table(dataset_id, table_id, project=None): - """Deletes a table in a given dataset. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - bigquery_client.delete_table(table_ref) - - print('Table {}:{} deleted.'.format(dataset_id, table_id)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--project', default=None) - - subparsers = parser.add_subparsers(dest='command') - - list_projects_parser = subparsers.add_parser( - 'list-projects', help=list_projects.__doc__) - - list_datasets_parser = subparsers.add_parser( - 'list-datasets', help=list_datasets.__doc__) - - create_dataset_parser = subparsers.add_parser( - 'list-datasets', help=list_datasets.__doc__) - create_dataset_parser.add_argument('dataset_id') - - list_tables_parser = subparsers.add_parser( - 'list-tables', help=list_tables.__doc__) - list_tables_parser.add_argument('dataset_id') - - create_table_parser = subparsers.add_parser( - 'create-table', help=create_table.__doc__) - create_table_parser.add_argument('dataset_id') - create_table_parser.add_argument('table_id') - - list_rows_parser = subparsers.add_parser( - 'list-rows', help=list_rows.__doc__) - list_rows_parser.add_argument('dataset_id') - list_rows_parser.add_argument('table_id') - - copy_table_parser = subparsers.add_parser( - 'copy-table', help=copy_table.__doc__) - copy_table_parser.add_argument('dataset_id') - copy_table_parser.add_argument('table_id') - copy_table_parser.add_argument('new_table_id') - - delete_table_parser = subparsers.add_parser( - 'delete-table', help=delete_table.__doc__) - delete_table_parser.add_argument('dataset_id') - delete_table_parser.add_argument('table_id') - - args = parser.parse_args() - - if args.command == 'list-projects': - list_projects() - elif args.command == 'list-datasets': - list_datasets(args.project) - elif args.command == 'create-dataset': - create_dataset(args.dataset_id, args.project) - elif args.command == 'list-tables': - list_tables(args.dataset_id, args.project) - elif args.command == 'create-table': - create_table(args.dataset_id, args.table_id, args.project) - elif args.command == 'list-rows': - list_rows(args.dataset_id, args.table_id, args.project) - elif args.command == 'copy-table': - copy_table(args.dataset_id, args.table_id, args.new_table_id) - elif args.command == 'delete-table': - delete_table(args.dataset_id, args.table_id, args.project) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py deleted file mode 100644 index 5f666ccc6..000000000 --- a/samples/snippets/snippets_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from google.cloud import bigquery -from google.cloud.exceptions import NotFound -import pytest - -import snippets - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_table' - - -def test_list_projects(): - snippets.list_projects() - # No need to check the ouput, lack of exception is enough. - - -def test_list_datasets(capsys): - # Requires the dataset to have been created in the test project. - snippets.list_datasets() - - out, _ = capsys.readouterr() - - assert DATASET_ID in out - - -@pytest.fixture -def cleanup_dataset(): - dataset_id = 'test_temporary_dataset' - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - - if dataset_exists(dataset_ref, bigquery_client): - bigquery_client.delete_dataset(dataset_ref) - - yield dataset_id - - if dataset_exists(dataset_ref, bigquery_client): - bigquery_client.delete_dataset(dataset_ref) - - -def dataset_exists(dataset, client): - try: - client.get_dataset(dataset) - return True - except NotFound: - return False - - -def test_create_dataset(capsys, cleanup_dataset): - snippets.create_dataset(cleanup_dataset) - - out, _ = capsys.readouterr() - - assert cleanup_dataset in out - - -def test_list_tables(capsys): - # Requires the dataset and table to have been created in the test project. - snippets.list_tables(DATASET_ID) - - out, _ = capsys.readouterr() - - assert TABLE_ID in out - - -def test_list_rows(capsys): - # Requires the dataset and table to have been created in the test project. - - # Check for the schema. It's okay if the table is empty as long as there - # aren't any errors. - - snippets.list_rows(DATASET_ID, TABLE_ID) - - out, _ = capsys.readouterr() - - assert 'Name' in out - assert 'Age' in out - - -@pytest.fixture -def temporary_table(): - """Fixture that returns a factory for tables that do not yet exist and - will be automatically deleted after the test.""" - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(DATASET_ID) - tables = [] - - def factory(table_id): - new_table_ref = dataset_ref.table(table_id) - if table_exists(new_table_ref, bigquery_client): - bigquery_client.delete_table(new_table_ref) - tables.append(new_table_ref) - return new_table_ref - - yield factory - - for table in tables: - if table_exists(table, bigquery_client): - bigquery_client.delete_table(table) - - -def table_exists(table, client): - try: - client.get_table(table) - return True - except NotFound: - return False - - -def test_create_table(temporary_table): - bigquery_client = bigquery.Client() - new_table = temporary_table('test_create_table') - snippets.create_table(DATASET_ID, new_table.table_id) - assert table_exists(new_table, bigquery_client) - - -@pytest.mark.slow -def test_copy_table(temporary_table): - bigquery_client = bigquery.Client() - new_table = temporary_table('test_copy_table') - snippets.copy_table(DATASET_ID, TABLE_ID, new_table.table_id) - assert table_exists(new_table, bigquery_client) - - -def test_delete_table(): - # Create a table to delete - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(DATASET_ID) - table_ref = dataset_ref.table('test_delete_table') - table = bigquery.Table(table_ref) - - if not table_exists(table, bigquery_client): - table.schema = [bigquery.SchemaField('id', 'INTEGER')] - table = bigquery_client.create_table(table) - - snippets.delete_table(DATASET_ID, table.table_id) - - assert not table_exists(table, bigquery_client) diff --git a/samples/snippets/stream_data.py b/samples/snippets/stream_data.py deleted file mode 100644 index c54960048..000000000 --- a/samples/snippets/stream_data.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Loads a single row of data directly into BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python stream_data.py example_dataset example_table \\ - '["Gandalf", 2000]' - -The dataset and table should already exist. -""" - -import argparse -import json -from pprint import pprint - -from google.cloud import bigquery - - -def stream_data(dataset_id, table_id, json_data): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - data = json.loads(json_data) - - # Get the table from the API so that the schema is available. - table = bigquery_client.get_table(table_ref) - - rows = [data] - errors = bigquery_client.create_rows(table, rows) - - if not errors: - print('Loaded 1 row into {}:{}'.format(dataset_id, table_id)) - else: - print('Errors:') - pprint(errors) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'json_data', - help='The row to load into BigQuery as an array in JSON format.') - - args = parser.parse_args() - - stream_data( - args.dataset_id, - args.table_id, - args.json_data) diff --git a/samples/snippets/stream_data_test.py b/samples/snippets/stream_data_test.py deleted file mode 100644 index 25982b88a..000000000 --- a/samples/snippets/stream_data_test.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import stream_data - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -def test_stream_data(capsys): - stream_data.stream_data( - DATASET_ID, - TABLE_ID, - '["Gandalf", 2000]') - - out, _ = capsys.readouterr() - - assert 'Loaded 1 row' in out From 1bc5a166017fa793d146cf53e3db88a100b1af49 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 17 Apr 2018 17:47:05 -0700 Subject: [PATCH 067/100] BigQuery: Removes unused API samples [(#1459)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1459) * BigQuery: Removes unused API samples * removes unused sample data files --- samples/snippets/resources/data.csv | 1 - samples/snippets/resources/data.json | 1 - samples/snippets/resources/schema.json | 1 - 3 files changed, 3 deletions(-) delete mode 100644 samples/snippets/resources/data.csv delete mode 100644 samples/snippets/resources/data.json delete mode 100644 samples/snippets/resources/schema.json diff --git a/samples/snippets/resources/data.csv b/samples/snippets/resources/data.csv deleted file mode 100644 index affe39ef8..000000000 --- a/samples/snippets/resources/data.csv +++ /dev/null @@ -1 +0,0 @@ -Gandalf,2000,140.0,1 diff --git a/samples/snippets/resources/data.json b/samples/snippets/resources/data.json deleted file mode 100644 index b8eef90c5..000000000 --- a/samples/snippets/resources/data.json +++ /dev/null @@ -1 +0,0 @@ -{"Name": "Gandalf", "Age": 2000, "Weight": 140.0, "IsMagic": true} diff --git a/samples/snippets/resources/schema.json b/samples/snippets/resources/schema.json deleted file mode 100644 index a48971ef8..000000000 --- a/samples/snippets/resources/schema.json +++ /dev/null @@ -1 +0,0 @@ -[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}] \ No newline at end of file From d60ebc2888f52a8e96adebc311bc95edc71092cd Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 26 Apr 2018 10:26:41 -0700 Subject: [PATCH 068/100] Update READMEs to fix numbering and add git clone [(#1464)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1464) --- samples/snippets/README.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index cca0ff3e5..02bc856f9 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -31,10 +31,16 @@ credentials for applications. Install Dependencies ++++++++++++++++++++ +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + #. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup #. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. From 02b0a53c88dad797ffdb25f32d1260ecc1159405 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Fri, 18 May 2018 12:50:17 -0700 Subject: [PATCH 069/100] BigQuery: Moves BigQuery tutorial for Dataproc to python-docs-samples [(#1494)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1494) --- samples/snippets/natality_tutorial.py | 81 ++++++++++++++++++++++ samples/snippets/natality_tutorial_test.py | 42 +++++++++++ 2 files changed, 123 insertions(+) create mode 100644 samples/snippets/natality_tutorial.py create mode 100644 samples/snippets/natality_tutorial_test.py diff --git a/samples/snippets/natality_tutorial.py b/samples/snippets/natality_tutorial.py new file mode 100644 index 000000000..5bfa8f1d2 --- /dev/null +++ b/samples/snippets/natality_tutorial.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_natality_tutorial(): + # [START bigquery_query_natality_tutorial] + """Create a Google BigQuery linear regression input table. + + In the code below, the following actions are taken: + * A new dataset is created "natality_regression." + * A query is run against the public dataset, + bigquery-public-data.samples.natality, selecting only the data of + interest to the regression, the output of which is stored in a new + "regression_input" table. + * The output table is moved over the wire to the user's default project via + the built-in BigQuery Connector for Spark that bridges BigQuery and + Cloud Dataproc. + """ + + from google.cloud import bigquery + + # Create a new Google BigQuery client using Google Cloud Platform project + # defaults. + client = bigquery.Client() + + # Prepare a reference to a new dataset for storing the query results. + dataset_ref = client.dataset('natality_regression') + dataset = bigquery.Dataset(dataset_ref) + + # Create the new BigQuery dataset. + dataset = client.create_dataset(dataset) + + # In the new BigQuery dataset, create a reference to a new table for + # storing the query results. + table_ref = dataset.table('regression_input') + + # Configure the query job. + job_config = bigquery.QueryJobConfig() + + # Set the destination table to the table reference created above. + job_config.destination = table_ref + + # Set up a query in Standard SQL, which is the default for the BigQuery + # Python client library. + # The query selects the fields of interest. + query = """ + SELECT + weight_pounds, mother_age, father_age, gestation_weeks, + weight_gain_pounds, apgar_5min + FROM + `bigquery-public-data.samples.natality` + WHERE + weight_pounds IS NOT NULL + AND mother_age IS NOT NULL + AND father_age IS NOT NULL + AND gestation_weeks IS NOT NULL + AND weight_gain_pounds IS NOT NULL + AND apgar_5min IS NOT NULL + """ + + # Run the query. + query_job = client.query(query, job_config=job_config) + query_job.result() # Waits for the query to finish + # [END bigquery_query_natality_tutorial] + + +if __name__ == '__main__': + run_natality_tutorial() diff --git a/samples/snippets/natality_tutorial_test.py b/samples/snippets/natality_tutorial_test.py new file mode 100644 index 000000000..5165f7244 --- /dev/null +++ b/samples/snippets/natality_tutorial_test.py @@ -0,0 +1,42 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +from google.cloud import exceptions + +import natality_tutorial + + +def dataset_exists(dataset, client): + try: + client.get_dataset(dataset) + return True + except exceptions.NotFound: + return False + + +def test_natality_tutorial(): + client = bigquery.Client() + dataset_ref = client.dataset('natality_regression') + assert not dataset_exists(dataset_ref, client) + + natality_tutorial.run_natality_tutorial() + + assert dataset_exists(dataset_ref, client) + + table = client.get_table( + bigquery.Table(dataset_ref.table('regression_input'))) + assert table.num_rows > 0 + + client.delete_dataset(dataset_ref, delete_contents=True) From bdd51ba4964abddbe996af7bfd459bf7654e92fd Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Thu, 21 Jun 2018 12:42:25 -0700 Subject: [PATCH 070/100] BigQuery: Adds authorized view tutorial [(#1535)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1535) --- samples/snippets/authorized_view_tutorial.py | 108 ++++++++++++++++++ .../snippets/authorized_view_tutorial_test.py | 58 ++++++++++ samples/snippets/requirements.txt | 2 +- 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/authorized_view_tutorial.py create mode 100644 samples/snippets/authorized_view_tutorial_test.py diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py new file mode 100644 index 000000000..a6dff0612 --- /dev/null +++ b/samples/snippets/authorized_view_tutorial.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_authorized_view_tutorial(): + # Note to user: This is a group email for testing purposes. Replace with + # your own group email address when running this code. + analyst_group_email = 'example-analyst-group@google.com' + + # [START bigquery_authorized_view_tutorial] + # Create a source dataset + # [START bigquery_avt_create_source_dataset] + from google.cloud import bigquery + + client = bigquery.Client() + source_dataset_id = 'github_source_data' + + source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) + # Specify the geographic location where the dataset should reside. + source_dataset.location = 'US' + source_dataset = client.create_dataset(source_dataset) # API request + # [END bigquery_avt_create_source_dataset] + + # Populate a source table + # [START bigquery_avt_create_source_table] + source_table_id = 'github_contributors' + job_config = bigquery.QueryJobConfig() + job_config.destination = source_dataset.table(source_table_id) + sql = """ + SELECT commit, author, committer, repo_name + FROM `bigquery-public-data.github_repos.commits` + LIMIT 1000 + """ + query_job = client.query( + sql, + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location='US', + job_config=job_config) # API request - starts the query + + query_job.result() # Waits for the query to finish + # [END bigquery_avt_create_source_table] + + # Create a separate dataset to store your view + # [START bigquery_avt_create_shared_dataset] + shared_dataset_id = 'shared_views' + shared_dataset = bigquery.Dataset(client.dataset(shared_dataset_id)) + shared_dataset.location = 'US' + shared_dataset = client.create_dataset(shared_dataset) # API request + # [END bigquery_avt_create_shared_dataset] + + # Create the view in the new dataset + # [START bigquery_avt_create_view] + shared_view_id = 'github_analyst_view' + view = bigquery.Table(shared_dataset.table(shared_view_id)) + sql_template = """ + SELECT + commit, author.name as author, + committer.name as committer, repo_name + FROM + `{}.{}.{}` + """ + view.view_query = sql_template.format( + client.project, source_dataset_id, source_table_id) + view = client.create_table(view) # API request + # [END bigquery_avt_create_view] + + # Assign access controls to the dataset containing the view + # [START bigquery_avt_shared_dataset_access] + # analyst_group_email = 'data_analysts@example.com' + access_entries = shared_dataset.access_entries + access_entries.append( + bigquery.AccessEntry('READER', 'groupByEmail', analyst_group_email) + ) + shared_dataset.access_entries = access_entries + shared_dataset = client.update_dataset( + shared_dataset, ['access_entries']) # API request + # [END bigquery_avt_shared_dataset_access] + + # Authorize the view to access the source dataset + # [START bigquery_avt_source_dataset_access] + access_entries = source_dataset.access_entries + access_entries.append( + bigquery.AccessEntry(None, 'view', view.reference.to_api_repr()) + ) + source_dataset.access_entries = access_entries + source_dataset = client.update_dataset( + source_dataset, ['access_entries']) # API request + # [START bigquery_avt_source_dataset_access] + # [END bigquery_authorized_view_tutorial] + return (source_dataset, shared_dataset) + + +if __name__ == '__main__': + run_authorized_view_tutorial() diff --git a/samples/snippets/authorized_view_tutorial_test.py b/samples/snippets/authorized_view_tutorial_test.py new file mode 100644 index 000000000..fbc34771c --- /dev/null +++ b/samples/snippets/authorized_view_tutorial_test.py @@ -0,0 +1,58 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + +import authorized_view_tutorial + + +@pytest.fixture(scope='module') +def client(): + return bigquery.Client() + + +@pytest.fixture +def to_delete(client): + doomed = [] + yield doomed + for item in doomed: + if isinstance(item, (bigquery.Dataset, bigquery.DatasetReference)): + client.delete_dataset(item, delete_contents=True) + elif isinstance(item, (bigquery.Table, bigquery.TableReference)): + client.delete_table(item) + else: + item.delete() + + +def test_authorized_view_tutorial(client, to_delete): + source_dataset, shared_dataset = ( + authorized_view_tutorial.run_authorized_view_tutorial()) + to_delete.extend([source_dataset, shared_dataset]) + + analyst_email = 'example-analyst-group@google.com' + analyst_entries = [entry for entry in shared_dataset.access_entries + if entry.entity_id == analyst_email] + assert len(analyst_entries) == 1 + assert analyst_entries[0].role == 'READER' + + authorized_view_entries = [entry for entry in source_dataset.access_entries + if entry.entity_type == 'view'] + expected_view_ref = { + 'projectId': client.project, + 'datasetId': 'shared_views', + 'tableId': 'github_analyst_view', + } + assert len(authorized_view_entries) == 1 + assert authorized_view_entries[0].entity_id == expected_view_ref diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 62490b384..97152c191 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-cloud-bigquery==0.31.0 +google-cloud-bigquery==1.3.0 google-auth-oauthlib==0.2.0 pytz==2018.3 From 066abf48950f77b45785c0c78e7be830ccb62a5f Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Thu, 21 Jun 2018 12:53:52 -0700 Subject: [PATCH 071/100] Region tag end typo [(#1536)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1536) --- samples/snippets/authorized_view_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index a6dff0612..c1465174b 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -99,7 +99,7 @@ def run_authorized_view_tutorial(): source_dataset.access_entries = access_entries source_dataset = client.update_dataset( source_dataset, ['access_entries']) # API request - # [START bigquery_avt_source_dataset_access] + # [END bigquery_avt_source_dataset_access] # [END bigquery_authorized_view_tutorial] return (source_dataset, shared_dataset) From a93f1ccbc4a8e899b8428cf4c103f88205a830a3 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Thu, 12 Jul 2018 10:41:41 -0700 Subject: [PATCH 072/100] Code snippets for Jupyter data visualization tutorial [(#1560)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1560) --- samples/snippets/jupyter_tutorial_test.py | 165 ++++++++++++++++++++++ samples/snippets/requirements.txt | 5 +- 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/jupyter_tutorial_test.py diff --git a/samples/snippets/jupyter_tutorial_test.py b/samples/snippets/jupyter_tutorial_test.py new file mode 100644 index 000000000..a10f1a61b --- /dev/null +++ b/samples/snippets/jupyter_tutorial_test.py @@ -0,0 +1,165 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import IPython +from IPython.terminal import interactiveshell +from IPython.testing import tools +import matplotlib +import pytest + + +# Ignore semicolon lint warning because semicolons are used in notebooks +# flake8: noqa E703 + + +@pytest.fixture(scope='session') +def ipython(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture() +def ipython_interactive(request, ipython): + """Activate IPython's builtin hooks + + for the duration of the test scope. + """ + with ipython.builtin_trap: + yield ipython + + +def _strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [line for line in sample_text.split('\n') + if len(line) > 0 and '# [' not in line] + return '\n'.join(magic_lines) + + +def test_jupyter_tutorial(ipython): + matplotlib.use('agg') + ip = IPython.get_ipython() + ip.extension_manager.load_extension('google.cloud.bigquery') + + sample = """ + # [START bigquery_jupyter_magic_gender_by_year] + %%bigquery + SELECT + source_year AS year, + COUNT(is_male) AS birth_count + FROM `bigquery-public-data.samples.natality` + GROUP BY year + ORDER BY year DESC + LIMIT 15 + # [END bigquery_jupyter_magic_gender_by_year] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + sample = """ + # [START bigquery_jupyter_magic_gender_by_year_var] + %%bigquery total_births + SELECT + source_year AS year, + COUNT(is_male) AS birth_count + FROM `bigquery-public-data.samples.natality` + GROUP BY year + ORDER BY year DESC + LIMIT 15 + # [END bigquery_jupyter_magic_gender_by_year_var] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert 'total_births' in ip.user_ns # verify that variable exists + total_births = ip.user_ns['total_births'] + # [START bigquery_jupyter_plot_births_by_year] + total_births.plot(kind='bar', x='year', y='birth_count'); + # [END bigquery_jupyter_plot_births_by_year] + + sample = """ + # [START bigquery_jupyter_magic_gender_by_weekday] + %%bigquery births_by_weekday + SELECT + wday, + SUM(CASE WHEN is_male THEN 1 ELSE 0 END) AS male_births, + SUM(CASE WHEN is_male THEN 0 ELSE 1 END) AS female_births + FROM `bigquery-public-data.samples.natality` + WHERE wday IS NOT NULL + GROUP BY wday + ORDER BY wday ASC + # [END bigquery_jupyter_magic_gender_by_weekday] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert 'births_by_weekday' in ip.user_ns # verify that variable exists + births_by_weekday = ip.user_ns['births_by_weekday'] + # [START bigquery_jupyter_plot_births_by_weekday] + births_by_weekday.plot(x='wday'); + # [END bigquery_jupyter_plot_births_by_weekday] + + # [START bigquery_jupyter_import_and_client] + from google.cloud import bigquery + client = bigquery.Client() + # [END bigquery_jupyter_import_and_client] + + # [START bigquery_jupyter_query_plurality_by_year] + sql = """ + SELECT + plurality, + COUNT(1) AS count, + year + FROM + `bigquery-public-data.samples.natality` + WHERE + NOT IS_NAN(plurality) AND plurality > 1 + GROUP BY + plurality, year + ORDER BY + count DESC + """ + df = client.query(sql).to_dataframe() + df.head() + # [END bigquery_jupyter_query_plurality_by_year] + + # [START bigquery_jupyter_plot_plurality_by_year] + pivot_table = df.pivot(index='year', columns='plurality', values='count') + pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7)); + # [END bigquery_jupyter_plot_plurality_by_year] + + # [START bigquery_jupyter_query_weight_by_gestation] + sql = """ + SELECT + gestation_weeks, + AVG(weight_pounds) AS ave_weight + FROM + `bigquery-public-data.samples.natality` + WHERE + NOT IS_NAN(gestation_weeks) AND gestation_weeks <> 99 + GROUP BY + gestation_weeks + ORDER BY + gestation_weeks + """ + df = client.query(sql).to_dataframe() + # [END bigquery_jupyter_query_weight_by_gestation] + + # [START bigquery_jupyter_plot_weight_by_gestation] + ax = df.plot( + kind='bar', x='gestation_weeks', y='ave_weight', figsize=(15, 7)) + ax.set_title('Average Weight by Gestation Weeks') + ax.set_xlabel('Gestation Weeks') + ax.set_ylabel('Average Weight'); + # [END bigquery_jupyter_plot_weight_by_gestation] diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 97152c191..7b65d6ac8 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,6 @@ -google-cloud-bigquery==1.3.0 +google-cloud-bigquery[pandas]==1.3.0 google-auth-oauthlib==0.2.0 +ipython==5.5; python_version < "3" +ipython; python_version > "3" +matplotlib pytz==2018.3 From adde57a747a8851e068eb723c0de4cd734782f67 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 17 Jul 2018 15:00:57 -0700 Subject: [PATCH 073/100] updates last query per Steve's feedback [(#1578)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1578) --- samples/snippets/jupyter_tutorial_test.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/samples/snippets/jupyter_tutorial_test.py b/samples/snippets/jupyter_tutorial_test.py index a10f1a61b..0affbabcb 100644 --- a/samples/snippets/jupyter_tutorial_test.py +++ b/samples/snippets/jupyter_tutorial_test.py @@ -139,11 +139,11 @@ def test_jupyter_tutorial(ipython): pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7)); # [END bigquery_jupyter_plot_plurality_by_year] - # [START bigquery_jupyter_query_weight_by_gestation] + # [START bigquery_jupyter_query_births_by_gestation] sql = """ SELECT gestation_weeks, - AVG(weight_pounds) AS ave_weight + COUNT(1) AS count FROM `bigquery-public-data.samples.natality` WHERE @@ -154,12 +154,11 @@ def test_jupyter_tutorial(ipython): gestation_weeks """ df = client.query(sql).to_dataframe() - # [END bigquery_jupyter_query_weight_by_gestation] + # [END bigquery_jupyter_query_births_by_gestation] - # [START bigquery_jupyter_plot_weight_by_gestation] - ax = df.plot( - kind='bar', x='gestation_weeks', y='ave_weight', figsize=(15, 7)) - ax.set_title('Average Weight by Gestation Weeks') + # [START bigquery_jupyter_plot_births_by_gestation] + ax = df.plot(kind='bar', x='gestation_weeks', y='count', figsize=(15,7)) + ax.set_title('Count of Births by Gestation Weeks') ax.set_xlabel('Gestation Weeks') - ax.set_ylabel('Average Weight'); - # [END bigquery_jupyter_plot_weight_by_gestation] + ax.set_ylabel('Count'); + # [END bigquery_jupyter_plot_births_by_gestation] From b48dd8710554d8c8a6f30436cac9acf645074246 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 17 Jul 2018 16:22:27 -0700 Subject: [PATCH 074/100] updates tutorial to add datasets to to_delete before running [(#1579)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1579) --- samples/snippets/authorized_view_tutorial.py | 1 - samples/snippets/authorized_view_tutorial_test.py | 10 +++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index c1465174b..abe4e0cb1 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -101,7 +101,6 @@ def run_authorized_view_tutorial(): source_dataset, ['access_entries']) # API request # [END bigquery_avt_source_dataset_access] # [END bigquery_authorized_view_tutorial] - return (source_dataset, shared_dataset) if __name__ == '__main__': diff --git a/samples/snippets/authorized_view_tutorial_test.py b/samples/snippets/authorized_view_tutorial_test.py index fbc34771c..954c47072 100644 --- a/samples/snippets/authorized_view_tutorial_test.py +++ b/samples/snippets/authorized_view_tutorial_test.py @@ -37,10 +37,14 @@ def to_delete(client): def test_authorized_view_tutorial(client, to_delete): - source_dataset, shared_dataset = ( - authorized_view_tutorial.run_authorized_view_tutorial()) - to_delete.extend([source_dataset, shared_dataset]) + source_dataset_ref = client.dataset('github_source_data') + shared_dataset_ref = client.dataset('shared_views') + to_delete.extend([source_dataset_ref, shared_dataset_ref]) + authorized_view_tutorial.run_authorized_view_tutorial() + + source_dataset = client.get_dataset(source_dataset_ref) + shared_dataset = client.get_dataset(shared_dataset_ref) analyst_email = 'example-analyst-group@google.com' analyst_entries = [entry for entry in shared_dataset.access_entries if entry.entity_id == analyst_email] From 27babfefb0eec117416fd8a415884cd184d5e73a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 31 Aug 2018 11:01:23 -0700 Subject: [PATCH 075/100] Remove ipython Python 2 modifier from requirements.txt [(#1675)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1675) If ipython has tagged their packages correctly, then the modifier is not necessary. Sending a PR to check. Bug 113341391. --- samples/snippets/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 7b65d6ac8..e90d0443c 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,6 +1,5 @@ google-cloud-bigquery[pandas]==1.3.0 google-auth-oauthlib==0.2.0 -ipython==5.5; python_version < "3" -ipython; python_version > "3" +ipython matplotlib pytz==2018.3 From c8de756bb308988b18f94e8e5ad4706d58c7e769 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 20 Nov 2018 15:40:29 -0800 Subject: [PATCH 076/100] Auto-update dependencies. [(#1846)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1846) ACK, merging. --- samples/snippets/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index e90d0443c..4f8e5641f 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery[pandas]==1.3.0 +google-cloud-bigquery[pandas]==1.7.0 google-auth-oauthlib==0.2.0 -ipython -matplotlib -pytz==2018.3 +ipython==7.1.1 +matplotlib==3.0.2 +pytz==2018.7 From d22c0e7f77f23fdd0e4c9d6940b2505896cf8bb1 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 2 Jan 2019 11:58:00 -0800 Subject: [PATCH 077/100] Refactor BQ user credentials sample to use region tags [(#1952)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1952) Also add comments for the needed values and use a concrete value where possible according to the sample rubric. --- samples/snippets/user_credentials.py | 50 +++++++++++++++-------- samples/snippets/user_credentials_test.py | 7 ++-- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index 9ab062fb3..300d9112d 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -23,20 +23,15 @@ import argparse -def run_query(credentials, project, query): - from google.cloud import bigquery - - client = bigquery.Client(project=project, credentials=credentials) - query_job = client.query(query) - - # Print the results. - for row in query_job.result(): # Wait for the job to complete. - print(row) - - -def authenticate_and_query(project, query, launch_browser=True): +def main(project, launch_browser=True): + # [START bigquery_auth_user_flow] from google_auth_oauthlib import flow + # TODO: Use a local server in the auth flow? This is recommended but does + # not work if accessing the application remotely, such as over SSH or + # from a remote Jupyter notebook. + # launch_browser = True + appflow = flow.InstalledAppFlow.from_client_secrets_file( 'client_secrets.json', scopes=['https://www.googleapis.com/auth/bigquery']) @@ -46,7 +41,31 @@ def authenticate_and_query(project, query, launch_browser=True): else: appflow.run_console() - run_query(appflow.credentials, project, query) + credentials = appflow.credentials + # [END bigquery_auth_user_flow] + + # [START bigquery_auth_user_query] + from google.cloud import bigquery + + # TODO: This project will be billed for the query processing. The user must + # have the bigquery.jobs.create permission on this project to run a + # query. See: + # https://cloud.google.com/bigquery/docs/access-control#permissions + # project = 'user-project-id' + + client = bigquery.Client(project=project, credentials=credentials) + + query_string = """SELECT name, SUM(number) as total + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE name = 'William' + GROUP BY name; + """ + query_job = client.query(query_string) + + # Print the results. + for row in query_job.result(): # Wait for the job to complete. + print("{}: {}".format(row['name'], row['total'])) + # [END bigquery_auth_user_query] if __name__ == '__main__': @@ -58,9 +77,8 @@ def authenticate_and_query(project, query, launch_browser=True): help='Use a local server flow to authenticate. ', action='store_true') parser.add_argument('project', help='Project to use for BigQuery billing.') - parser.add_argument('query', help='BigQuery SQL Query.') args = parser.parse_args() - authenticate_and_query( - args.project, args.query, launch_browser=args.launch_browser) + main( + args.project, launch_browser=args.launch_browser) diff --git a/samples/snippets/user_credentials_test.py b/samples/snippets/user_credentials_test.py index 02acc19c3..009b9be7f 100644 --- a/samples/snippets/user_credentials_test.py +++ b/samples/snippets/user_credentials_test.py @@ -18,7 +18,7 @@ import mock import pytest -from user_credentials import authenticate_and_query +from user_credentials import main PROJECT = os.environ['GCLOUD_PROJECT'] @@ -36,6 +36,7 @@ def mock_flow(): def test_auth_query_console(mock_flow, capsys): - authenticate_and_query(PROJECT, 'SELECT 1+1;', launch_browser=False) + main(PROJECT, launch_browser=False) out, _ = capsys.readouterr() - assert '2' in out + # Fun fact: William P. Wood was the 1st director of the US Secret Service. + assert 'William' in out From 2105e2506ec387ed8e44a9162734ddb1f01e14d2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 8 Jan 2019 13:42:50 -0800 Subject: [PATCH 078/100] Make TODO clearer what action to take. [(#1963)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1963) * Make TODO clearer what action to take. Describe the variables to set explicitly (rather than "this"), as suggested in internal CL 227863277. * Use standard 'uncomment the variable below' for TODO * Move variable below TODO. --- samples/snippets/user_credentials.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index 300d9112d..14ca76e29 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -27,10 +27,13 @@ def main(project, launch_browser=True): # [START bigquery_auth_user_flow] from google_auth_oauthlib import flow - # TODO: Use a local server in the auth flow? This is recommended but does - # not work if accessing the application remotely, such as over SSH or - # from a remote Jupyter notebook. + # TODO: Uncomment the line below to set the `launch_browser` variable. # launch_browser = True + # + # The `launch_browser` boolean variable indicates if a local server in the + # auth flow. A value of `True` is recommended, but a local server does not + # work if accessing the application remotely, such as over SSH or from a + # remote Jupyter notebook. appflow = flow.InstalledAppFlow.from_client_secrets_file( 'client_secrets.json', @@ -47,11 +50,13 @@ def main(project, launch_browser=True): # [START bigquery_auth_user_query] from google.cloud import bigquery - # TODO: This project will be billed for the query processing. The user must - # have the bigquery.jobs.create permission on this project to run a - # query. See: - # https://cloud.google.com/bigquery/docs/access-control#permissions + # TODO: Uncomment the line below to set the `project` variable. # project = 'user-project-id' + # + # The `project` variable defines the project to be billed for query + # processing. The user must have the bigquery.jobs.create permission on + # this project to run a query. See: + # https://cloud.google.com/bigquery/docs/access-control#permissions client = bigquery.Client(project=project, credentials=credentials) From 25335f99c807b3b706e2273f2d96c6377aaa0701 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 9 Jan 2019 10:18:27 -0800 Subject: [PATCH 079/100] Add missing explanation about local_server callback URL Per feedback on CL 227863277 --- samples/snippets/user_credentials.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index 14ca76e29..4917fdd3a 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -30,10 +30,10 @@ def main(project, launch_browser=True): # TODO: Uncomment the line below to set the `launch_browser` variable. # launch_browser = True # - # The `launch_browser` boolean variable indicates if a local server in the - # auth flow. A value of `True` is recommended, but a local server does not - # work if accessing the application remotely, such as over SSH or from a - # remote Jupyter notebook. + # The `launch_browser` boolean variable indicates if a local server is used + # as the callback URL in the auth flow. A value of `True` is recommended, + # but a local server does not work if accessing the application remotely, + # such as over SSH or from a remote Jupyter notebook. appflow = flow.InstalledAppFlow.from_client_secrets_file( 'client_secrets.json', From d4f004d459e1c56c028da963d89bf8be730d8fa5 Mon Sep 17 00:00:00 2001 From: DPEBot Date: Wed, 6 Feb 2019 12:06:35 -0800 Subject: [PATCH 080/100] Auto-update dependencies. [(#1980)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1980) * Auto-update dependencies. * Update requirements.txt * Update requirements.txt --- samples/snippets/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 4f8e5641f..bfdc27e7d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery[pandas]==1.7.0 +google-cloud-bigquery[pandas]==1.9.0 google-auth-oauthlib==0.2.0 -ipython==7.1.1 +ipython==7.2.0 matplotlib==3.0.2 -pytz==2018.7 +pytz==2018.9 From 1998fe43506134bb4a84d02fd4faff5e36f229ff Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 5 Aug 2019 15:17:40 -0700 Subject: [PATCH 081/100] =?UTF-8?q?BigQuery:=20Add=20sample=20for=20explic?= =?UTF-8?q?itly=20creating=20client=20from=20service=20acco=E2=80=A6=20[(#?= =?UTF-8?q?2312)](https://github.com/GoogleCloudPlatform/python-docs-sampl?= =?UTF-8?q?es/issues/2312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * BigQuery: Add sample for explicitly creating client from service account credentials. * Flake8 * Move imports to improve indentation. * Combine auth and client functions. * flake8 --- .../snippets/authenticate_service_account.py | 43 +++++++++++++++++++ .../authenticate_service_account_test.py | 33 ++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 samples/snippets/authenticate_service_account.py create mode 100644 samples/snippets/authenticate_service_account_test.py diff --git a/samples/snippets/authenticate_service_account.py b/samples/snippets/authenticate_service_account.py new file mode 100644 index 000000000..03cb06fd1 --- /dev/null +++ b/samples/snippets/authenticate_service_account.py @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def main(): + key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + + # [START bigquery_client_json_credentials] + from google.cloud import bigquery + from google.oauth2 import service_account + + # TODO(developer): Set key_path to the path to the service account key + # file. + # key_path = "path/to/service_account.json" + + credentials = service_account.Credentials.from_service_account_file( + key_path, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + + client = bigquery.Client( + credentials=credentials, + project=credentials.project_id, + ) + # [END bigquery_client_json_credentials] + return client + + +if __name__ == "__main__": + main() diff --git a/samples/snippets/authenticate_service_account_test.py b/samples/snippets/authenticate_service_account_test.py new file mode 100644 index 000000000..8c436ec20 --- /dev/null +++ b/samples/snippets/authenticate_service_account_test.py @@ -0,0 +1,33 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth + +import authenticate_service_account + + +def mock_credentials(*args, **kwargs): + credentials, _ = google.auth.default( + ["https://www.googleapis.com/auth/cloud-platform"] + ) + return credentials + + +def test_main(monkeypatch): + monkeypatch.setattr( + 'google.oauth2.service_account.Credentials.from_service_account_file', + mock_credentials, + ) + client = authenticate_service_account.main() + assert client is not None From 6ecfa781152c90123e07ec0de09ebacc67dd6541 Mon Sep 17 00:00:00 2001 From: Gus Class Date: Wed, 23 Oct 2019 16:27:00 -0700 Subject: [PATCH 082/100] Adds updates including compute [(#2436)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2436) * Adds updates including compute * Python 2 compat pytest * Fixing weird \r\n issue from GH merge * Put asset tests back in * Re-add pod operator test * Hack parameter for k8s pod operator --- samples/snippets/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index bfdc27e7d..70d644472 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery[pandas]==1.9.0 -google-auth-oauthlib==0.2.0 -ipython==7.2.0 -matplotlib==3.0.2 -pytz==2018.9 +google-cloud-bigquery[pandas]==1.20.0 +google-auth-oauthlib==0.4.0 +ipython==7.8.0 +matplotlib==3.1.1 +pytz==2019.2 From b695f036edcc75b9504da70c3201778c8b36c49b Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Wed, 1 Apr 2020 19:11:50 -0700 Subject: [PATCH 083/100] Simplify noxfile setup. [(#2806)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2806) * chore(deps): update dependency requests to v2.23.0 * Simplify noxfile and add version control. * Configure appengine/standard to only test Python 2.7. * Update Kokokro configs to match noxfile. * Add requirements-test to each folder. * Remove Py2 versions from everything execept appengine/standard. * Remove conftest.py. * Remove appengine/standard/conftest.py * Remove 'no-sucess-flaky-report' from pytest.ini. * Add GAE SDK back to appengine/standard tests. * Fix typo. * Roll pytest to python 2 version. * Add a bunch of testing requirements. * Remove typo. * Add appengine lib directory back in. * Add some additional requirements. * Fix issue with flake8 args. * Even more requirements. * Readd appengine conftest.py. * Add a few more requirements. * Even more Appengine requirements. * Add webtest for appengine/standard/mailgun. * Add some additional requirements. * Add workaround for issue with mailjet-rest. * Add responses for appengine/standard/mailjet. Co-authored-by: Renovate Bot --- samples/snippets/requirements-test.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 samples/snippets/requirements-test.txt diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt new file mode 100644 index 000000000..41c4d5110 --- /dev/null +++ b/samples/snippets/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==5.3.2 +mock==3.0.5 From fb9da7dd0d08fc5f25719dea77186b45a1e23e92 Mon Sep 17 00:00:00 2001 From: shollyman Date: Fri, 17 Apr 2020 16:53:37 -0700 Subject: [PATCH 084/100] fix: address issues with concurrent BigQuery tests [(#3426)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3426) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/snippets/authorized_view_tutorial.py | 19 ++++++++- .../snippets/authorized_view_tutorial_test.py | 28 +++++++------ samples/snippets/natality_tutorial.py | 12 ++++-- samples/snippets/natality_tutorial_test.py | 38 ++++++++++-------- samples/snippets/quickstart.py | 8 +++- samples/snippets/quickstart_test.py | 40 +++++++++---------- 6 files changed, 88 insertions(+), 57 deletions(-) diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index abe4e0cb1..edf83cf7f 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -15,7 +15,7 @@ # limitations under the License. -def run_authorized_view_tutorial(): +def run_authorized_view_tutorial(override_values={}): # Note to user: This is a group email for testing purposes. Replace with # your own group email address when running this code. analyst_group_email = 'example-analyst-group@google.com' @@ -28,6 +28,14 @@ def run_authorized_view_tutorial(): client = bigquery.Client() source_dataset_id = 'github_source_data' + # [END bigquery_authorized_view_tutorial] + # [END bigquery_avt_create_source_dataset] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + source_dataset_id = override_values.get("source_dataset_id", source_dataset_id) + # [START bigquery_authorized_view_tutorial] + # [START bigquery_avt_create_source_dataset] + source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) # Specify the geographic location where the dataset should reside. source_dataset.location = 'US' @@ -57,6 +65,15 @@ def run_authorized_view_tutorial(): # Create a separate dataset to store your view # [START bigquery_avt_create_shared_dataset] shared_dataset_id = 'shared_views' + + # [END bigquery_authorized_view_tutorial] + # [END bigquery_avt_create_shared_dataset] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + shared_dataset_id = override_values.get("shared_dataset_id", shared_dataset_id) + # [START bigquery_authorized_view_tutorial] + # [START bigquery_avt_create_shared_dataset] + shared_dataset = bigquery.Dataset(client.dataset(shared_dataset_id)) shared_dataset.location = 'US' shared_dataset = client.create_dataset(shared_dataset) # API request diff --git a/samples/snippets/authorized_view_tutorial_test.py b/samples/snippets/authorized_view_tutorial_test.py index 954c47072..03079085f 100644 --- a/samples/snippets/authorized_view_tutorial_test.py +++ b/samples/snippets/authorized_view_tutorial_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import uuid + from google.cloud import bigquery import pytest @@ -24,24 +26,24 @@ def client(): @pytest.fixture -def to_delete(client): +def datasets_to_delete(client): doomed = [] yield doomed for item in doomed: - if isinstance(item, (bigquery.Dataset, bigquery.DatasetReference)): - client.delete_dataset(item, delete_contents=True) - elif isinstance(item, (bigquery.Table, bigquery.TableReference)): - client.delete_table(item) - else: - item.delete() + client.delete_dataset(item, delete_contents=True) -def test_authorized_view_tutorial(client, to_delete): - source_dataset_ref = client.dataset('github_source_data') - shared_dataset_ref = client.dataset('shared_views') - to_delete.extend([source_dataset_ref, shared_dataset_ref]) +def test_authorized_view_tutorial(client, datasets_to_delete): + override_values = { + "source_dataset_id": "github_source_data_{}".format(str(uuid.uuid4()).replace("-", "_")), + "shared_dataset_id": "shared_views_{}".format(str(uuid.uuid4()).replace("-", "_")), + } + source_dataset_ref = client.dataset(override_values["source_dataset_id"]) + shared_dataset_ref = client.dataset(override_values["shared_dataset_id"]) + datasets_to_delete.extend([override_values["source_dataset_id"], + override_values["shared_dataset_id"]]) - authorized_view_tutorial.run_authorized_view_tutorial() + authorized_view_tutorial.run_authorized_view_tutorial(override_values) source_dataset = client.get_dataset(source_dataset_ref) shared_dataset = client.get_dataset(shared_dataset_ref) @@ -55,7 +57,7 @@ def test_authorized_view_tutorial(client, to_delete): if entry.entity_type == 'view'] expected_view_ref = { 'projectId': client.project, - 'datasetId': 'shared_views', + 'datasetId': override_values["shared_dataset_id"], 'tableId': 'github_analyst_view', } assert len(authorized_view_entries) == 1 diff --git a/samples/snippets/natality_tutorial.py b/samples/snippets/natality_tutorial.py index 5bfa8f1d2..6a097add3 100644 --- a/samples/snippets/natality_tutorial.py +++ b/samples/snippets/natality_tutorial.py @@ -15,7 +15,7 @@ # limitations under the License. -def run_natality_tutorial(): +def run_natality_tutorial(override_values={}): # [START bigquery_query_natality_tutorial] """Create a Google BigQuery linear regression input table. @@ -37,8 +37,14 @@ def run_natality_tutorial(): client = bigquery.Client() # Prepare a reference to a new dataset for storing the query results. - dataset_ref = client.dataset('natality_regression') - dataset = bigquery.Dataset(dataset_ref) + dataset_id = 'natality_regression' + # [END bigquery_query_natality_tutorial] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + dataset_id = override_values.get("dataset_id", dataset_id) + # [START bigquery_query_natality_tutorial] + + dataset = bigquery.Dataset(client.dataset(dataset_id)) # Create the new BigQuery dataset. dataset = client.create_dataset(dataset) diff --git a/samples/snippets/natality_tutorial_test.py b/samples/snippets/natality_tutorial_test.py index 5165f7244..785df59df 100644 --- a/samples/snippets/natality_tutorial_test.py +++ b/samples/snippets/natality_tutorial_test.py @@ -12,31 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +import uuid + from google.cloud import bigquery -from google.cloud import exceptions +import pytest import natality_tutorial -def dataset_exists(dataset, client): - try: - client.get_dataset(dataset) - return True - except exceptions.NotFound: - return False +@pytest.fixture(scope='module') +def client(): + return bigquery.Client() -def test_natality_tutorial(): - client = bigquery.Client() - dataset_ref = client.dataset('natality_regression') - assert not dataset_exists(dataset_ref, client) +@pytest.fixture +def datasets_to_delete(client): + doomed = [] + yield doomed + for item in doomed: + client.delete_dataset(item, delete_contents=True) - natality_tutorial.run_natality_tutorial() - assert dataset_exists(dataset_ref, client) +def test_natality_tutorial(client, datasets_to_delete): + override_values = { + "dataset_id": "natality_regression_{}".format(str(uuid.uuid4()).replace("-", "_")), + } + datasets_to_delete.append(override_values["dataset_id"]) - table = client.get_table( - bigquery.Table(dataset_ref.table('regression_input'))) - assert table.num_rows > 0 + natality_tutorial.run_natality_tutorial(override_values) - client.delete_dataset(dataset_ref, delete_contents=True) + table_ref = bigquery.Dataset(client.dataset(override_values["dataset_id"])).table("regression_input") + table = client.get_table(table_ref) + assert table.num_rows > 0 diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 10ae58e84..cb6dcc303 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -15,7 +15,7 @@ # limitations under the License. -def run_quickstart(): +def run_quickstart(override_values={}): # [START bigquery_quickstart] # Imports the Google Cloud client library from google.cloud import bigquery @@ -26,6 +26,12 @@ def run_quickstart(): # The name for the new dataset dataset_id = 'my_new_dataset' + # [END bigquery_quickstart] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + dataset_id = override_values.get("dataset_id", dataset_id) + # [START bigquery_quickstart] + # Prepares a reference to the new dataset dataset_ref = bigquery_client.dataset(dataset_id) dataset = bigquery.Dataset(dataset_ref) diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index 02931086a..2b461a8f2 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import uuid + from google.cloud import bigquery -from google.cloud.exceptions import NotFound import pytest import quickstart @@ -24,31 +25,26 @@ DATASET_ID = 'my_new_dataset' -@pytest.fixture -def temporary_dataset(): - """Fixture that ensures the test dataset does not exist before or - after a test.""" - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(DATASET_ID) - - if dataset_exists(dataset_ref, bigquery_client): - bigquery_client.delete_dataset(dataset_ref) +@pytest.fixture(scope='module') +def client(): + return bigquery.Client() - yield - if dataset_exists(dataset_ref, bigquery_client): - bigquery_client.delete_dataset(dataset_ref) +@pytest.fixture +def datasets_to_delete(client): + doomed = [] + yield doomed + for item in doomed: + client.delete_dataset(item, delete_contents=True) -def dataset_exists(dataset, client): - try: - client.get_dataset(dataset) - return True - except NotFound: - return False +def test_quickstart(capsys, client, datasets_to_delete): + override_values = { + "dataset_id": "my_new_dataset_{}".format(str(uuid.uuid4()).replace("-", "_")), + } + datasets_to_delete.append(override_values["dataset_id"]) -def test_quickstart(capsys, temporary_dataset): - quickstart.run_quickstart() + quickstart.run_quickstart(override_values) out, _ = capsys.readouterr() - assert DATASET_ID in out + assert override_values["dataset_id"] in out From 797349b96d3f0d0b400e9ae9e1258722dbf18420 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:00:37 +0200 Subject: [PATCH 085/100] chore(deps): update dependency pytz to v2019.3 [(#3190)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3190) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 70d644472..9b40cd64a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,4 +2,4 @@ google-cloud-bigquery[pandas]==1.20.0 google-auth-oauthlib==0.4.0 ipython==7.8.0 matplotlib==3.1.1 -pytz==2019.2 +pytz==2019.3 From 544f861dfce6b5ed31a70de5f7f5086050593449 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:09:05 +0200 Subject: [PATCH 086/100] chore(deps): update dependency matplotlib to v3.2.1 [(#3180)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3180) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/snippets/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9b40cd64a..4cbdfb2de 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,6 @@ google-cloud-bigquery[pandas]==1.20.0 google-auth-oauthlib==0.4.0 ipython==7.8.0 -matplotlib==3.1.1 +matplotlib==3.2.1 pytz==2019.3 + From 3e4c835952b25964a1e49ef60109d7b257d796e5 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:14:29 +0200 Subject: [PATCH 087/100] chore(deps): update dependency ipython to v7.13.0 [(#3177)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3177) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/snippets/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 4cbdfb2de..ba7b7bda7 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,6 +1,5 @@ google-cloud-bigquery[pandas]==1.20.0 google-auth-oauthlib==0.4.0 -ipython==7.8.0 +ipython==7.13.0 matplotlib==3.2.1 pytz==2019.3 - From 9deac728198502eebc7854abaacdd1f5aa025049 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:41:12 +0200 Subject: [PATCH 088/100] chore(deps): update dependency google-cloud-bigquery to v1.24.0 [(#3049)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3049) * chore(deps): update dependency google-cloud-bigquery to v1.24.0 * chore(deps): update ipython version * fix: fix requirements order * explicitly add grpc to resolve errors * adjust arguments * undo mistake * bump auth version Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Leah Cole Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> Co-authored-by: Christopher Wilcox --- samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index ba7b7bda7..9b737b7a0 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery[pandas]==1.20.0 -google-auth-oauthlib==0.4.0 +google-cloud-bigquery[pandas]==1.24.0 +google-auth-oauthlib==0.4.1 ipython==7.13.0 matplotlib==3.2.1 pytz==2019.3 From 93f384186d190cd0759b34915a293ba00922cf1a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:47:33 +0200 Subject: [PATCH 089/100] chore(deps): update dependency google-auth-oauthlib to v0.4.1 [(#2797)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2797) * chore(deps): update dependency google-auth-oauthlib to v0.4.1 * resolve dependency finding errors * fix new matplotlib error Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> Co-authored-by: Leah Cole Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Christopher Wilcox From 700ffb9652c6e2f5247548d26b43404cdc2d8ffa Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 28 Apr 2020 15:05:51 +0200 Subject: [PATCH 090/100] chore(deps): update dependency pytz to v2020 [(#3566)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3566) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9b737b7a0..650c6f4ab 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,4 +2,4 @@ google-cloud-bigquery[pandas]==1.24.0 google-auth-oauthlib==0.4.1 ipython==7.13.0 matplotlib==3.2.1 -pytz==2019.3 +pytz==2020.1 From 69af2218a00d33e649ecf5abb3946f67ea1663ed Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 5 May 2020 01:21:57 +0200 Subject: [PATCH 091/100] chore(deps): update dependency ipython to v7.14.0 [(#3670)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3670) Co-authored-by: Takashi Matsuo --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 650c6f4ab..a1eabc954 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery[pandas]==1.24.0 google-auth-oauthlib==0.4.1 -ipython==7.13.0 +ipython==7.14.0 matplotlib==3.2.1 pytz==2020.1 From c3c4a0b041341f6ca64f022ce31651c3be994609 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Tue, 26 May 2020 14:42:53 -0700 Subject: [PATCH 092/100] [bigquery] opt in to use the build specific projects [(#3883)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3883) fixes #3882 --- samples/snippets/README.rst | 8 +++++-- samples/snippets/README.rst.in | 4 +++- samples/snippets/noxfile_config.py | 37 ++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 samples/snippets/noxfile_config.py diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 02bc856f9..9446125cd 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -14,6 +14,11 @@ This directory contains samples for Google BigQuery. `Google BigQuery`_ is Googl .. _Google BigQuery: https://cloud.google.com/bigquery/docs + +To run the sample, you need to have `BigQuery Admin` role. + + + Setup ------------------------------------------------------------------------------- @@ -108,7 +113,7 @@ To run this sample: $ python user_credentials.py - usage: user_credentials.py [-h] [--launch-browser] project query + usage: user_credentials.py [-h] [--launch-browser] project Command-line application to run a query using user credentials. @@ -117,7 +122,6 @@ To run this sample: positional arguments: project Project to use for BigQuery billing. - query BigQuery SQL Query. optional arguments: -h, --help show this help message and exit diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in index 008b51795..74b7fa940 100644 --- a/samples/snippets/README.rst.in +++ b/samples/snippets/README.rst.in @@ -11,6 +11,8 @@ product: analyzing data to find meaningful insights, use familiar SQL, and take advantage of our pay-as-you-go model. +required_role: BigQuery Admin + setup: - auth - install_deps @@ -26,4 +28,4 @@ samples: cloud_client_library: true -folder: bigquery/cloud-client \ No newline at end of file +folder: bigquery/cloud-client diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py new file mode 100644 index 000000000..950c3a070 --- /dev/null +++ b/samples/snippets/noxfile_config.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be inported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + # 'gcloud_project_env': 'GCLOUD_PROJECT', + 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} From 4d06dfcb14309873bc69adbbd245cc181d7aef71 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 30 May 2020 10:59:14 +0200 Subject: [PATCH 093/100] chore(deps): update dependency ipython to v7.15.0 [(#3917)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3917) Co-authored-by: Takashi Matsuo --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index a1eabc954..acad65571 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery[pandas]==1.24.0 google-auth-oauthlib==0.4.1 -ipython==7.14.0 +ipython==7.15.0 matplotlib==3.2.1 pytz==2020.1 From 863cb329863167335c1bb12323ef8a43cdb8573f Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Tue, 9 Jun 2020 14:34:27 -0700 Subject: [PATCH 094/100] Replace GCLOUD_PROJECT with GOOGLE_CLOUD_PROJECT. [(#4022)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4022) --- samples/snippets/noxfile_config.py | 2 +- samples/snippets/user_credentials_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py index 950c3a070..cfd0d4391 100644 --- a/samples/snippets/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -28,7 +28,7 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - # 'gcloud_project_env': 'GCLOUD_PROJECT', + # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any diff --git a/samples/snippets/user_credentials_test.py b/samples/snippets/user_credentials_test.py index 009b9be7f..494903c75 100644 --- a/samples/snippets/user_credentials_test.py +++ b/samples/snippets/user_credentials_test.py @@ -21,7 +21,7 @@ from user_credentials import main -PROJECT = os.environ['GCLOUD_PROJECT'] +PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] @pytest.fixture From 95091a4c643a1303ea84dc9bfaf7d880fefa05da Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 11 Jun 2020 06:36:11 +0200 Subject: [PATCH 095/100] Update dependency google-cloud-bigquery to v1.25.0 [(#4024)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4024) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-cloud-bigquery](https://togithub.com/googleapis/python-bigquery) | minor | `==1.24.0` -> `==1.25.0` | --- ### Release Notes
googleapis/python-bigquery ### [`v1.25.0`](https://togithub.com/googleapis/python-bigquery/blob/master/CHANGELOG.md#​1250-httpswwwgithubcomgoogleapispython-bigquerycomparev1240v1250-2020-06-06) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v1.24.0...v1.25.0) ##### Features - add BigQuery storage client support to DB API ([#​36](https://www.github.com/googleapis/python-bigquery/issues/36)) ([ba9b2f8](https://www.github.com/googleapis/python-bigquery/commit/ba9b2f87e36320d80f6f6460b77e6daddb0fa214)) - **bigquery:** add create job method ([#​32](https://www.github.com/googleapis/python-bigquery/issues/32)) ([2abdef8](https://www.github.com/googleapis/python-bigquery/commit/2abdef82bed31601d1ca1aa92a10fea1e09f5297)) - **bigquery:** add support of model for extract job ([#​71](https://www.github.com/googleapis/python-bigquery/issues/71)) ([4a7a514](https://www.github.com/googleapis/python-bigquery/commit/4a7a514659a9f6f9bbd8af46bab3f8782d6b4b98)) - add HOUR support for time partitioning interval ([#​91](https://www.github.com/googleapis/python-bigquery/issues/91)) ([0dd90b9](https://www.github.com/googleapis/python-bigquery/commit/0dd90b90e3714c1d18f8a404917a9454870e338a)) - add support for policy tags ([#​77](https://www.github.com/googleapis/python-bigquery/issues/77)) ([38a5c01](https://www.github.com/googleapis/python-bigquery/commit/38a5c01ca830daf165592357c45f2fb4016aad23)) - make AccessEntry objects hashable ([#​93](https://www.github.com/googleapis/python-bigquery/issues/93)) ([23a173b](https://www.github.com/googleapis/python-bigquery/commit/23a173bc5a25c0c8200adc5af62eb05624c9099e)) - **bigquery:** expose start index parameter for query result ([#​121](https://www.github.com/googleapis/python-bigquery/issues/121)) ([be86de3](https://www.github.com/googleapis/python-bigquery/commit/be86de330a3c3801653a0ccef90e3d9bdb3eee7a)) - **bigquery:** unit and system test for dataframe with int column with Nan values ([#​39](https://www.github.com/googleapis/python-bigquery/issues/39)) ([5fd840e](https://www.github.com/googleapis/python-bigquery/commit/5fd840e9d4c592c4f736f2fd4792c9670ba6795e)) ##### Bug Fixes - allow partial streaming_buffer statistics ([#​37](https://www.github.com/googleapis/python-bigquery/issues/37)) ([645f0fd](https://www.github.com/googleapis/python-bigquery/commit/645f0fdb35ee0e81ee70f7459e796a42a1f03210)) - distinguish server timeouts from transport timeouts ([#​43](https://www.github.com/googleapis/python-bigquery/issues/43)) ([a17be5f](https://www.github.com/googleapis/python-bigquery/commit/a17be5f01043f32d9fbfb2ddf456031ea9205c8f)) - improve cell magic error message on missing query ([#​58](https://www.github.com/googleapis/python-bigquery/issues/58)) ([6182cf4](https://www.github.com/googleapis/python-bigquery/commit/6182cf48aef8f463bb96891cfc44a96768121dbc)) - **bigquery:** fix repr of model reference ([#​66](https://www.github.com/googleapis/python-bigquery/issues/66)) ([26c6204](https://www.github.com/googleapis/python-bigquery/commit/26c62046f4ec8880cf6561cc90a8b821dcc84ec5)) - **bigquery:** fix start index with page size for list rows ([#​27](https://www.github.com/googleapis/python-bigquery/issues/27)) ([400673b](https://www.github.com/googleapis/python-bigquery/commit/400673b5d0f2a6a3d828fdaad9d222ca967ffeff))
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index acad65571..b5ec18a01 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery[pandas]==1.24.0 +google-cloud-bigquery[pandas]==1.25.0 google-auth-oauthlib==0.4.1 ipython==7.15.0 matplotlib==3.2.1 From 1da73530a387205ae892b81b35077c94f89eb7f9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 19 Jun 2020 19:04:04 +0200 Subject: [PATCH 096/100] chore(deps): update dependency matplotlib to v3.2.2 [(#4118)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4118) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [matplotlib](https://matplotlib.org) ([source](https://togithub.com/matplotlib/matplotlib)) | patch | `==3.2.1` -> `==3.2.2` | --- ### Release Notes
matplotlib/matplotlib ### [`v3.2.2`](https://togithub.com/matplotlib/matplotlib/releases/v3.2.2) [Compare Source](https://togithub.com/matplotlib/matplotlib/compare/v3.2.1...v3.2.2) This is the second bugfix release of the 3.2.x series. This release contains several critical bug-fixes: - support fractional HiDPI scaling with Qt backends - support new Python and fix syntax errors in legacy Python - support new Qt 5 and fix support for Qt 4 - fix animation writer fallback - fix figure resizing - fix handling of large arcs - fix issues with tight layout - fix saving figures after closing windows or under certain size conditions - fix scatter when specifying a single color - fix several memory leaks - fix unexpected autoscaling behavior - fix various issues with usetex - various minor bug and documentation fixes
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index b5ec18a01..93f36e038 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery[pandas]==1.25.0 google-auth-oauthlib==0.4.1 ipython==7.15.0 -matplotlib==3.2.1 +matplotlib==3.2.2 pytz==2020.1 From 353c993ee1d4844cb2c48e24d7fb70dae771515c Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 26 Jun 2020 11:52:22 +0200 Subject: [PATCH 097/100] chore: update templates --- .github/CODEOWNERS | 8 + .kokoro/samples/lint/common.cfg | 34 +++ .kokoro/samples/lint/continuous.cfg | 6 + .kokoro/samples/lint/periodic.cfg | 6 + .kokoro/samples/lint/presubmit.cfg | 6 + .kokoro/samples/python3.6/common.cfg | 34 +++ .kokoro/samples/python3.6/continuous.cfg | 7 + .kokoro/samples/python3.6/periodic.cfg | 6 + .kokoro/samples/python3.6/presubmit.cfg | 6 + .kokoro/samples/python3.7/common.cfg | 34 +++ .kokoro/samples/python3.7/continuous.cfg | 6 + .kokoro/samples/python3.7/periodic.cfg | 6 + .kokoro/samples/python3.7/presubmit.cfg | 6 + .kokoro/samples/python3.8/common.cfg | 34 +++ .kokoro/samples/python3.8/continuous.cfg | 6 + .kokoro/samples/python3.8/periodic.cfg | 6 + .kokoro/samples/python3.8/presubmit.cfg | 6 + .kokoro/test-samples.sh | 104 ++++++++ samples/AUTHORING_GUIDE.md | 1 + samples/CONTRIBUTING.md | 1 + samples/snippets/README.rst | 29 ++- .../snippets/authenticate_service_account.py | 8 +- .../authenticate_service_account_test.py | 2 +- samples/snippets/authorized_view_tutorial.py | 34 +-- .../snippets/authorized_view_tutorial_test.py | 37 +-- samples/snippets/jupyter_tutorial_test.py | 38 +-- samples/snippets/natality_tutorial.py | 6 +- samples/snippets/natality_tutorial_test.py | 10 +- samples/snippets/noxfile.py | 222 ++++++++++++++++++ samples/snippets/noxfile_config.py | 8 +- samples/snippets/quickstart.py | 6 +- samples/snippets/quickstart_test.py | 4 +- samples/snippets/simple_app.py | 9 +- samples/snippets/simple_app_test.py | 2 +- samples/snippets/user_credentials.py | 24 +- samples/snippets/user_credentials_test.py | 7 +- scripts/decrypt-secrets.sh | 33 +++ scripts/readme-gen/readme_gen.py | 66 ++++++ scripts/readme-gen/templates/README.tmpl.rst | 87 +++++++ scripts/readme-gen/templates/auth.tmpl.rst | 9 + .../templates/auth_api_key.tmpl.rst | 14 ++ .../templates/install_deps.tmpl.rst | 29 +++ .../templates/install_portaudio.tmpl.rst | 35 +++ synth.py | 9 +- testing/.gitignore | 3 + 45 files changed, 957 insertions(+), 97 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .kokoro/samples/lint/common.cfg create mode 100644 .kokoro/samples/lint/continuous.cfg create mode 100644 .kokoro/samples/lint/periodic.cfg create mode 100644 .kokoro/samples/lint/presubmit.cfg create mode 100644 .kokoro/samples/python3.6/common.cfg create mode 100644 .kokoro/samples/python3.6/continuous.cfg create mode 100644 .kokoro/samples/python3.6/periodic.cfg create mode 100644 .kokoro/samples/python3.6/presubmit.cfg create mode 100644 .kokoro/samples/python3.7/common.cfg create mode 100644 .kokoro/samples/python3.7/continuous.cfg create mode 100644 .kokoro/samples/python3.7/periodic.cfg create mode 100644 .kokoro/samples/python3.7/presubmit.cfg create mode 100644 .kokoro/samples/python3.8/common.cfg create mode 100644 .kokoro/samples/python3.8/continuous.cfg create mode 100644 .kokoro/samples/python3.8/periodic.cfg create mode 100644 .kokoro/samples/python3.8/presubmit.cfg create mode 100755 .kokoro/test-samples.sh create mode 100644 samples/AUTHORING_GUIDE.md create mode 100644 samples/CONTRIBUTING.md create mode 100644 samples/snippets/noxfile.py create mode 100755 scripts/decrypt-secrets.sh create mode 100644 scripts/readme-gen/readme_gen.py create mode 100644 scripts/readme-gen/templates/README.tmpl.rst create mode 100644 scripts/readme-gen/templates/auth.tmpl.rst create mode 100644 scripts/readme-gen/templates/auth_api_key.tmpl.rst create mode 100644 scripts/readme-gen/templates/install_deps.tmpl.rst create mode 100644 scripts/readme-gen/templates/install_portaudio.tmpl.rst create mode 100644 testing/.gitignore diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..d90c5db8c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +/samples/ @shollyman @googleapis/python-samples-owners diff --git a/.kokoro/samples/lint/common.cfg b/.kokoro/samples/lint/common.cfg new file mode 100644 index 000000000..3e41df313 --- /dev/null +++ b/.kokoro/samples/lint/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "lint" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/lint/continuous.cfg b/.kokoro/samples/lint/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/lint/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/lint/periodic.cfg b/.kokoro/samples/lint/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/.kokoro/samples/lint/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/lint/presubmit.cfg b/.kokoro/samples/lint/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/lint/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg new file mode 100644 index 000000000..a56768eae --- /dev/null +++ b/.kokoro/samples/python3.6/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.6" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg new file mode 100644 index 000000000..7218af149 --- /dev/null +++ b/.kokoro/samples/python3.6/continuous.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.6/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/common.cfg b/.kokoro/samples/python3.7/common.cfg new file mode 100644 index 000000000..c93747180 --- /dev/null +++ b/.kokoro/samples/python3.7/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.7" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.7/continuous.cfg b/.kokoro/samples/python3.7/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.7/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/periodic.cfg b/.kokoro/samples/python3.7/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/.kokoro/samples/python3.7/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/presubmit.cfg b/.kokoro/samples/python3.7/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.7/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/common.cfg b/.kokoro/samples/python3.8/common.cfg new file mode 100644 index 000000000..9808f15e3 --- /dev/null +++ b/.kokoro/samples/python3.8/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.8" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.8/continuous.cfg b/.kokoro/samples/python3.8/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.8/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/periodic.cfg b/.kokoro/samples/python3.8/periodic.cfg new file mode 100644 index 000000000..50fec9649 --- /dev/null +++ b/.kokoro/samples/python3.8/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/presubmit.cfg b/.kokoro/samples/python3.8/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.8/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh new file mode 100755 index 000000000..905732a40 --- /dev/null +++ b/.kokoro/test-samples.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-bigquery + +# Run periodic samples tests at latest release +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + LATEST_RELEASE=$(git describe --abbrev=0 --tags) + git checkout $LATEST_RELEASE +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the Build Cop Bot. + # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop + $KOKORO_GFILE_DIR/linux_amd64/buildcop + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" \ No newline at end of file diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md new file mode 100644 index 000000000..55c97b32f --- /dev/null +++ b/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md new file mode 100644 index 000000000..34c882b6f --- /dev/null +++ b/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 9446125cd..7c3e19e68 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -1,3 +1,4 @@ + .. This file is automatically generated. Do not edit this file directly. Google BigQuery Python Samples @@ -15,14 +16,11 @@ This directory contains samples for Google BigQuery. `Google BigQuery`_ is Googl .. _Google BigQuery: https://cloud.google.com/bigquery/docs -To run the sample, you need to have `BigQuery Admin` role. - - - Setup ------------------------------------------------------------------------------- + Authentication ++++++++++++++ @@ -33,6 +31,9 @@ credentials for applications. .. _Authentication Getting Started Guide: https://cloud.google.com/docs/authentication/getting-started + + + Install Dependencies ++++++++++++++++++++ @@ -47,7 +48,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash @@ -63,9 +64,15 @@ Install Dependencies .. _pip: https://pip.pypa.io/ .. _virtualenv: https://virtualenv.pypa.io/ + + + + + Samples ------------------------------------------------------------------------------- + Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -82,6 +89,8 @@ To run this sample: $ python quickstart.py + + Simple Application +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -98,6 +107,8 @@ To run this sample: $ python simple_app.py + + User Credentials +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -113,6 +124,7 @@ To run this sample: $ python user_credentials.py + usage: user_credentials.py [-h] [--launch-browser] project Command-line application to run a query using user credentials. @@ -131,6 +143,10 @@ To run this sample: + + + + The client library ------------------------------------------------------------------------------- @@ -146,4 +162,5 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/samples/snippets/authenticate_service_account.py b/samples/snippets/authenticate_service_account.py index 03cb06fd1..58cd2b542 100644 --- a/samples/snippets/authenticate_service_account.py +++ b/samples/snippets/authenticate_service_account.py @@ -27,14 +27,10 @@ def main(): # key_path = "path/to/service_account.json" credentials = service_account.Credentials.from_service_account_file( - key_path, - scopes=["https://www.googleapis.com/auth/cloud-platform"], + key_path, scopes=["https://www.googleapis.com/auth/cloud-platform"], ) - client = bigquery.Client( - credentials=credentials, - project=credentials.project_id, - ) + client = bigquery.Client(credentials=credentials, project=credentials.project_id,) # [END bigquery_client_json_credentials] return client diff --git a/samples/snippets/authenticate_service_account_test.py b/samples/snippets/authenticate_service_account_test.py index 8c436ec20..131c69d2c 100644 --- a/samples/snippets/authenticate_service_account_test.py +++ b/samples/snippets/authenticate_service_account_test.py @@ -26,7 +26,7 @@ def mock_credentials(*args, **kwargs): def test_main(monkeypatch): monkeypatch.setattr( - 'google.oauth2.service_account.Credentials.from_service_account_file', + "google.oauth2.service_account.Credentials.from_service_account_file", mock_credentials, ) client = authenticate_service_account.main() diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index edf83cf7f..6b5cc378f 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -18,7 +18,7 @@ def run_authorized_view_tutorial(override_values={}): # Note to user: This is a group email for testing purposes. Replace with # your own group email address when running this code. - analyst_group_email = 'example-analyst-group@google.com' + analyst_group_email = "example-analyst-group@google.com" # [START bigquery_authorized_view_tutorial] # Create a source dataset @@ -26,7 +26,7 @@ def run_authorized_view_tutorial(override_values={}): from google.cloud import bigquery client = bigquery.Client() - source_dataset_id = 'github_source_data' + source_dataset_id = "github_source_data" # [END bigquery_authorized_view_tutorial] # [END bigquery_avt_create_source_dataset] @@ -38,13 +38,13 @@ def run_authorized_view_tutorial(override_values={}): source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) # Specify the geographic location where the dataset should reside. - source_dataset.location = 'US' + source_dataset.location = "US" source_dataset = client.create_dataset(source_dataset) # API request # [END bigquery_avt_create_source_dataset] # Populate a source table # [START bigquery_avt_create_source_table] - source_table_id = 'github_contributors' + source_table_id = "github_contributors" job_config = bigquery.QueryJobConfig() job_config.destination = source_dataset.table(source_table_id) sql = """ @@ -56,15 +56,16 @@ def run_authorized_view_tutorial(override_values={}): sql, # Location must match that of the dataset(s) referenced in the query # and of the destination table. - location='US', - job_config=job_config) # API request - starts the query + location="US", + job_config=job_config, + ) # API request - starts the query query_job.result() # Waits for the query to finish # [END bigquery_avt_create_source_table] # Create a separate dataset to store your view # [START bigquery_avt_create_shared_dataset] - shared_dataset_id = 'shared_views' + shared_dataset_id = "shared_views" # [END bigquery_authorized_view_tutorial] # [END bigquery_avt_create_shared_dataset] @@ -75,13 +76,13 @@ def run_authorized_view_tutorial(override_values={}): # [START bigquery_avt_create_shared_dataset] shared_dataset = bigquery.Dataset(client.dataset(shared_dataset_id)) - shared_dataset.location = 'US' + shared_dataset.location = "US" shared_dataset = client.create_dataset(shared_dataset) # API request # [END bigquery_avt_create_shared_dataset] # Create the view in the new dataset # [START bigquery_avt_create_view] - shared_view_id = 'github_analyst_view' + shared_view_id = "github_analyst_view" view = bigquery.Table(shared_dataset.table(shared_view_id)) sql_template = """ SELECT @@ -91,7 +92,8 @@ def run_authorized_view_tutorial(override_values={}): `{}.{}.{}` """ view.view_query = sql_template.format( - client.project, source_dataset_id, source_table_id) + client.project, source_dataset_id, source_table_id + ) view = client.create_table(view) # API request # [END bigquery_avt_create_view] @@ -100,25 +102,27 @@ def run_authorized_view_tutorial(override_values={}): # analyst_group_email = 'data_analysts@example.com' access_entries = shared_dataset.access_entries access_entries.append( - bigquery.AccessEntry('READER', 'groupByEmail', analyst_group_email) + bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) ) shared_dataset.access_entries = access_entries shared_dataset = client.update_dataset( - shared_dataset, ['access_entries']) # API request + shared_dataset, ["access_entries"] + ) # API request # [END bigquery_avt_shared_dataset_access] # Authorize the view to access the source dataset # [START bigquery_avt_source_dataset_access] access_entries = source_dataset.access_entries access_entries.append( - bigquery.AccessEntry(None, 'view', view.reference.to_api_repr()) + bigquery.AccessEntry(None, "view", view.reference.to_api_repr()) ) source_dataset.access_entries = access_entries source_dataset = client.update_dataset( - source_dataset, ['access_entries']) # API request + source_dataset, ["access_entries"] + ) # API request # [END bigquery_avt_source_dataset_access] # [END bigquery_authorized_view_tutorial] -if __name__ == '__main__': +if __name__ == "__main__": run_authorized_view_tutorial() diff --git a/samples/snippets/authorized_view_tutorial_test.py b/samples/snippets/authorized_view_tutorial_test.py index 03079085f..4c74020bd 100644 --- a/samples/snippets/authorized_view_tutorial_test.py +++ b/samples/snippets/authorized_view_tutorial_test.py @@ -20,7 +20,7 @@ import authorized_view_tutorial -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def client(): return bigquery.Client() @@ -35,30 +35,39 @@ def datasets_to_delete(client): def test_authorized_view_tutorial(client, datasets_to_delete): override_values = { - "source_dataset_id": "github_source_data_{}".format(str(uuid.uuid4()).replace("-", "_")), - "shared_dataset_id": "shared_views_{}".format(str(uuid.uuid4()).replace("-", "_")), + "source_dataset_id": "github_source_data_{}".format( + str(uuid.uuid4()).replace("-", "_") + ), + "shared_dataset_id": "shared_views_{}".format( + str(uuid.uuid4()).replace("-", "_") + ), } source_dataset_ref = client.dataset(override_values["source_dataset_id"]) shared_dataset_ref = client.dataset(override_values["shared_dataset_id"]) - datasets_to_delete.extend([override_values["source_dataset_id"], - override_values["shared_dataset_id"]]) + datasets_to_delete.extend( + [override_values["source_dataset_id"], override_values["shared_dataset_id"]] + ) authorized_view_tutorial.run_authorized_view_tutorial(override_values) source_dataset = client.get_dataset(source_dataset_ref) shared_dataset = client.get_dataset(shared_dataset_ref) - analyst_email = 'example-analyst-group@google.com' - analyst_entries = [entry for entry in shared_dataset.access_entries - if entry.entity_id == analyst_email] + analyst_email = "example-analyst-group@google.com" + analyst_entries = [ + entry + for entry in shared_dataset.access_entries + if entry.entity_id == analyst_email + ] assert len(analyst_entries) == 1 - assert analyst_entries[0].role == 'READER' + assert analyst_entries[0].role == "READER" - authorized_view_entries = [entry for entry in source_dataset.access_entries - if entry.entity_type == 'view'] + authorized_view_entries = [ + entry for entry in source_dataset.access_entries if entry.entity_type == "view" + ] expected_view_ref = { - 'projectId': client.project, - 'datasetId': override_values["shared_dataset_id"], - 'tableId': 'github_analyst_view', + "projectId": client.project, + "datasetId": override_values["shared_dataset_id"], + "tableId": "github_analyst_view", } assert len(authorized_view_entries) == 1 assert authorized_view_entries[0].entity_id == expected_view_ref diff --git a/samples/snippets/jupyter_tutorial_test.py b/samples/snippets/jupyter_tutorial_test.py index 0affbabcb..353590b82 100644 --- a/samples/snippets/jupyter_tutorial_test.py +++ b/samples/snippets/jupyter_tutorial_test.py @@ -22,7 +22,7 @@ # flake8: noqa E703 -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def ipython(): config = tools.default_config() config.TerminalInteractiveShell.simple_prompt = True @@ -42,15 +42,16 @@ def ipython_interactive(request, ipython): def _strip_region_tags(sample_text): """Remove blank lines and region tags from sample text""" - magic_lines = [line for line in sample_text.split('\n') - if len(line) > 0 and '# [' not in line] - return '\n'.join(magic_lines) + magic_lines = [ + line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line + ] + return "\n".join(magic_lines) def test_jupyter_tutorial(ipython): - matplotlib.use('agg') + matplotlib.use("agg") ip = IPython.get_ipython() - ip.extension_manager.load_extension('google.cloud.bigquery') + ip.extension_manager.load_extension("google.cloud.bigquery") sample = """ # [START bigquery_jupyter_magic_gender_by_year] @@ -82,10 +83,10 @@ def test_jupyter_tutorial(ipython): result = ip.run_cell(_strip_region_tags(sample)) result.raise_error() # Throws an exception if the cell failed. - assert 'total_births' in ip.user_ns # verify that variable exists - total_births = ip.user_ns['total_births'] + assert "total_births" in ip.user_ns # verify that variable exists + total_births = ip.user_ns["total_births"] # [START bigquery_jupyter_plot_births_by_year] - total_births.plot(kind='bar', x='year', y='birth_count'); + total_births.plot(kind="bar", x="year", y="birth_count") # [END bigquery_jupyter_plot_births_by_year] sample = """ @@ -104,14 +105,15 @@ def test_jupyter_tutorial(ipython): result = ip.run_cell(_strip_region_tags(sample)) result.raise_error() # Throws an exception if the cell failed. - assert 'births_by_weekday' in ip.user_ns # verify that variable exists - births_by_weekday = ip.user_ns['births_by_weekday'] + assert "births_by_weekday" in ip.user_ns # verify that variable exists + births_by_weekday = ip.user_ns["births_by_weekday"] # [START bigquery_jupyter_plot_births_by_weekday] - births_by_weekday.plot(x='wday'); + births_by_weekday.plot(x="wday") # [END bigquery_jupyter_plot_births_by_weekday] # [START bigquery_jupyter_import_and_client] from google.cloud import bigquery + client = bigquery.Client() # [END bigquery_jupyter_import_and_client] @@ -135,8 +137,8 @@ def test_jupyter_tutorial(ipython): # [END bigquery_jupyter_query_plurality_by_year] # [START bigquery_jupyter_plot_plurality_by_year] - pivot_table = df.pivot(index='year', columns='plurality', values='count') - pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7)); + pivot_table = df.pivot(index="year", columns="plurality", values="count") + pivot_table.plot(kind="bar", stacked=True, figsize=(15, 7)) # [END bigquery_jupyter_plot_plurality_by_year] # [START bigquery_jupyter_query_births_by_gestation] @@ -157,8 +159,8 @@ def test_jupyter_tutorial(ipython): # [END bigquery_jupyter_query_births_by_gestation] # [START bigquery_jupyter_plot_births_by_gestation] - ax = df.plot(kind='bar', x='gestation_weeks', y='count', figsize=(15,7)) - ax.set_title('Count of Births by Gestation Weeks') - ax.set_xlabel('Gestation Weeks') - ax.set_ylabel('Count'); + ax = df.plot(kind="bar", x="gestation_weeks", y="count", figsize=(15, 7)) + ax.set_title("Count of Births by Gestation Weeks") + ax.set_xlabel("Gestation Weeks") + ax.set_ylabel("Count") # [END bigquery_jupyter_plot_births_by_gestation] diff --git a/samples/snippets/natality_tutorial.py b/samples/snippets/natality_tutorial.py index 6a097add3..b2b607b0d 100644 --- a/samples/snippets/natality_tutorial.py +++ b/samples/snippets/natality_tutorial.py @@ -37,7 +37,7 @@ def run_natality_tutorial(override_values={}): client = bigquery.Client() # Prepare a reference to a new dataset for storing the query results. - dataset_id = 'natality_regression' + dataset_id = "natality_regression" # [END bigquery_query_natality_tutorial] # To facilitate testing, we replace values with alternatives # provided by the testing harness. @@ -51,7 +51,7 @@ def run_natality_tutorial(override_values={}): # In the new BigQuery dataset, create a reference to a new table for # storing the query results. - table_ref = dataset.table('regression_input') + table_ref = dataset.table("regression_input") # Configure the query job. job_config = bigquery.QueryJobConfig() @@ -83,5 +83,5 @@ def run_natality_tutorial(override_values={}): # [END bigquery_query_natality_tutorial] -if __name__ == '__main__': +if __name__ == "__main__": run_natality_tutorial() diff --git a/samples/snippets/natality_tutorial_test.py b/samples/snippets/natality_tutorial_test.py index 785df59df..fae72fa46 100644 --- a/samples/snippets/natality_tutorial_test.py +++ b/samples/snippets/natality_tutorial_test.py @@ -20,7 +20,7 @@ import natality_tutorial -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def client(): return bigquery.Client() @@ -35,12 +35,16 @@ def datasets_to_delete(client): def test_natality_tutorial(client, datasets_to_delete): override_values = { - "dataset_id": "natality_regression_{}".format(str(uuid.uuid4()).replace("-", "_")), + "dataset_id": "natality_regression_{}".format( + str(uuid.uuid4()).replace("-", "_") + ), } datasets_to_delete.append(override_values["dataset_id"]) natality_tutorial.run_natality_tutorial(override_values) - table_ref = bigquery.Dataset(client.dataset(override_values["dataset_id"])).table("regression_input") + table_ref = bigquery.Dataset(client.dataset(override_values["dataset_id"])).table( + "regression_input" + ) table = client.get_table(table_ref) assert table.num_rows > 0 diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py new file mode 100644 index 000000000..5660f08be --- /dev/null +++ b/samples/snippets/noxfile.py @@ -0,0 +1,222 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py index cfd0d4391..6ecfa40e0 100644 --- a/samples/snippets/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -22,16 +22,14 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - + "ignored_versions": ["2.7"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - + "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, + "envs": {}, } diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index cb6dcc303..56d6fd843 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -24,7 +24,7 @@ def run_quickstart(override_values={}): bigquery_client = bigquery.Client() # The name for the new dataset - dataset_id = 'my_new_dataset' + dataset_id = "my_new_dataset" # [END bigquery_quickstart] # To facilitate testing, we replace values with alternatives @@ -39,9 +39,9 @@ def run_quickstart(override_values={}): # Creates the new dataset dataset = bigquery_client.create_dataset(dataset) - print('Dataset {} created.'.format(dataset.dataset_id)) + print("Dataset {} created.".format(dataset.dataset_id)) # [END bigquery_quickstart] -if __name__ == '__main__': +if __name__ == "__main__": run_quickstart() diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index 2b461a8f2..a5e3a13e3 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -22,10 +22,10 @@ # Must match the dataset listed in quickstart.py (there's no easy way to # extract this). -DATASET_ID = 'my_new_dataset' +DATASET_ID = "my_new_dataset" -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def client(): return bigquery.Client() diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index a09e97f12..c21ae86f4 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -18,6 +18,7 @@ # [START bigquery_simple_app_all] # [START bigquery_simple_app_deps] from google.cloud import bigquery + # [END bigquery_simple_app_deps] @@ -26,7 +27,8 @@ def query_stackoverflow(): client = bigquery.Client() # [END bigquery_simple_app_client] # [START bigquery_simple_app_query] - query_job = client.query(""" + query_job = client.query( + """ SELECT CONCAT( 'https://stackoverflow.com/questions/', @@ -35,7 +37,8 @@ def query_stackoverflow(): FROM `bigquery-public-data.stackoverflow.posts_questions` WHERE tags like '%google-bigquery%' ORDER BY view_count DESC - LIMIT 10""") + LIMIT 10""" + ) results = query_job.result() # Waits for job to complete. # [END bigquery_simple_app_query] @@ -46,6 +49,6 @@ def query_stackoverflow(): # [END bigquery_simple_app_print] -if __name__ == '__main__': +if __name__ == "__main__": query_stackoverflow() # [END bigquery_simple_app_all] diff --git a/samples/snippets/simple_app_test.py b/samples/snippets/simple_app_test.py index 33f9f1adf..5c608e1fd 100644 --- a/samples/snippets/simple_app_test.py +++ b/samples/snippets/simple_app_test.py @@ -18,4 +18,4 @@ def test_query_stackoverflow(capsys): simple_app.query_stackoverflow() out, _ = capsys.readouterr() - assert 'views' in out + assert "views" in out diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index 4917fdd3a..6089d9fd9 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -36,8 +36,8 @@ def main(project, launch_browser=True): # such as over SSH or from a remote Jupyter notebook. appflow = flow.InstalledAppFlow.from_client_secrets_file( - 'client_secrets.json', - scopes=['https://www.googleapis.com/auth/bigquery']) + "client_secrets.json", scopes=["https://www.googleapis.com/auth/bigquery"] + ) if launch_browser: appflow.run_local_server() @@ -69,21 +69,21 @@ def main(project, launch_browser=True): # Print the results. for row in query_job.result(): # Wait for the job to complete. - print("{}: {}".format(row['name'], row['total'])) + print("{}: {}".format(row["name"], row["total"])) # [END bigquery_auth_user_query] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.add_argument( - '--launch-browser', - help='Use a local server flow to authenticate. ', - action='store_true') - parser.add_argument('project', help='Project to use for BigQuery billing.') + "--launch-browser", + help="Use a local server flow to authenticate. ", + action="store_true", + ) + parser.add_argument("project", help="Project to use for BigQuery billing.") args = parser.parse_args() - main( - args.project, launch_browser=args.launch_browser) + main(args.project, launch_browser=args.launch_browser) diff --git a/samples/snippets/user_credentials_test.py b/samples/snippets/user_credentials_test.py index 494903c75..829502d25 100644 --- a/samples/snippets/user_credentials_test.py +++ b/samples/snippets/user_credentials_test.py @@ -21,13 +21,12 @@ from user_credentials import main -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] @pytest.fixture def mock_flow(): - flow_patch = mock.patch( - 'google_auth_oauthlib.flow.InstalledAppFlow', autospec=True) + flow_patch = mock.patch("google_auth_oauthlib.flow.InstalledAppFlow", autospec=True) with flow_patch as flow_mock: flow_mock.from_client_secrets_file.return_value = flow_mock @@ -39,4 +38,4 @@ def test_auth_query_console(mock_flow, capsys): main(PROJECT, launch_browser=False) out, _ = capsys.readouterr() # Fun fact: William P. Wood was the 1st director of the US Secret Service. - assert 'William' in out + assert "William" in out diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh new file mode 100755 index 000000000..ff599eb2a --- /dev/null +++ b/scripts/decrypt-secrets.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$( dirname "$DIR" ) + +# Work from the project root. +cd $ROOT + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions access latest --secret="python-docs-samples-test-env" \ + > testing/test-env.sh +gcloud secrets versions access latest \ + --secret="python-docs-samples-service-account" \ + > testing/service-account.json +gcloud secrets versions access latest \ + --secret="python-docs-samples-client-secrets" \ + > testing/client-secrets.json \ No newline at end of file diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py new file mode 100644 index 000000000..d309d6e97 --- /dev/null +++ b/scripts/readme-gen/readme_gen.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates READMEs using configuration defined in yaml.""" + +import argparse +import io +import os +import subprocess + +import jinja2 +import yaml + + +jinja_env = jinja2.Environment( + trim_blocks=True, + loader=jinja2.FileSystemLoader( + os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + +README_TMPL = jinja_env.get_template('README.tmpl.rst') + + +def get_help(file): + return subprocess.check_output(['python', file, '--help']).decode() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('source') + parser.add_argument('--destination', default='README.rst') + + args = parser.parse_args() + + source = os.path.abspath(args.source) + root = os.path.dirname(source) + destination = os.path.join(root, args.destination) + + jinja_env.globals['get_help'] = get_help + + with io.open(source, 'r') as f: + config = yaml.load(f) + + # This allows get_help to execute in the right directory. + os.chdir(root) + + output = README_TMPL.render(config) + + with io.open(destination, 'w') as f: + f.write(output) + + +if __name__ == '__main__': + main() diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst new file mode 100644 index 000000000..4fd239765 --- /dev/null +++ b/scripts/readme-gen/templates/README.tmpl.rst @@ -0,0 +1,87 @@ +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + + +This directory contains samples for {{product.name}}. {{product.description}} + +{{description}} + +.. _{{product.name}}: {{product.url}} + +{% if required_api_url %} +To run the sample, you need to enable the API at: {{required_api_url}} +{% endif %} + +{% if required_role %} +To run the sample, you need to have `{{required_role}}` role. +{% endif %} + +{{other_required_steps}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +{% if not sample.hide_cloudshell_button %} +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst +{% endif %} + + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.file)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/scripts/readme-gen/templates/auth.tmpl.rst b/scripts/readme-gen/templates/auth.tmpl.rst new file mode 100644 index 000000000..1446b94a5 --- /dev/null +++ b/scripts/readme-gen/templates/auth.tmpl.rst @@ -0,0 +1,9 @@ +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started diff --git a/scripts/readme-gen/templates/auth_api_key.tmpl.rst b/scripts/readme-gen/templates/auth_api_key.tmpl.rst new file mode 100644 index 000000000..11957ce27 --- /dev/null +++ b/scripts/readme-gen/templates/auth_api_key.tmpl.rst @@ -0,0 +1,14 @@ +Authentication +++++++++++++++ + +Authentication for this service is done via an `API Key`_. To obtain an API +Key: + +1. Open the `Cloud Platform Console`_ +2. Make sure that billing is enabled for your project. +3. From the **Credentials** page, create a new **API Key** or use an existing + one for your project. + +.. _API Key: + https://developers.google.com/api-client-library/python/guide/aaa_apikeys +.. _Cloud Console: https://console.cloud.google.com/project?_ diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst new file mode 100644 index 000000000..a0406dba8 --- /dev/null +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -0,0 +1,29 @@ +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/scripts/readme-gen/templates/install_portaudio.tmpl.rst b/scripts/readme-gen/templates/install_portaudio.tmpl.rst new file mode 100644 index 000000000..5ea33d18c --- /dev/null +++ b/scripts/readme-gen/templates/install_portaudio.tmpl.rst @@ -0,0 +1,35 @@ +Install PortAudio ++++++++++++++++++ + +Install `PortAudio`_. This is required by the `PyAudio`_ library to stream +audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the +platform. + +* For Mac OS X, you can use `Homebrew`_:: + + brew install portaudio + + **Note**: if you encounter an error when running `pip install` that indicates + it can't find `portaudio.h`, try running `pip install` with the following + flags:: + + pip install --global-option='build_ext' \ + --global-option='-I/usr/local/include' \ + --global-option='-L/usr/local/lib' \ + pyaudio + +* For Debian / Ubuntu Linux:: + + apt-get install portaudio19-dev python-all-dev + +* Windows may work without having to install PortAudio explicitly (it will get + installed with PyAudio). + +For more details, see the `PyAudio installation`_ page. + + +.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/ +.. _PortAudio: http://www.portaudio.com/ +.. _PyAudio installation: + https://people.csail.mit.edu/hubert/pyaudio/#downloads +.. _Homebrew: http://brew.sh diff --git a/synth.py b/synth.py index 7fba81a5c..7aaf2d0a2 100644 --- a/synth.py +++ b/synth.py @@ -16,6 +16,7 @@ import synthtool as s from synthtool import gcp +from synthtool.languages import python gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() @@ -58,9 +59,15 @@ # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- -templated_files = common.py_library(cov_level=100) +templated_files = common.py_library(cov_level=100, samples=True) s.move(templated_files, excludes=["noxfile.py"]) +# ---------------------------------------------------------------------------- +# Samples templates +# ---------------------------------------------------------------------------- +python.py_samples() + + s.replace( "docs/conf.py", r'\{"members": True\}', diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 000000000..b05fbd630 --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,3 @@ +test-env.sh +service-account.json +client-secrets.json \ No newline at end of file From d6b557963714c919d3fe6ef7eada86e4afd67cc5 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 21 Jul 2020 14:00:48 +0200 Subject: [PATCH 098/100] chore: fix samples tests dependencies --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 93f36e038..6339b3047 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery[pandas]==1.25.0 +google-cloud-bigquery[pandas,bqstorage,pyarrow]==1.25.0 google-auth-oauthlib==0.4.1 ipython==7.15.0 matplotlib==3.2.2 From 62dc5e819bf2ea8e839e8f17da0cea082ef1e0e4 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 21 Jul 2020 13:44:40 +0200 Subject: [PATCH 099/100] chore: exclude orphan samples doc files from TOC --- docs/conf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 332b81b10..251e1f4ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,7 +90,12 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build"] +exclude_patterns = [ + "_build", + "samples/AUTHORING_GUIDE.md", + "samples/CONTRIBUTING.md", + "samples/snippets/README.rst", +] # The reST default role (used for this markup: `text`) to use for all # documents. From ac1ab9218af32a0e53f2f5de17678a852892946a Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 21 Jul 2020 14:57:14 +0200 Subject: [PATCH 100/100] Change the project env variable to use in tests --- samples/snippets/noxfile_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py index 6ecfa40e0..b1f495f01 100644 --- a/samples/snippets/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -27,8 +27,8 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", + "gcloud_project_env": 'GOOGLE_CLOUD_PROJECT', + # "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": {},