diff --git a/bigquery/cloud-client/README.rst b/bigquery/cloud-client/README.rst index 60b51a7cac7..cca0ff3e505 100644 --- a/bigquery/cloud-client/README.rst +++ b/bigquery/cloud-client/README.rst @@ -55,22 +55,6 @@ Install Dependencies Samples ------------------------------------------------------------------------------- -Simple Application -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python simple_app.py - - Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -87,92 +71,11 @@ To run this sample: $ python quickstart.py -Query -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python query.py - - usage: query.py [-h] [--use_standard_sql] - [--destination_table DESTINATION_TABLE] - query - - Command-line application to perform queries in BigQuery. - - For more information, see the README.rst. - - Example invocation: - $ python query.py '#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus' - - positional arguments: - query BigQuery SQL Query. - - optional arguments: - -h, --help show this help message and exit - --use_standard_sql Use standard SQL syntax. - --destination_table DESTINATION_TABLE - Destination table to use for results. Example: - my_dataset.my_table - - - -Parameterized Query -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/query_params.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python query_params.py - - usage: query_params.py [-h] {named,positional,array,timestamp,struct} ... - - Command-line app to perform queries with parameters in BigQuery. - - For more information, see the README.rst. - - Example invocation: - $ python query_params.py named 'romeoandjuliet' 100 - $ python query_params.py positional 'romeoandjuliet' 100 - - positional arguments: - {named,positional,array,timestamp,struct} - samples - named Run a query with named parameters. - positional Run a query with positional parameters. - array Run a query with an array parameter. - timestamp Run a query with a timestamp parameter. - struct Run a query with a struct parameter. - - optional arguments: - -h, --help show this help message and exit - - - -Snippets +Simple Application +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/snippets.py,bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst @@ -181,56 +84,14 @@ To run this sample: .. code-block:: bash - $ python snippets.py - - usage: snippets.py [-h] [--project PROJECT] - {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} - ... - - Samples that demonstrate basic operations in the BigQuery API. - - For more information, see the README.rst. - - Example invocation: - $ python snippets.py list-datasets - - The dataset and table should already exist. - - positional arguments: - {list-projects,list-datasets,list-tables,create-table,list-rows,copy-table,delete-table} - list-projects - list-datasets Lists all datasets in a given project. If no project - is specified, then the currently active project is - used. - list-datasets Lists all datasets in a given project. If no project - is specified, then the currently active project is - used. - list-tables Lists all of the tables in a given dataset. If no - project is specified, then the currently active - project is used. - create-table Creates a simple table in the given dataset. If no - project is specified, then the currently active - project is used. - list-rows Prints rows in the given table. Will print 25 rows at - most for brevity as tables can contain large amounts - of rows. If no project is specified, then the - currently active project is used. - copy-table Copies a table. If no project is specified, then the - currently active project is used. - delete-table Deletes a table in a given dataset. If no project is - specified, then the currently active project is used. - - optional arguments: - -h, --help show this help message and exit - --project PROJECT - + $ python simple_app.py -Load data from a file +User Credentials +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_file.py,bigquery/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/user_credentials.py,bigquery/cloud-client/README.rst @@ -239,140 +100,22 @@ To run this sample: .. code-block:: bash - $ python load_data_from_file.py - - usage: load_data_from_file.py [-h] dataset_id table_id source_file_name - - Loads data into BigQuery from a local file. + $ python user_credentials.py - For more information, see the README.rst. + usage: user_credentials.py [-h] [--launch-browser] project query - Example invocation: - $ python load_data_from_file.py example_dataset example_table \ - example-data.csv + Command-line application to run a query using user credentials. - The dataset and table should already exist. + You must supply a client secrets file, which would normally be bundled with + your application. positional arguments: - dataset_id - table_id - source_file_name Path to a .csv file to upload. + project Project to use for BigQuery billing. + query BigQuery SQL Query. optional arguments: -h, --help show this help message and exit - - - -Load data from Cloud Storage -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/load_data_from_gcs.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python load_data_from_gcs.py - - usage: load_data_from_gcs.py [-h] dataset_id table_id source - - Loads data into BigQuery from an object in Google Cloud Storage. - - For more information, see the README.rst. - - Example invocation: - $ python load_data_from_gcs.py example_dataset example_table \ - gs://example-bucket/example-data.csv - - The dataset and table should already exist. - - positional arguments: - dataset_id - table_id - source The Google Cloud Storage object to load. Must be in the format - gs://bucket_name/object_name - - optional arguments: - -h, --help show this help message and exit - - - -Load streaming data -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/stream_data.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python stream_data.py - - usage: stream_data.py [-h] dataset_id table_id json_data - - Loads a single row of data directly into BigQuery. - - For more information, see the README.rst. - - Example invocation: - $ python stream_data.py example_dataset example_table \ - '["Gandalf", 2000]' - - The dataset and table should already exist. - - positional arguments: - dataset_id - table_id - json_data The row to load into BigQuery as an array in JSON format. - - optional arguments: - -h, --help show this help message and exit - - - -Export data to Cloud Storage -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/export_data_to_gcs.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python export_data_to_gcs.py - - usage: export_data_to_gcs.py [-h] dataset_id table_id destination - - Exports data from BigQuery to an object in Google Cloud Storage. - - For more information, see the README.rst. - - Example invocation: - $ python export_data_to_gcs.py example_dataset example_table \ - gs://example-bucket/example-data.csv - - The dataset and table should already exist. - - positional arguments: - dataset_id - table_id - destination The destination Google Cloud Storage object. Must be in the - format gs://bucket_name/object_name - - optional arguments: - -h, --help show this help message and exit + --launch-browser Use a local server flow to authenticate. diff --git a/bigquery/cloud-client/README.rst.in b/bigquery/cloud-client/README.rst.in index 55572c0709e..008b5179565 100644 --- a/bigquery/cloud-client/README.rst.in +++ b/bigquery/cloud-client/README.rst.in @@ -4,7 +4,7 @@ product: name: Google BigQuery short_name: BigQuery url: https://cloud.google.com/bigquery/docs - description: > + description: > `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on @@ -16,30 +16,12 @@ setup: - install_deps samples: -- name: Simple Application - file: simple_app.py - name: Quickstart file: quickstart.py -- name: Query - file: query.py - show_help: true -- name: Parameterized Query - file: query_params.py - show_help: true -- name: Snippets - file: snippets.py - show_help: true -- name: Load data from a file - file: load_data_from_file.py - show_help: true -- name: Load data from Cloud Storage - file: load_data_from_gcs.py - show_help: true -- name: Load streaming data - file: stream_data.py - show_help: true -- name: Export data to Cloud Storage - file: export_data_to_gcs.py +- name: Simple Application + file: simple_app.py +- name: User Credentials + file: user_credentials.py show_help: true cloud_client_library: true diff --git a/bigquery/cloud-client/auth_snippets.py b/bigquery/cloud-client/auth_snippets.py deleted file mode 100644 index 1dc6fddd204..00000000000 --- a/bigquery/cloud-client/auth_snippets.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Demonstrates how to authenticate to Google BigQuery using the Google Cloud -Client Libraries.""" - -import argparse - - -def implicit(): - from google.cloud import bigquery - - # If you don't specify credentials when constructing the client, the - # client library will look for credentials in the environment. - bigquery_client = bigquery.Client() - - # Make an authenticated API request - datasets = list(bigquery_client.list_datasets()) - print(datasets) - - -def explicit(): - from google.cloud import bigquery - - # Explicitly use service account credentials by specifying the private key - # file. All clients in google-cloud-python have this helper, see - # https://googlecloudplatform.github.io/google-cloud-python/latest/core/auth.html#service-accounts - bigquery_client = bigquery.Client.from_service_account_json( - 'service_account.json') - - # Make an authenticated API request - datasets = list(bigquery_client.list_datasets()) - print(datasets) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - subparsers = parser.add_subparsers(dest='command') - subparsers.add_parser('implicit', help=implicit.__doc__) - subparsers.add_parser('explicit', help=explicit.__doc__) - - args = parser.parse_args() - - if args.command == 'implicit': - implicit() - elif args.command == 'explicit': - explicit() diff --git a/bigquery/cloud-client/auth_snippets_test.py b/bigquery/cloud-client/auth_snippets_test.py deleted file mode 100644 index 5b5f2cac00c..00000000000 --- a/bigquery/cloud-client/auth_snippets_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import mock - -import auth_snippets - - -def test_implicit(): - auth_snippets.implicit() - - -def test_explicit(): - with open(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) as creds_file: - creds_file_data = creds_file.read() - - open_mock = mock.mock_open(read_data=creds_file_data) - - with mock.patch('io.open', open_mock): - auth_snippets.explicit() diff --git a/bigquery/cloud-client/export_data_to_gcs.py b/bigquery/cloud-client/export_data_to_gcs.py deleted file mode 100644 index 5993ef0f6a2..00000000000 --- a/bigquery/cloud-client/export_data_to_gcs.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Exports data from BigQuery to an object in Google Cloud Storage. - -For more information, see the README.rst. - -Example invocation: - $ python export_data_to_gcs.py example_dataset example_table \\ - gs://example-bucket/example-data.csv - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def export_data_to_gcs(dataset_id, table_id, destination): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - job = bigquery_client.extract_table(table_ref, destination) - - job.result() # Waits for job to complete - - print('Exported {}:{} to {}'.format( - dataset_id, table_id, destination)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'destination', help='The destination Google Cloud Storage object. ' - 'Must be in the format gs://bucket_name/object_name') - - args = parser.parse_args() - - export_data_to_gcs( - args.dataset_id, - args.table_id, - args.destination) diff --git a/bigquery/cloud-client/export_data_to_gcs_test.py b/bigquery/cloud-client/export_data_to_gcs_test.py deleted file mode 100644 index a41cfd226fc..00000000000 --- a/bigquery/cloud-client/export_data_to_gcs_test.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import export_data_to_gcs - -BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_table' - - -def test_export_data_to_gcs(capsys): - export_data_to_gcs.export_data_to_gcs( - DATASET_ID, - TABLE_ID, - 'gs://{}/test-export-data-to-gcs.csv'.format(BUCKET)) - - out, _ = capsys.readouterr() - - assert 'Exported' in out diff --git a/bigquery/cloud-client/load_data_from_file.py b/bigquery/cloud-client/load_data_from_file.py deleted file mode 100644 index e311daa1e62..00000000000 --- a/bigquery/cloud-client/load_data_from_file.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Loads data into BigQuery from a local file. - -For more information, see the README.rst. - -Example invocation: - $ python load_data_from_file.py example_dataset example_table \\ - example-data.csv - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def load_data_from_file(dataset_id, table_id, source_file_name): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - with open(source_file_name, 'rb') as source_file: - # This example uses CSV, but you can use other formats. - # See https://cloud.google.com/bigquery/loading-data - job_config = bigquery.LoadJobConfig() - job_config.source_format = 'text/csv' - job = bigquery_client.load_table_from_file( - source_file, table_ref, job_config=job_config) - - job.result() # Waits for job to complete - - print('Loaded {} rows into {}:{}.'.format( - job.output_rows, dataset_id, table_id)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'source_file_name', help='Path to a .csv file to upload.') - - args = parser.parse_args() - - load_data_from_file( - args.dataset_id, - args.table_id, - args.source_file_name) diff --git a/bigquery/cloud-client/load_data_from_file_test.py b/bigquery/cloud-client/load_data_from_file_test.py deleted file mode 100644 index 960fe62c9d7..00000000000 --- a/bigquery/cloud-client/load_data_from_file_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import load_data_from_file - -RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -def test_load_table(capsys): - data_path = os.path.join(RESOURCES, 'data.csv') - - load_data_from_file.load_data_from_file( - DATASET_ID, - TABLE_ID, - data_path) - - out, _ = capsys.readouterr() - - assert 'Loaded 1 rows' in out diff --git a/bigquery/cloud-client/load_data_from_gcs.py b/bigquery/cloud-client/load_data_from_gcs.py deleted file mode 100644 index 285e6d1b22a..00000000000 --- a/bigquery/cloud-client/load_data_from_gcs.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Loads data into BigQuery from an object in Google Cloud Storage. - -For more information, see the README.rst. - -Example invocation: - $ python load_data_from_gcs.py example_dataset example_table \\ - gs://example-bucket/example-data.csv - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def load_data_from_gcs(dataset_id, table_id, source): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - job = bigquery_client.load_table_from_uri(source, table_ref) - - job.result() # Waits for job to complete - - print('Loaded {} rows into {}:{}.'.format( - job.output_rows, dataset_id, table_id)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'source', help='The Google Cloud Storage object to load. Must be in ' - 'the format gs://bucket_name/object_name') - - args = parser.parse_args() - - load_data_from_gcs( - args.dataset_id, - args.table_id, - args.source) diff --git a/bigquery/cloud-client/load_data_from_gcs_test.py b/bigquery/cloud-client/load_data_from_gcs_test.py deleted file mode 100644 index dbd39fc5cec..00000000000 --- a/bigquery/cloud-client/load_data_from_gcs_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import load_data_from_gcs - -BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -def test_load_table(capsys): - cloud_storage_input_uri = 'gs://{}/data.csv'.format(BUCKET) - - load_data_from_gcs.load_data_from_gcs( - DATASET_ID, - TABLE_ID, - cloud_storage_input_uri) - - out, _ = capsys.readouterr() - - assert 'Loaded 1 rows' in out diff --git a/bigquery/cloud-client/query.py b/bigquery/cloud-client/query.py deleted file mode 100755 index 19605bbad4d..00000000000 --- a/bigquery/cloud-client/query.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to perform queries in BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python query.py '#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus' -""" - -import argparse - -from google.cloud import bigquery - - -def query(query): - client = bigquery.Client() - query_job = client.query(query) - - # Print the results. - for row in query_job.result(): # Waits for job to complete. - print(row) - - -def query_standard_sql(query): - client = bigquery.Client() - job_config = bigquery.QueryJobConfig() - - # Set use_legacy_sql to False to use standard SQL syntax. - # Note that queries are treated as standard SQL by default. - job_config.use_legacy_sql = False - query_job = client.query(query, job_config=job_config) - - # Print the results. - for row in query_job.result(): # Waits for job to complete. - print(row) - - -def query_destination_table(query, dest_dataset_id, dest_table_id): - client = bigquery.Client() - job_config = bigquery.QueryJobConfig() - - # Allow for query results larger than the maximum response size. - job_config.allow_large_results = True - - # When large results are allowed, a destination table must be set. - dest_dataset_ref = client.dataset(dest_dataset_id) - dest_table_ref = dest_dataset_ref.table(dest_table_id) - job_config.destination = dest_table_ref - - # Allow the results table to be overwritten. - job_config.write_disposition = 'WRITE_TRUNCATE' - - query_job = client.query(query, job_config=job_config) - - # Print the results. - for row in query_job.result(): # Waits for job to complete. - print(row) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('query', help='BigQuery SQL Query.') - parser.add_argument( - '--use_standard_sql', - action='store_true', - help='Use standard SQL syntax.') - parser.add_argument( - '--destination_table', - type=str, - help=( - 'Destination table to use for results. ' - 'Example: my_dataset.my_table')) - - args = parser.parse_args() - - if args.use_standard_sql: - query_standard_sql(args.query) - elif args.destination_table: - dataset, table = args.destination_table.split('.') - query_destination_table(args.query, dataset, table) - else: - query(args.query) diff --git a/bigquery/cloud-client/query_params.py b/bigquery/cloud-client/query_params.py deleted file mode 100644 index 594f51c758d..00000000000 --- a/bigquery/cloud-client/query_params.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line app to perform queries with parameters in BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python query_params.py named 'romeoandjuliet' 100 - $ python query_params.py positional 'romeoandjuliet' 100 -""" - -import argparse -import datetime - -from google.cloud import bigquery -import pytz - - -def query_positional_params(corpus, min_word_count): - client = bigquery.Client() - query = """ - SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = ? - AND word_count >= ? - ORDER BY word_count DESC; - """ - # Set the name to None to use positional parameters (? symbol in the - # query). Note that you cannot mix named and positional parameters. - # See: https://cloud.google.com/bigquery/docs/parameterized-queries/ - query_params = [ - bigquery.ScalarQueryParameter(None, 'STRING', corpus), - bigquery.ScalarQueryParameter(None, 'INT64', min_word_count) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Wait for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_named_params(corpus, min_word_count): - client = bigquery.Client() - query = """ - SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = @corpus - AND word_count >= @min_word_count - ORDER BY word_count DESC; - """ - query_params = [ - bigquery.ScalarQueryParameter('corpus', 'STRING', corpus), - bigquery.ScalarQueryParameter( - 'min_word_count', 'INT64', min_word_count) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Wait for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_array_params(gender, states): - client = bigquery.Client() - query = """ - SELECT name, sum(number) as count - FROM `bigquery-public-data.usa_names.usa_1910_2013` - WHERE gender = @gender - AND state IN UNNEST(@states) - GROUP BY name - ORDER BY count DESC - LIMIT 10; - """ - query_params = [ - bigquery.ScalarQueryParameter('gender', 'STRING', gender), - bigquery.ArrayQueryParameter('states', 'STRING', states) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Wait for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_timestamp_params(year, month, day, hour, minute): - client = bigquery.Client() - query = 'SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);' - query_params = [ - bigquery.ScalarQueryParameter( - 'ts_value', - 'TIMESTAMP', - datetime.datetime(year, month, day, hour, minute, tzinfo=pytz.UTC)) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Waits for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -def query_struct_params(x, y): - client = bigquery.Client() - query = 'SELECT @struct_value AS s;' - query_params = [ - bigquery.StructQueryParameter( - 'struct_value', - bigquery.ScalarQueryParameter('x', 'INT64', x), - bigquery.ScalarQueryParameter('y', 'STRING', y) - ) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query(query, job_config=job_config) - - query_job.result() # Waits for job to complete - - # Print the results. - destination_table_ref = query_job.destination - table = client.get_table(destination_table_ref) - for row in client.list_rows(table): - print(row) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - subparsers = parser.add_subparsers(dest='sample', help='samples') - named_parser = subparsers.add_parser( - 'named', - help='Run a query with named parameters.') - named_parser.add_argument( - 'corpus', - help='Corpus to search from Shakespeare dataset.') - named_parser.add_argument( - 'min_word_count', - help='Minimum count of words to query.', - type=int) - positional_parser = subparsers.add_parser( - 'positional', - help='Run a query with positional parameters.') - positional_parser.add_argument( - 'corpus', - help='Corpus to search from Shakespeare dataset.') - positional_parser.add_argument( - 'min_word_count', - help='Minimum count of words to query.', - type=int) - array_parser = subparsers.add_parser( - 'array', - help='Run a query with an array parameter.') - array_parser.add_argument( - 'gender', - choices=['F', 'M'], - help='Gender of baby in the Social Security baby names database.') - array_parser.add_argument( - 'states', - help='U.S. States to consider for popular baby names.', - nargs='+') - timestamp_parser = subparsers.add_parser( - 'timestamp', - help='Run a query with a timestamp parameter.') - timestamp_parser.add_argument('year', type=int) - timestamp_parser.add_argument('month', type=int) - timestamp_parser.add_argument('day', type=int) - timestamp_parser.add_argument('hour', type=int) - timestamp_parser.add_argument('minute', type=int) - struct_parser = subparsers.add_parser( - 'struct', - help='Run a query with a struct parameter.') - struct_parser.add_argument('x', help='Integer for x', type=int) - struct_parser.add_argument('y', help='String for y') - args = parser.parse_args() - - if args.sample == 'named': - query_named_params(args.corpus, args.min_word_count) - elif args.sample == 'positional': - query_positional_params(args.corpus, args.min_word_count) - elif args.sample == 'array': - query_array_params(args.gender, args.states) - elif args.sample == 'timestamp': - query_timestamp_params( - args.year, args.month, args.day, args.hour, args.minute) - elif args.sample == 'struct': - query_struct_params(args.x, args.y) - else: - print('Unexpected value for sample') diff --git a/bigquery/cloud-client/query_params_test.py b/bigquery/cloud-client/query_params_test.py deleted file mode 100644 index f4b4931372b..00000000000 --- a/bigquery/cloud-client/query_params_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import query_params - - -def test_query_array_params(capsys): - query_params.query_array_params( - gender='M', - states=['WA', 'WI', 'WV', 'WY']) - out, _ = capsys.readouterr() - assert 'James' in out - - -def test_query_named_params(capsys): - query_params.query_named_params( - corpus='romeoandjuliet', - min_word_count=100) - out, _ = capsys.readouterr() - assert 'the' in out - - -def test_query_positional_params(capsys): - query_params.query_positional_params( - corpus='romeoandjuliet', - min_word_count=100) - out, _ = capsys.readouterr() - assert 'the' in out - - -def test_query_struct_params(capsys): - query_params.query_struct_params(765, "hello world") - out, _ = capsys.readouterr() - assert '765' in out - assert 'hello world' in out - - -def test_query_timestamp_params(capsys): - query_params.query_timestamp_params(2016, 12, 7, 8, 0) - out, _ = capsys.readouterr() - assert '2016, 12, 7, 9, 0' in out diff --git a/bigquery/cloud-client/query_test.py b/bigquery/cloud-client/query_test.py deleted file mode 100644 index 3d456cb5967..00000000000 --- a/bigquery/cloud-client/query_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import query - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_destination_table' - - -def test_query(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query_string = '''#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus - LIMIT 10;''' - - query.query(query_string) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out - - -def test_query_standard_sql(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query_string = '''SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus - LIMIT 10;''' - - query.query_standard_sql(query_string) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out - - -def test_query_destination_table(capsys): - # Query only outputs the first 10 rows, sort results to avoid randomness - query_string = '''#standardSQL - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus - ORDER BY corpus - LIMIT 10;''' - - query.query_destination_table(query_string, DATASET_ID, TABLE_ID) - - out, _ = capsys.readouterr() - - assert 'antonyandcleopatra' in out diff --git a/bigquery/cloud-client/simple_app.py b/bigquery/cloud-client/simple_app.py index 9f266ed7aac..a09e97f1246 100644 --- a/bigquery/cloud-client/simple_app.py +++ b/bigquery/cloud-client/simple_app.py @@ -15,17 +15,17 @@ # limitations under the License. """Simple application that performs a query with BigQuery.""" -# [START all] +# [START bigquery_simple_app_all] # [START bigquery_simple_app_deps] from google.cloud import bigquery # [END bigquery_simple_app_deps] def query_stackoverflow(): - # [START create_client] + # [START bigquery_simple_app_client] client = bigquery.Client() - # [END create_client] - # [START run_query] + # [END bigquery_simple_app_client] + # [START bigquery_simple_app_query] query_job = client.query(""" SELECT CONCAT( @@ -38,14 +38,14 @@ def query_stackoverflow(): LIMIT 10""") results = query_job.result() # Waits for job to complete. - # [END run_query] + # [END bigquery_simple_app_query] - # [START print_results] + # [START bigquery_simple_app_print] for row in results: print("{} : {} views".format(row.url, row.view_count)) - # [END print_results] + # [END bigquery_simple_app_print] if __name__ == '__main__': query_stackoverflow() -# [END all] +# [END bigquery_simple_app_all] diff --git a/bigquery/cloud-client/snippets.py b/bigquery/cloud-client/snippets.py deleted file mode 100644 index ee75f7fc269..00000000000 --- a/bigquery/cloud-client/snippets.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Samples that demonstrate basic operations in the BigQuery API. - -For more information, see the README.rst. - -Example invocation: - $ python snippets.py list-datasets - -The dataset and table should already exist. -""" - -import argparse - -from google.cloud import bigquery - - -def list_projects(): - bigquery_client = bigquery.Client() - - for project in bigquery_client.list_projects(): - print(project.project_id) - - -def list_datasets(project=None): - """Lists all datasets in a given project. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - - for dataset in bigquery_client.list_datasets(): - print(dataset.dataset_id) - - -def create_dataset(dataset_id, project=None): - """Craetes a dataset in a given project. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - - dataset_ref = bigquery_client.dataset(dataset_id) - - dataset = bigquery_client.create_dataset(bigquery.Dataset(dataset_ref)) - - print('Created dataset {}.'.format(dataset.dataset_id)) - - -def list_tables(dataset_id, project=None): - """Lists all of the tables in a given dataset. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - - for table in bigquery_client.list_dataset_tables(dataset_ref): - print(table.table_id) - - -def create_table(dataset_id, table_id, project=None): - """Creates a simple table in the given dataset. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - - table_ref = dataset_ref.table(table_id) - table = bigquery.Table(table_ref) - - # Set the table schema - table.schema = ( - bigquery.SchemaField('Name', 'STRING'), - bigquery.SchemaField('Age', 'INTEGER'), - bigquery.SchemaField('Weight', 'FLOAT'), - ) - - table = bigquery_client.create_table(table) - - print('Created table {} in dataset {}.'.format(table_id, dataset_id)) - - -def list_rows(dataset_id, table_id, project=None): - """Prints rows in the given table. - - Will print 25 rows at most for brevity as tables can contain large amounts - of rows. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - # Get the table from the API so that the schema is available. - table = bigquery_client.get_table(table_ref) - - # Load at most 25 results. - rows = bigquery_client.list_rows(table, max_results=25) - - # Use format to create a simple table. - format_string = '{!s:<16} ' * len(table.schema) - - # Print schema field names - field_names = [field.name for field in table.schema] - print(format_string.format(*field_names)) - - for row in rows: - print(format_string.format(*row)) - - -def copy_table(dataset_id, table_id, new_table_id, project=None): - """Copies a table. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - # This sample shows the destination table in the same dataset and project, - # however, it's possible to copy across datasets and projects. You can - # also copy multiple source tables into a single destination table by - # providing addtional arguments to `copy_table`. - destination_table_ref = dataset_ref.table(new_table_id) - - # Create a job to copy the table to the destination table. - # Start by creating a job configuration - job_config = bigquery.CopyJobConfig() - - # Configure the job to create the table if it doesn't exist. - job_config.create_disposition = ( - bigquery.job.CreateDisposition.CREATE_IF_NEEDED) - - copy_job = bigquery_client.copy_table( - table_ref, destination_table_ref, job_config=job_config) - - print('Waiting for job to finish...') - copy_job.result() - - print('Table {} copied to {}.'.format(table_id, new_table_id)) - - -def delete_table(dataset_id, table_id, project=None): - """Deletes a table in a given dataset. - - If no project is specified, then the currently active project is used. - """ - bigquery_client = bigquery.Client(project=project) - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - - bigquery_client.delete_table(table_ref) - - print('Table {}:{} deleted.'.format(dataset_id, table_id)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--project', default=None) - - subparsers = parser.add_subparsers(dest='command') - - list_projects_parser = subparsers.add_parser( - 'list-projects', help=list_projects.__doc__) - - list_datasets_parser = subparsers.add_parser( - 'list-datasets', help=list_datasets.__doc__) - - create_dataset_parser = subparsers.add_parser( - 'list-datasets', help=list_datasets.__doc__) - create_dataset_parser.add_argument('dataset_id') - - list_tables_parser = subparsers.add_parser( - 'list-tables', help=list_tables.__doc__) - list_tables_parser.add_argument('dataset_id') - - create_table_parser = subparsers.add_parser( - 'create-table', help=create_table.__doc__) - create_table_parser.add_argument('dataset_id') - create_table_parser.add_argument('table_id') - - list_rows_parser = subparsers.add_parser( - 'list-rows', help=list_rows.__doc__) - list_rows_parser.add_argument('dataset_id') - list_rows_parser.add_argument('table_id') - - copy_table_parser = subparsers.add_parser( - 'copy-table', help=copy_table.__doc__) - copy_table_parser.add_argument('dataset_id') - copy_table_parser.add_argument('table_id') - copy_table_parser.add_argument('new_table_id') - - delete_table_parser = subparsers.add_parser( - 'delete-table', help=delete_table.__doc__) - delete_table_parser.add_argument('dataset_id') - delete_table_parser.add_argument('table_id') - - args = parser.parse_args() - - if args.command == 'list-projects': - list_projects() - elif args.command == 'list-datasets': - list_datasets(args.project) - elif args.command == 'create-dataset': - create_dataset(args.dataset_id, args.project) - elif args.command == 'list-tables': - list_tables(args.dataset_id, args.project) - elif args.command == 'create-table': - create_table(args.dataset_id, args.table_id, args.project) - elif args.command == 'list-rows': - list_rows(args.dataset_id, args.table_id, args.project) - elif args.command == 'copy-table': - copy_table(args.dataset_id, args.table_id, args.new_table_id) - elif args.command == 'delete-table': - delete_table(args.dataset_id, args.table_id, args.project) diff --git a/bigquery/cloud-client/snippets_test.py b/bigquery/cloud-client/snippets_test.py deleted file mode 100644 index 5f666ccc6c8..00000000000 --- a/bigquery/cloud-client/snippets_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from google.cloud import bigquery -from google.cloud.exceptions import NotFound -import pytest - -import snippets - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_table' - - -def test_list_projects(): - snippets.list_projects() - # No need to check the ouput, lack of exception is enough. - - -def test_list_datasets(capsys): - # Requires the dataset to have been created in the test project. - snippets.list_datasets() - - out, _ = capsys.readouterr() - - assert DATASET_ID in out - - -@pytest.fixture -def cleanup_dataset(): - dataset_id = 'test_temporary_dataset' - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - - if dataset_exists(dataset_ref, bigquery_client): - bigquery_client.delete_dataset(dataset_ref) - - yield dataset_id - - if dataset_exists(dataset_ref, bigquery_client): - bigquery_client.delete_dataset(dataset_ref) - - -def dataset_exists(dataset, client): - try: - client.get_dataset(dataset) - return True - except NotFound: - return False - - -def test_create_dataset(capsys, cleanup_dataset): - snippets.create_dataset(cleanup_dataset) - - out, _ = capsys.readouterr() - - assert cleanup_dataset in out - - -def test_list_tables(capsys): - # Requires the dataset and table to have been created in the test project. - snippets.list_tables(DATASET_ID) - - out, _ = capsys.readouterr() - - assert TABLE_ID in out - - -def test_list_rows(capsys): - # Requires the dataset and table to have been created in the test project. - - # Check for the schema. It's okay if the table is empty as long as there - # aren't any errors. - - snippets.list_rows(DATASET_ID, TABLE_ID) - - out, _ = capsys.readouterr() - - assert 'Name' in out - assert 'Age' in out - - -@pytest.fixture -def temporary_table(): - """Fixture that returns a factory for tables that do not yet exist and - will be automatically deleted after the test.""" - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(DATASET_ID) - tables = [] - - def factory(table_id): - new_table_ref = dataset_ref.table(table_id) - if table_exists(new_table_ref, bigquery_client): - bigquery_client.delete_table(new_table_ref) - tables.append(new_table_ref) - return new_table_ref - - yield factory - - for table in tables: - if table_exists(table, bigquery_client): - bigquery_client.delete_table(table) - - -def table_exists(table, client): - try: - client.get_table(table) - return True - except NotFound: - return False - - -def test_create_table(temporary_table): - bigquery_client = bigquery.Client() - new_table = temporary_table('test_create_table') - snippets.create_table(DATASET_ID, new_table.table_id) - assert table_exists(new_table, bigquery_client) - - -@pytest.mark.slow -def test_copy_table(temporary_table): - bigquery_client = bigquery.Client() - new_table = temporary_table('test_copy_table') - snippets.copy_table(DATASET_ID, TABLE_ID, new_table.table_id) - assert table_exists(new_table, bigquery_client) - - -def test_delete_table(): - # Create a table to delete - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(DATASET_ID) - table_ref = dataset_ref.table('test_delete_table') - table = bigquery.Table(table_ref) - - if not table_exists(table, bigquery_client): - table.schema = [bigquery.SchemaField('id', 'INTEGER')] - table = bigquery_client.create_table(table) - - snippets.delete_table(DATASET_ID, table.table_id) - - assert not table_exists(table, bigquery_client) diff --git a/bigquery/cloud-client/stream_data.py b/bigquery/cloud-client/stream_data.py deleted file mode 100644 index c5496004820..00000000000 --- a/bigquery/cloud-client/stream_data.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Loads a single row of data directly into BigQuery. - -For more information, see the README.rst. - -Example invocation: - $ python stream_data.py example_dataset example_table \\ - '["Gandalf", 2000]' - -The dataset and table should already exist. -""" - -import argparse -import json -from pprint import pprint - -from google.cloud import bigquery - - -def stream_data(dataset_id, table_id, json_data): - bigquery_client = bigquery.Client() - dataset_ref = bigquery_client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - data = json.loads(json_data) - - # Get the table from the API so that the schema is available. - table = bigquery_client.get_table(table_ref) - - rows = [data] - errors = bigquery_client.create_rows(table, rows) - - if not errors: - print('Loaded 1 row into {}:{}'.format(dataset_id, table_id)) - else: - print('Errors:') - pprint(errors) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id') - parser.add_argument('table_id') - parser.add_argument( - 'json_data', - help='The row to load into BigQuery as an array in JSON format.') - - args = parser.parse_args() - - stream_data( - args.dataset_id, - args.table_id, - args.json_data) diff --git a/bigquery/cloud-client/stream_data_test.py b/bigquery/cloud-client/stream_data_test.py deleted file mode 100644 index 25982b88ad5..00000000000 --- a/bigquery/cloud-client/stream_data_test.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import stream_data - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -def test_stream_data(capsys): - stream_data.stream_data( - DATASET_ID, - TABLE_ID, - '["Gandalf", 2000]') - - out, _ = capsys.readouterr() - - assert 'Loaded 1 row' in out diff --git a/bigquery/rest/README.rst b/bigquery/rest/README.rst deleted file mode 100644 index 33fcbdc94c9..00000000000 --- a/bigquery/rest/README.rst +++ /dev/null @@ -1,95 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google BigQuery Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/rest/README.rst - - -This directory contains samples for Google BigQuery. `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on analyzing data to find meaningful insights, use familiar SQL, and take advantage of our pay-as-you-go model. - - - - -.. _Google BigQuery: https://cloud.google.com/bigquery/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Label a dataset or table -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/rest/labels.py,bigquery/rest/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python labels.py - - usage: labels.py [-h] [--project_id PROJECT_ID] [--table_id TABLE_ID] - dataset_id label_key label_value - - Application to add or modify a label on a BigQuery dataset or table. - - positional arguments: - dataset_id BigQuery dataset ID. - label_key Key for new/modified label. - label_value Value for new/modified label. - - optional arguments: - -h, --help show this help message and exit - --project_id PROJECT_ID - Google Cloud project ID. If not set, uses a default - value from the environment. - --table_id TABLE_ID BigQuery table ID. If present, a label is added to the - specified table instead of the dataset. - - - - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigquery/rest/README.rst.in b/bigquery/rest/README.rst.in deleted file mode 100644 index 8acc9bd5c5e..00000000000 --- a/bigquery/rest/README.rst.in +++ /dev/null @@ -1,25 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google BigQuery - short_name: BigQuery - url: https://cloud.google.com/bigquery/docs - description: > - `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost - analytics data warehouse. BigQuery is NoOps—there is no infrastructure to - manage and you don't need a database administrator—so you can focus on - analyzing data to find meaningful insights, use familiar SQL, and take - advantage of our pay-as-you-go model. - -setup: -- auth -- install_deps - -samples: -- name: Label a dataset or table - show_help: true - file: labels.py - -cloud_client_library: false - -folder: bigquery/rest \ No newline at end of file diff --git a/bigquery/rest/labels.py b/bigquery/rest/labels.py deleted file mode 100644 index 2d0b0486208..00000000000 --- a/bigquery/rest/labels.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Application to add or modify a label on a BigQuery dataset or table.""" - -import argparse - -import google.auth -import google.auth.transport.requests - - -def label_dataset(dataset_id, label_key, label_value, project_id=None): - """Add or modify a label on a dataset.""" - # Authenticate requests using Google Application Default credentials. - credentials, default_project_id = google.auth.default( - scopes=['https://www.googleapis.com/auth/bigquery']) - session = google.auth.transport.requests.AuthorizedSession(credentials) - - if project_id is None: - project_id = default_project_id - - # Send a PATCH request to add or modify a label. - url_format = ( - 'https://www.googleapis.com/bigquery/v2/' - 'projects/{project_id}/datasets/{dataset_id}') - response = session.patch( - url_format.format(project_id=project_id, dataset_id=dataset_id), - params={'fields': 'labels'}, - json={ - 'labels': { - label_key: label_value, - } - }) - - # Check the response for errors. - response.raise_for_status() - - # Print the new label value from the response. - labels = response.json()['labels'] - print( - 'Updated label "{}" with value "{}"'.format( - label_key, - labels[label_key])) - - -def label_table(dataset_id, table_id, label_key, label_value, project_id=None): - """Add or modify a label on a table.""" - # Authenticate requests using Google Application Default credentials. - credentials, default_project_id = google.auth.default( - scopes=['https://www.googleapis.com/auth/bigquery']) - session = google.auth.transport.requests.AuthorizedSession(credentials) - - if project_id is None: - project_id = default_project_id - - # Send a PATCH request to add or modify a label. - url_format = ( - 'https://www.googleapis.com/bigquery/v2/' - 'projects/{project_id}/datasets/{dataset_id}/tables/{table_id}') - response = session.patch( - url_format.format( - project_id=project_id, - dataset_id=dataset_id, - table_id=table_id), - params={'fields': 'labels'}, - json={ - 'labels': { - label_key: label_value, - } - }) - - # Check the response for errors. - response.raise_for_status() - - # Print the new label value from the response. - labels = response.json()['labels'] - print( - 'Updated label "{}" with value "{}"'.format( - label_key, - labels[label_key])) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('dataset_id', help='BigQuery dataset ID.') - parser.add_argument('label_key', help='Key for new/modified label.') - parser.add_argument('label_value', help='Value for new/modified label.') - parser.add_argument( - '--project_id', - help=( - 'Google Cloud project ID. ' - 'If not set, uses a default value from the environment.'), - default=None) - parser.add_argument( - '--table_id', - help=( - 'BigQuery table ID. ' - 'If present, a label is added to the specified table instead of ' - 'the dataset.'), - default=None) - - args = parser.parse_args() - - if args.table_id is None: - label_dataset( - args.dataset_id, - args.label_key, - args.label_value, - project_id=args.project_id) - else: - label_table( - args.dataset_id, - args.table_id, - args.label_key, - args.label_value, - project_id=args.project_id) diff --git a/bigquery/rest/labels_test.py b/bigquery/rest/labels_test.py deleted file mode 100644 index ca40587bcf8..00000000000 --- a/bigquery/rest/labels_test.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2016, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from labels import label_dataset, label_table - -PROJECT = os.environ['GCLOUD_PROJECT'] - - -def test_label_dataset(capsys): - label_dataset( - 'test_dataset', - 'environment', - 'test', - project_id=PROJECT) - - out, _ = capsys.readouterr() - result = out.split('\n')[0] - - assert 'Updated label "environment" with value "test"' in result - - -def test_label_table(capsys): - label_table( - 'test_dataset', - 'test_table', - 'data-owner', - 'my-team', - project_id=PROJECT) - - out, _ = capsys.readouterr() - result = out.split('\n')[0] - - assert 'Updated label "data-owner" with value "my-team"' in result diff --git a/bigquery/rest/requirements.txt b/bigquery/rest/requirements.txt deleted file mode 100644 index 88961892e01..00000000000 --- a/bigquery/rest/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-auth==1.4.1 -requests==2.18.4