diff --git a/bigquery/cloud-client/.gitignore b/bigquery/cloud-client/.gitignore deleted file mode 100644 index 0dc05ffadec..00000000000 --- a/bigquery/cloud-client/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -client_secrets.json -service_account.json diff --git a/bigquery/cloud-client/README.rst b/bigquery/cloud-client/README.rst index 9446125cd8a..ae3027f050d 100644 --- a/bigquery/cloud-client/README.rst +++ b/bigquery/cloud-client/README.rst @@ -1,149 +1,3 @@ -.. This file is automatically generated. Do not edit this file directly. +These samples have been moved. -Google BigQuery Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/README.rst - - -This directory contains samples for Google BigQuery. `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on analyzing data to find meaningful insights, use familiar SQL, and take advantage of our pay-as-you-go model. - - - - -.. _Google BigQuery: https://cloud.google.com/bigquery/docs - - -To run the sample, you need to have `BigQuery Admin` role. - - - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Quickstart -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/quickstart.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python quickstart.py - - -Simple Application -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python simple_app.py - - -User Credentials -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/user_credentials.py,bigquery/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python user_credentials.py - - usage: user_credentials.py [-h] [--launch-browser] project - - Command-line application to run a query using user credentials. - - You must supply a client secrets file, which would normally be bundled with - your application. - - positional arguments: - project Project to use for BigQuery billing. - - optional arguments: - -h, --help show this help message and exit - --launch-browser Use a local server flow to authenticate. - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file +https://github.com/googleapis/python-bigquery/tree/master/samples/snippets diff --git a/bigquery/cloud-client/README.rst.in b/bigquery/cloud-client/README.rst.in deleted file mode 100644 index 74b7fa94037..00000000000 --- a/bigquery/cloud-client/README.rst.in +++ /dev/null @@ -1,31 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google BigQuery - short_name: BigQuery - url: https://cloud.google.com/bigquery/docs - description: > - `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost - analytics data warehouse. BigQuery is NoOps—there is no infrastructure to - manage and you don't need a database administrator—so you can focus on - analyzing data to find meaningful insights, use familiar SQL, and take - advantage of our pay-as-you-go model. - -required_role: BigQuery Admin - -setup: -- auth -- install_deps - -samples: -- name: Quickstart - file: quickstart.py -- name: Simple Application - file: simple_app.py -- name: User Credentials - file: user_credentials.py - show_help: true - -cloud_client_library: true - -folder: bigquery/cloud-client diff --git a/bigquery/cloud-client/authenticate_service_account.py b/bigquery/cloud-client/authenticate_service_account.py deleted file mode 100644 index 03cb06fd16b..00000000000 --- a/bigquery/cloud-client/authenticate_service_account.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - - -def main(): - key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - - # [START bigquery_client_json_credentials] - from google.cloud import bigquery - from google.oauth2 import service_account - - # TODO(developer): Set key_path to the path to the service account key - # file. - # key_path = "path/to/service_account.json" - - credentials = service_account.Credentials.from_service_account_file( - key_path, - scopes=["https://www.googleapis.com/auth/cloud-platform"], - ) - - client = bigquery.Client( - credentials=credentials, - project=credentials.project_id, - ) - # [END bigquery_client_json_credentials] - return client - - -if __name__ == "__main__": - main() diff --git a/bigquery/cloud-client/authenticate_service_account_test.py b/bigquery/cloud-client/authenticate_service_account_test.py deleted file mode 100644 index 8c436ec206f..00000000000 --- a/bigquery/cloud-client/authenticate_service_account_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import google.auth - -import authenticate_service_account - - -def mock_credentials(*args, **kwargs): - credentials, _ = google.auth.default( - ["https://www.googleapis.com/auth/cloud-platform"] - ) - return credentials - - -def test_main(monkeypatch): - monkeypatch.setattr( - 'google.oauth2.service_account.Credentials.from_service_account_file', - mock_credentials, - ) - client = authenticate_service_account.main() - assert client is not None diff --git a/bigquery/cloud-client/authorized_view_tutorial.py b/bigquery/cloud-client/authorized_view_tutorial.py deleted file mode 100644 index edf83cf7fac..00000000000 --- a/bigquery/cloud-client/authorized_view_tutorial.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def run_authorized_view_tutorial(override_values={}): - # Note to user: This is a group email for testing purposes. Replace with - # your own group email address when running this code. - analyst_group_email = 'example-analyst-group@google.com' - - # [START bigquery_authorized_view_tutorial] - # Create a source dataset - # [START bigquery_avt_create_source_dataset] - from google.cloud import bigquery - - client = bigquery.Client() - source_dataset_id = 'github_source_data' - - # [END bigquery_authorized_view_tutorial] - # [END bigquery_avt_create_source_dataset] - # To facilitate testing, we replace values with alternatives - # provided by the testing harness. - source_dataset_id = override_values.get("source_dataset_id", source_dataset_id) - # [START bigquery_authorized_view_tutorial] - # [START bigquery_avt_create_source_dataset] - - source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) - # Specify the geographic location where the dataset should reside. - source_dataset.location = 'US' - source_dataset = client.create_dataset(source_dataset) # API request - # [END bigquery_avt_create_source_dataset] - - # Populate a source table - # [START bigquery_avt_create_source_table] - source_table_id = 'github_contributors' - job_config = bigquery.QueryJobConfig() - job_config.destination = source_dataset.table(source_table_id) - sql = """ - SELECT commit, author, committer, repo_name - FROM `bigquery-public-data.github_repos.commits` - LIMIT 1000 - """ - query_job = client.query( - sql, - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location='US', - job_config=job_config) # API request - starts the query - - query_job.result() # Waits for the query to finish - # [END bigquery_avt_create_source_table] - - # Create a separate dataset to store your view - # [START bigquery_avt_create_shared_dataset] - shared_dataset_id = 'shared_views' - - # [END bigquery_authorized_view_tutorial] - # [END bigquery_avt_create_shared_dataset] - # To facilitate testing, we replace values with alternatives - # provided by the testing harness. - shared_dataset_id = override_values.get("shared_dataset_id", shared_dataset_id) - # [START bigquery_authorized_view_tutorial] - # [START bigquery_avt_create_shared_dataset] - - shared_dataset = bigquery.Dataset(client.dataset(shared_dataset_id)) - shared_dataset.location = 'US' - shared_dataset = client.create_dataset(shared_dataset) # API request - # [END bigquery_avt_create_shared_dataset] - - # Create the view in the new dataset - # [START bigquery_avt_create_view] - shared_view_id = 'github_analyst_view' - view = bigquery.Table(shared_dataset.table(shared_view_id)) - sql_template = """ - SELECT - commit, author.name as author, - committer.name as committer, repo_name - FROM - `{}.{}.{}` - """ - view.view_query = sql_template.format( - client.project, source_dataset_id, source_table_id) - view = client.create_table(view) # API request - # [END bigquery_avt_create_view] - - # Assign access controls to the dataset containing the view - # [START bigquery_avt_shared_dataset_access] - # analyst_group_email = 'data_analysts@example.com' - access_entries = shared_dataset.access_entries - access_entries.append( - bigquery.AccessEntry('READER', 'groupByEmail', analyst_group_email) - ) - shared_dataset.access_entries = access_entries - shared_dataset = client.update_dataset( - shared_dataset, ['access_entries']) # API request - # [END bigquery_avt_shared_dataset_access] - - # Authorize the view to access the source dataset - # [START bigquery_avt_source_dataset_access] - access_entries = source_dataset.access_entries - access_entries.append( - bigquery.AccessEntry(None, 'view', view.reference.to_api_repr()) - ) - source_dataset.access_entries = access_entries - source_dataset = client.update_dataset( - source_dataset, ['access_entries']) # API request - # [END bigquery_avt_source_dataset_access] - # [END bigquery_authorized_view_tutorial] - - -if __name__ == '__main__': - run_authorized_view_tutorial() diff --git a/bigquery/cloud-client/authorized_view_tutorial_test.py b/bigquery/cloud-client/authorized_view_tutorial_test.py deleted file mode 100644 index 03079085f06..00000000000 --- a/bigquery/cloud-client/authorized_view_tutorial_test.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid - -from google.cloud import bigquery -import pytest - -import authorized_view_tutorial - - -@pytest.fixture(scope='module') -def client(): - return bigquery.Client() - - -@pytest.fixture -def datasets_to_delete(client): - doomed = [] - yield doomed - for item in doomed: - client.delete_dataset(item, delete_contents=True) - - -def test_authorized_view_tutorial(client, datasets_to_delete): - override_values = { - "source_dataset_id": "github_source_data_{}".format(str(uuid.uuid4()).replace("-", "_")), - "shared_dataset_id": "shared_views_{}".format(str(uuid.uuid4()).replace("-", "_")), - } - source_dataset_ref = client.dataset(override_values["source_dataset_id"]) - shared_dataset_ref = client.dataset(override_values["shared_dataset_id"]) - datasets_to_delete.extend([override_values["source_dataset_id"], - override_values["shared_dataset_id"]]) - - authorized_view_tutorial.run_authorized_view_tutorial(override_values) - - source_dataset = client.get_dataset(source_dataset_ref) - shared_dataset = client.get_dataset(shared_dataset_ref) - analyst_email = 'example-analyst-group@google.com' - analyst_entries = [entry for entry in shared_dataset.access_entries - if entry.entity_id == analyst_email] - assert len(analyst_entries) == 1 - assert analyst_entries[0].role == 'READER' - - authorized_view_entries = [entry for entry in source_dataset.access_entries - if entry.entity_type == 'view'] - expected_view_ref = { - 'projectId': client.project, - 'datasetId': override_values["shared_dataset_id"], - 'tableId': 'github_analyst_view', - } - assert len(authorized_view_entries) == 1 - assert authorized_view_entries[0].entity_id == expected_view_ref diff --git a/bigquery/cloud-client/jupyter_tutorial_test.py b/bigquery/cloud-client/jupyter_tutorial_test.py deleted file mode 100644 index 0affbabcb03..00000000000 --- a/bigquery/cloud-client/jupyter_tutorial_test.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import IPython -from IPython.terminal import interactiveshell -from IPython.testing import tools -import matplotlib -import pytest - - -# Ignore semicolon lint warning because semicolons are used in notebooks -# flake8: noqa E703 - - -@pytest.fixture(scope='session') -def ipython(): - config = tools.default_config() - config.TerminalInteractiveShell.simple_prompt = True - shell = interactiveshell.TerminalInteractiveShell.instance(config=config) - return shell - - -@pytest.fixture() -def ipython_interactive(request, ipython): - """Activate IPython's builtin hooks - - for the duration of the test scope. - """ - with ipython.builtin_trap: - yield ipython - - -def _strip_region_tags(sample_text): - """Remove blank lines and region tags from sample text""" - magic_lines = [line for line in sample_text.split('\n') - if len(line) > 0 and '# [' not in line] - return '\n'.join(magic_lines) - - -def test_jupyter_tutorial(ipython): - matplotlib.use('agg') - ip = IPython.get_ipython() - ip.extension_manager.load_extension('google.cloud.bigquery') - - sample = """ - # [START bigquery_jupyter_magic_gender_by_year] - %%bigquery - SELECT - source_year AS year, - COUNT(is_male) AS birth_count - FROM `bigquery-public-data.samples.natality` - GROUP BY year - ORDER BY year DESC - LIMIT 15 - # [END bigquery_jupyter_magic_gender_by_year] - """ - result = ip.run_cell(_strip_region_tags(sample)) - result.raise_error() # Throws an exception if the cell failed. - - sample = """ - # [START bigquery_jupyter_magic_gender_by_year_var] - %%bigquery total_births - SELECT - source_year AS year, - COUNT(is_male) AS birth_count - FROM `bigquery-public-data.samples.natality` - GROUP BY year - ORDER BY year DESC - LIMIT 15 - # [END bigquery_jupyter_magic_gender_by_year_var] - """ - result = ip.run_cell(_strip_region_tags(sample)) - result.raise_error() # Throws an exception if the cell failed. - - assert 'total_births' in ip.user_ns # verify that variable exists - total_births = ip.user_ns['total_births'] - # [START bigquery_jupyter_plot_births_by_year] - total_births.plot(kind='bar', x='year', y='birth_count'); - # [END bigquery_jupyter_plot_births_by_year] - - sample = """ - # [START bigquery_jupyter_magic_gender_by_weekday] - %%bigquery births_by_weekday - SELECT - wday, - SUM(CASE WHEN is_male THEN 1 ELSE 0 END) AS male_births, - SUM(CASE WHEN is_male THEN 0 ELSE 1 END) AS female_births - FROM `bigquery-public-data.samples.natality` - WHERE wday IS NOT NULL - GROUP BY wday - ORDER BY wday ASC - # [END bigquery_jupyter_magic_gender_by_weekday] - """ - result = ip.run_cell(_strip_region_tags(sample)) - result.raise_error() # Throws an exception if the cell failed. - - assert 'births_by_weekday' in ip.user_ns # verify that variable exists - births_by_weekday = ip.user_ns['births_by_weekday'] - # [START bigquery_jupyter_plot_births_by_weekday] - births_by_weekday.plot(x='wday'); - # [END bigquery_jupyter_plot_births_by_weekday] - - # [START bigquery_jupyter_import_and_client] - from google.cloud import bigquery - client = bigquery.Client() - # [END bigquery_jupyter_import_and_client] - - # [START bigquery_jupyter_query_plurality_by_year] - sql = """ - SELECT - plurality, - COUNT(1) AS count, - year - FROM - `bigquery-public-data.samples.natality` - WHERE - NOT IS_NAN(plurality) AND plurality > 1 - GROUP BY - plurality, year - ORDER BY - count DESC - """ - df = client.query(sql).to_dataframe() - df.head() - # [END bigquery_jupyter_query_plurality_by_year] - - # [START bigquery_jupyter_plot_plurality_by_year] - pivot_table = df.pivot(index='year', columns='plurality', values='count') - pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7)); - # [END bigquery_jupyter_plot_plurality_by_year] - - # [START bigquery_jupyter_query_births_by_gestation] - sql = """ - SELECT - gestation_weeks, - COUNT(1) AS count - FROM - `bigquery-public-data.samples.natality` - WHERE - NOT IS_NAN(gestation_weeks) AND gestation_weeks <> 99 - GROUP BY - gestation_weeks - ORDER BY - gestation_weeks - """ - df = client.query(sql).to_dataframe() - # [END bigquery_jupyter_query_births_by_gestation] - - # [START bigquery_jupyter_plot_births_by_gestation] - ax = df.plot(kind='bar', x='gestation_weeks', y='count', figsize=(15,7)) - ax.set_title('Count of Births by Gestation Weeks') - ax.set_xlabel('Gestation Weeks') - ax.set_ylabel('Count'); - # [END bigquery_jupyter_plot_births_by_gestation] diff --git a/bigquery/cloud-client/natality_tutorial.py b/bigquery/cloud-client/natality_tutorial.py deleted file mode 100644 index 6a097add3b0..00000000000 --- a/bigquery/cloud-client/natality_tutorial.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def run_natality_tutorial(override_values={}): - # [START bigquery_query_natality_tutorial] - """Create a Google BigQuery linear regression input table. - - In the code below, the following actions are taken: - * A new dataset is created "natality_regression." - * A query is run against the public dataset, - bigquery-public-data.samples.natality, selecting only the data of - interest to the regression, the output of which is stored in a new - "regression_input" table. - * The output table is moved over the wire to the user's default project via - the built-in BigQuery Connector for Spark that bridges BigQuery and - Cloud Dataproc. - """ - - from google.cloud import bigquery - - # Create a new Google BigQuery client using Google Cloud Platform project - # defaults. - client = bigquery.Client() - - # Prepare a reference to a new dataset for storing the query results. - dataset_id = 'natality_regression' - # [END bigquery_query_natality_tutorial] - # To facilitate testing, we replace values with alternatives - # provided by the testing harness. - dataset_id = override_values.get("dataset_id", dataset_id) - # [START bigquery_query_natality_tutorial] - - dataset = bigquery.Dataset(client.dataset(dataset_id)) - - # Create the new BigQuery dataset. - dataset = client.create_dataset(dataset) - - # In the new BigQuery dataset, create a reference to a new table for - # storing the query results. - table_ref = dataset.table('regression_input') - - # Configure the query job. - job_config = bigquery.QueryJobConfig() - - # Set the destination table to the table reference created above. - job_config.destination = table_ref - - # Set up a query in Standard SQL, which is the default for the BigQuery - # Python client library. - # The query selects the fields of interest. - query = """ - SELECT - weight_pounds, mother_age, father_age, gestation_weeks, - weight_gain_pounds, apgar_5min - FROM - `bigquery-public-data.samples.natality` - WHERE - weight_pounds IS NOT NULL - AND mother_age IS NOT NULL - AND father_age IS NOT NULL - AND gestation_weeks IS NOT NULL - AND weight_gain_pounds IS NOT NULL - AND apgar_5min IS NOT NULL - """ - - # Run the query. - query_job = client.query(query, job_config=job_config) - query_job.result() # Waits for the query to finish - # [END bigquery_query_natality_tutorial] - - -if __name__ == '__main__': - run_natality_tutorial() diff --git a/bigquery/cloud-client/natality_tutorial_test.py b/bigquery/cloud-client/natality_tutorial_test.py deleted file mode 100644 index 785df59df81..00000000000 --- a/bigquery/cloud-client/natality_tutorial_test.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid - -from google.cloud import bigquery -import pytest - -import natality_tutorial - - -@pytest.fixture(scope='module') -def client(): - return bigquery.Client() - - -@pytest.fixture -def datasets_to_delete(client): - doomed = [] - yield doomed - for item in doomed: - client.delete_dataset(item, delete_contents=True) - - -def test_natality_tutorial(client, datasets_to_delete): - override_values = { - "dataset_id": "natality_regression_{}".format(str(uuid.uuid4()).replace("-", "_")), - } - datasets_to_delete.append(override_values["dataset_id"]) - - natality_tutorial.run_natality_tutorial(override_values) - - table_ref = bigquery.Dataset(client.dataset(override_values["dataset_id"])).table("regression_input") - table = client.get_table(table_ref) - assert table.num_rows > 0 diff --git a/bigquery/cloud-client/noxfile_config.py b/bigquery/cloud-client/noxfile_config.py deleted file mode 100644 index cfd0d439150..00000000000 --- a/bigquery/cloud-client/noxfile_config.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# You can copy this file into your directory, then it will be inported from -# the noxfile.py. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - 'envs': {}, -} diff --git a/bigquery/cloud-client/quickstart.py b/bigquery/cloud-client/quickstart.py deleted file mode 100644 index cb6dcc303df..00000000000 --- a/bigquery/cloud-client/quickstart.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def run_quickstart(override_values={}): - # [START bigquery_quickstart] - # Imports the Google Cloud client library - from google.cloud import bigquery - - # Instantiates a client - bigquery_client = bigquery.Client() - - # The name for the new dataset - dataset_id = 'my_new_dataset' - - # [END bigquery_quickstart] - # To facilitate testing, we replace values with alternatives - # provided by the testing harness. - dataset_id = override_values.get("dataset_id", dataset_id) - # [START bigquery_quickstart] - - # Prepares a reference to the new dataset - dataset_ref = bigquery_client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - - # Creates the new dataset - dataset = bigquery_client.create_dataset(dataset) - - print('Dataset {} created.'.format(dataset.dataset_id)) - # [END bigquery_quickstart] - - -if __name__ == '__main__': - run_quickstart() diff --git a/bigquery/cloud-client/quickstart_test.py b/bigquery/cloud-client/quickstart_test.py deleted file mode 100644 index 2b461a8f27b..00000000000 --- a/bigquery/cloud-client/quickstart_test.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid - -from google.cloud import bigquery -import pytest - -import quickstart - - -# Must match the dataset listed in quickstart.py (there's no easy way to -# extract this). -DATASET_ID = 'my_new_dataset' - - -@pytest.fixture(scope='module') -def client(): - return bigquery.Client() - - -@pytest.fixture -def datasets_to_delete(client): - doomed = [] - yield doomed - for item in doomed: - client.delete_dataset(item, delete_contents=True) - - -def test_quickstart(capsys, client, datasets_to_delete): - - override_values = { - "dataset_id": "my_new_dataset_{}".format(str(uuid.uuid4()).replace("-", "_")), - } - datasets_to_delete.append(override_values["dataset_id"]) - - quickstart.run_quickstart(override_values) - out, _ = capsys.readouterr() - assert override_values["dataset_id"] in out diff --git a/bigquery/cloud-client/requirements-test.txt b/bigquery/cloud-client/requirements-test.txt deleted file mode 100644 index 676ff949e8a..00000000000 --- a/bigquery/cloud-client/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==5.4.3 -mock==4.0.2 diff --git a/bigquery/cloud-client/requirements.txt b/bigquery/cloud-client/requirements.txt deleted file mode 100644 index fe0f79e2139..00000000000 --- a/bigquery/cloud-client/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -google-cloud-bigquery[pandas]==1.25.0 -google-auth-oauthlib==0.4.1 -ipython==7.16.1 -matplotlib==3.3.0 -pytz==2020.1 diff --git a/bigquery/cloud-client/simple_app.py b/bigquery/cloud-client/simple_app.py deleted file mode 100644 index a09e97f1246..00000000000 --- a/bigquery/cloud-client/simple_app.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Simple application that performs a query with BigQuery.""" -# [START bigquery_simple_app_all] -# [START bigquery_simple_app_deps] -from google.cloud import bigquery -# [END bigquery_simple_app_deps] - - -def query_stackoverflow(): - # [START bigquery_simple_app_client] - client = bigquery.Client() - # [END bigquery_simple_app_client] - # [START bigquery_simple_app_query] - query_job = client.query(""" - SELECT - CONCAT( - 'https://stackoverflow.com/questions/', - CAST(id as STRING)) as url, - view_count - FROM `bigquery-public-data.stackoverflow.posts_questions` - WHERE tags like '%google-bigquery%' - ORDER BY view_count DESC - LIMIT 10""") - - results = query_job.result() # Waits for job to complete. - # [END bigquery_simple_app_query] - - # [START bigquery_simple_app_print] - for row in results: - print("{} : {} views".format(row.url, row.view_count)) - # [END bigquery_simple_app_print] - - -if __name__ == '__main__': - query_stackoverflow() -# [END bigquery_simple_app_all] diff --git a/bigquery/cloud-client/simple_app_test.py b/bigquery/cloud-client/simple_app_test.py deleted file mode 100644 index 33f9f1adf69..00000000000 --- a/bigquery/cloud-client/simple_app_test.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import simple_app - - -def test_query_stackoverflow(capsys): - simple_app.query_stackoverflow() - out, _ = capsys.readouterr() - assert 'views' in out diff --git a/bigquery/cloud-client/user_credentials.py b/bigquery/cloud-client/user_credentials.py deleted file mode 100644 index 4917fdd3a41..00000000000 --- a/bigquery/cloud-client/user_credentials.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to run a query using user credentials. - -You must supply a client secrets file, which would normally be bundled with -your application. -""" - -import argparse - - -def main(project, launch_browser=True): - # [START bigquery_auth_user_flow] - from google_auth_oauthlib import flow - - # TODO: Uncomment the line below to set the `launch_browser` variable. - # launch_browser = True - # - # The `launch_browser` boolean variable indicates if a local server is used - # as the callback URL in the auth flow. A value of `True` is recommended, - # but a local server does not work if accessing the application remotely, - # such as over SSH or from a remote Jupyter notebook. - - appflow = flow.InstalledAppFlow.from_client_secrets_file( - 'client_secrets.json', - scopes=['https://www.googleapis.com/auth/bigquery']) - - if launch_browser: - appflow.run_local_server() - else: - appflow.run_console() - - credentials = appflow.credentials - # [END bigquery_auth_user_flow] - - # [START bigquery_auth_user_query] - from google.cloud import bigquery - - # TODO: Uncomment the line below to set the `project` variable. - # project = 'user-project-id' - # - # The `project` variable defines the project to be billed for query - # processing. The user must have the bigquery.jobs.create permission on - # this project to run a query. See: - # https://cloud.google.com/bigquery/docs/access-control#permissions - - client = bigquery.Client(project=project, credentials=credentials) - - query_string = """SELECT name, SUM(number) as total - FROM `bigquery-public-data.usa_names.usa_1910_current` - WHERE name = 'William' - GROUP BY name; - """ - query_job = client.query(query_string) - - # Print the results. - for row in query_job.result(): # Wait for the job to complete. - print("{}: {}".format(row['name'], row['total'])) - # [END bigquery_auth_user_query] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--launch-browser', - help='Use a local server flow to authenticate. ', - action='store_true') - parser.add_argument('project', help='Project to use for BigQuery billing.') - - args = parser.parse_args() - - main( - args.project, launch_browser=args.launch_browser) diff --git a/bigquery/cloud-client/user_credentials_test.py b/bigquery/cloud-client/user_credentials_test.py deleted file mode 100644 index 494903c75a1..00000000000 --- a/bigquery/cloud-client/user_credentials_test.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import google.auth -import mock -import pytest - -from user_credentials import main - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] - - -@pytest.fixture -def mock_flow(): - flow_patch = mock.patch( - 'google_auth_oauthlib.flow.InstalledAppFlow', autospec=True) - - with flow_patch as flow_mock: - flow_mock.from_client_secrets_file.return_value = flow_mock - flow_mock.credentials = google.auth.default()[0] - yield flow_mock - - -def test_auth_query_console(mock_flow, capsys): - main(PROJECT, launch_browser=False) - out, _ = capsys.readouterr() - # Fun fact: William P. Wood was the 1st director of the US Secret Service. - assert 'William' in out