Skip to content

Commit 85176d4

Browse files
dpebotdpebot
authored
Merge pull request #467 from GoogleCloudPlatform/bigquery-cloud-client-samples
2 parents 5d1f8c8 + a2d91fd commit 85176d4

13 files changed

+681
-0
lines changed

‎bigquery/cloud-client/export_data_to_gcs.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2016 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Exports data from BigQuery to an object in Google Cloud Storage.
18+
19+
For more information, see the README.md under /bigquery.
20+
21+
Example invocation:
22+
$ python export_data_to_gcs.py example_dataset example_table \
23+
gs://example-bucket/example-data.csv
24+
25+
The dataset and table should already exist.
26+
"""
27+
28+
import argparse
29+
import time
30+
import uuid
31+
32+
from gcloud import bigquery
33+
34+
35+
def export_data_to_gcs(dataset_name, table_name, destination):
36+
bigquery_client = bigquery.Client()
37+
dataset = bigquery_client.dataset(dataset_name)
38+
table = dataset.table(table_name)
39+
job_name = str(uuid.uuid4())
40+
41+
job = bigquery_client.extract_table_to_storage(
42+
job_name, table, destination)
43+
44+
job.begin()
45+
46+
wait_for_job(job)
47+
48+
print('Exported {}:{} to {}'.format(
49+
dataset_name, table_name, destination))
50+
51+
52+
def wait_for_job(job):
53+
while True:
54+
job.reload()
55+
if job.state == 'DONE':
56+
if job.error_result:
57+
raise RuntimeError(job.error_result)
58+
return
59+
time.sleep(1)
60+
61+
62+
if __name__ == '__main__':
63+
parser = argparse.ArgumentParser(
64+
description=__doc__,
65+
formatter_class=argparse.RawDescriptionHelpFormatter)
66+
parser.add_argument('dataset_name')
67+
parser.add_argument('table_name')
68+
parser.add_argument(
69+
'destination', help='The desintation Google Cloud Storage object.'
70+
'Must be in the format gs://bucket_name/object_name')
71+
72+
args = parser.parse_args()
73+
74+
export_data_to_gcs(
75+
args.dataset_name,
76+
args.table_name,
77+
args.destination)

‎bigquery/cloud-client/export_data_to_gcs_test.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
import export_data_to_gcs
15+
16+
17+
DATASET_ID = 'test_dataset'
18+
TABLE_ID = 'test_import_table'
19+
20+
21+
def test_export_data_to_gcs(cloud_config, capsys):
22+
export_data_to_gcs.export_data_to_gcs(
23+
DATASET_ID,
24+
TABLE_ID,
25+
'gs://{}/test-export-data-to-gcs.csv'.format(
26+
cloud_config.storage_bucket))
27+
28+
out, _ = capsys.readouterr()
29+
30+
assert 'Exported' in out

‎bigquery/cloud-client/load_data_from_file.py

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2016 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Loads data into BigQuery from a local file.
18+
19+
For more information, see the README.md under /bigquery.
20+
21+
Example invocation:
22+
$ python load_data_from_file.py example_dataset example_table \
23+
example-data.csv
24+
25+
The dataset and table should already exist.
26+
"""
27+
28+
import argparse
29+
import time
30+
from gcloud import bigquery
31+
32+
33+
def load_data_from_file(dataset_name, table_name, source_file_name):
34+
bigquery_client = bigquery.Client()
35+
dataset = bigquery_client.dataset(dataset_name)
36+
table = dataset.table(table_name)
37+
38+
# Reload the table to get the schema.
39+
table.reload()
40+
41+
with open(source_file_name, 'rb') as source_file:
42+
# This example uses CSV, but you can use other formats.
43+
# See https://cloud.google.com/bigquery/loading-data
44+
job = table.upload_from_file(
45+
source_file, source_format='text/csv')
46+
47+
job.begin()
48+
49+
wait_for_job(job)
50+
51+
print('Loaded {} rows into {}:{}.'.format(
52+
job.output_rows, dataset_name, table_name))
53+
54+
55+
def wait_for_job(job):
56+
while True:
57+
job.reload()
58+
if job.state == 'DONE':
59+
if job.error_result:
60+
raise RuntimeError(job.error_result)
61+
return
62+
time.sleep(1)
63+
64+
65+
if __name__ == '__main__':
66+
parser = argparse.ArgumentParser(
67+
description=__doc__,
68+
formatter_class=argparse.RawDescriptionHelpFormatter)
69+
parser.add_argument('dataset_name')
70+
parser.add_argument('table_name')
71+
parser.add_argument(
72+
'source_file_name', help='Path to a .csv file to upload.')
73+
74+
args = parser.parse_args()
75+
76+
load_data_from_file(
77+
args.dataset_name,
78+
args.table_name,
79+
args.source_file_name)

‎bigquery/cloud-client/load_data_from_file_test.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
import pytest
15+
16+
import load_data_from_file
17+
18+
DATASET_ID = 'test_dataset'
19+
TABLE_ID = 'test_import_table'
20+
21+
22+
@pytest.mark.xfail(
23+
strict=True,
24+
reason='https://github.com/GoogleCloudPlatform/gcloud-python/issues/2133')
25+
def test_load_table(resource, capsys):
26+
data_path = resource('data.csv')
27+
28+
load_data_from_file.load_data_from_file(
29+
DATASET_ID,
30+
TABLE_ID,
31+
data_path)
32+
33+
out, _ = capsys.readouterr()
34+
35+
assert 'Loaded 1 rows' in out

‎bigquery/cloud-client/load_data_from_gcs.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2016 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Loads data into BigQuery from an object in Google Cloud Storage.
18+
19+
For more information, see the README.md under /bigquery.
20+
21+
Example invocation:
22+
$ python load_data_from_gcs.py example_dataset example_table \
23+
gs://example-bucket/example-data.csv
24+
25+
The dataset and table should already exist.
26+
"""
27+
28+
import argparse
29+
import time
30+
import uuid
31+
32+
from gcloud import bigquery
33+
34+
35+
def load_data_from_gcs(dataset_name, table_name, source):
36+
bigquery_client = bigquery.Client()
37+
dataset = bigquery_client.dataset(dataset_name)
38+
table = dataset.table(table_name)
39+
job_name = str(uuid.uuid4())
40+
41+
job = bigquery_client.load_table_from_storage(
42+
job_name, table, source)
43+
44+
job.begin()
45+
46+
wait_for_job(job)
47+
48+
print('Loaded {} rows into {}:{}.'.format(
49+
job.output_rows, dataset_name, table_name))
50+
51+
52+
def wait_for_job(job):
53+
while True:
54+
job.reload()
55+
if job.state == 'DONE':
56+
if job.error_result:
57+
raise RuntimeError(job.error_result)
58+
return
59+
time.sleep(1)
60+
61+
62+
if __name__ == '__main__':
63+
parser = argparse.ArgumentParser(
64+
description=__doc__,
65+
formatter_class=argparse.RawDescriptionHelpFormatter)
66+
parser.add_argument('dataset_name')
67+
parser.add_argument('table_name')
68+
parser.add_argument(
69+
'source', help='The Google Cloud Storage object to load. Must be in '
70+
'the format gs://bucket_name/object_name')
71+
72+
args = parser.parse_args()
73+
74+
load_data_from_gcs(
75+
args.dataset_name,
76+
args.table_name,
77+
args.source)

‎bigquery/cloud-client/load_data_from_gcs_test.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
import load_data_from_gcs
15+
16+
DATASET_ID = 'test_dataset'
17+
TABLE_ID = 'test_import_table'
18+
19+
20+
def test_load_table(cloud_config, capsys):
21+
cloud_storage_input_uri = 'gs://{}/data.csv'.format(
22+
cloud_config.storage_bucket)
23+
24+
load_data_from_gcs.load_data_from_gcs(
25+
DATASET_ID,
26+
TABLE_ID,
27+
cloud_storage_input_uri)
28+
29+
out, _ = capsys.readouterr()
30+
31+
assert 'Loaded 1 rows' in out

‎bigquery/cloud-client/resources/data.csv

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Gandalf, 2000, 140.0, 1

‎bigquery/cloud-client/resources/data.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"Name": "Gandalf", "Age": 2000, "Weight": 140.0, "IsMagic": true}

‎bigquery/cloud-client/resources/schema.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}]

0 commit comments

Comments
 (0)