From 4f698f998b728deb8ff476549651006fb9133aa9 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Mon, 6 Nov 2017 18:07:47 -0800 Subject: [PATCH 01/10] Fixed Failed Kokoro Test (Dataproc) --- dataproc/dataproc_e2e_test.py | 6 +----- dataproc/submit_job_to_cluster.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py index d7e9c522074..b83b2b6de26 100644 --- a/dataproc/dataproc_e2e_test.py +++ b/dataproc/dataproc_e2e_test.py @@ -18,17 +18,13 @@ import os -from gcp_devrel.testing.flaky import flaky - import submit_job_to_cluster PROJECT = os.environ['GCLOUD_PROJECT'] BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -CLUSTER_NAME = 'testcluster2' +CLUSTER_NAME = 'testcluster3' ZONE = 'us-central1-b' - -@flaky def test_e2e(): output = submit_job_to_cluster.main( PROJECT, ZONE, CLUSTER_NAME, BUCKET) diff --git a/dataproc/submit_job_to_cluster.py b/dataproc/submit_job_to_cluster.py index 3ffde240ce6..ab833f02fcb 100644 --- a/dataproc/submit_job_to_cluster.py +++ b/dataproc/submit_job_to_cluster.py @@ -25,12 +25,12 @@ def get_default_pyspark_file(): """Gets the PySpark file from this directory""" current_dir = os.path.dirname(os.path.abspath(__file__)) - f = open(os.path.join(current_dir, DEFAULT_FILENAME), 'r') + f = open(os.path.join(current_dir, DEFAULT_FILENAME), 'rb') return f, DEFAULT_FILENAME def get_pyspark_file(filename): - f = open(filename, 'r') + f = open(filename, 'rb') return f, os.path.basename(filename) @@ -76,6 +76,14 @@ def create_cluster(dataproc, project, zone, region, cluster_name): 'config': { 'gceClusterConfig': { 'zoneUri': zone_uri + }, + 'masterConfig': { + 'numInstances': 1, + 'machineTypeUri': 'n1-standard-1' + }, + 'workerConfig': { + 'numInstances' : 2, + 'machineTypeUri': 'n1-standard-1' } } } From 06d05d5886bd2290f2b93b712a5362949d1cb718 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Mon, 6 Nov 2017 18:21:20 -0800 Subject: [PATCH 02/10] Fixed Lint Error --- dataproc/submit_job_to_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataproc/submit_job_to_cluster.py b/dataproc/submit_job_to_cluster.py index ab833f02fcb..1815078202f 100644 --- a/dataproc/submit_job_to_cluster.py +++ b/dataproc/submit_job_to_cluster.py @@ -82,7 +82,7 @@ def create_cluster(dataproc, project, zone, region, cluster_name): 'machineTypeUri': 'n1-standard-1' }, 'workerConfig': { - 'numInstances' : 2, + 'numInstances': 2, 'machineTypeUri': 'n1-standard-1' } } From 40b0042d812714ad76b7a82afccc66e4efe00945 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Mon, 6 Nov 2017 18:25:57 -0800 Subject: [PATCH 03/10] Update dataproc_e2e_test.py --- dataproc/dataproc_e2e_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py index b83b2b6de26..2b4ae662534 100644 --- a/dataproc/dataproc_e2e_test.py +++ b/dataproc/dataproc_e2e_test.py @@ -20,11 +20,14 @@ import submit_job_to_cluster +from gcp_devrel.testing.flaky import flaky + PROJECT = os.environ['GCLOUD_PROJECT'] BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] CLUSTER_NAME = 'testcluster3' ZONE = 'us-central1-b' +@flaky def test_e2e(): output = submit_job_to_cluster.main( PROJECT, ZONE, CLUSTER_NAME, BUCKET) From 0755aa65d0686d4203083a6ad5fafb8df05c4c6f Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Mon, 6 Nov 2017 18:26:30 -0800 Subject: [PATCH 04/10] Update dataproc_e2e_test.py --- dataproc/dataproc_e2e_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py index 2b4ae662534..9f7ba4cf5d5 100644 --- a/dataproc/dataproc_e2e_test.py +++ b/dataproc/dataproc_e2e_test.py @@ -18,10 +18,10 @@ import os -import submit_job_to_cluster - from gcp_devrel.testing.flaky import flaky +import submit_job_to_cluster + PROJECT = os.environ['GCLOUD_PROJECT'] BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] CLUSTER_NAME = 'testcluster3' From ac096571a6570618d4bdcd06e2b9e0c73fb4593f Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Mon, 6 Nov 2017 18:31:13 -0800 Subject: [PATCH 05/10] Fixing More Lint Errors --- dataproc/dataproc_e2e_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py index 9f7ba4cf5d5..0a45d080122 100644 --- a/dataproc/dataproc_e2e_test.py +++ b/dataproc/dataproc_e2e_test.py @@ -18,8 +18,6 @@ import os -from gcp_devrel.testing.flaky import flaky - import submit_job_to_cluster PROJECT = os.environ['GCLOUD_PROJECT'] @@ -27,7 +25,7 @@ CLUSTER_NAME = 'testcluster3' ZONE = 'us-central1-b' -@flaky + def test_e2e(): output = submit_job_to_cluster.main( PROJECT, ZONE, CLUSTER_NAME, BUCKET) From cd1dbfd25997a154a8a85cc754cc2a85b18a63c4 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 7 Nov 2017 13:51:01 -0800 Subject: [PATCH 06/10] Fixed b/65407087 --- appengine/standard/flask/hello_world/README.md | 5 ----- appengine/standard/flask/hello_world/app.yaml | 5 +++++ .../flask/hello_world/appengine_config.py | 18 ------------------ .../flask/hello_world/requirements.txt | 1 - testing/requirements.txt | 1 + 5 files changed, 6 insertions(+), 24 deletions(-) delete mode 100644 appengine/standard/flask/hello_world/appengine_config.py delete mode 100644 appengine/standard/flask/hello_world/requirements.txt diff --git a/appengine/standard/flask/hello_world/README.md b/appengine/standard/flask/hello_world/README.md index 04ec9402ab1..e2749c87e56 100644 --- a/appengine/standard/flask/hello_world/README.md +++ b/appengine/standard/flask/hello_world/README.md @@ -3,9 +3,4 @@ This sample shows how to use [Flask](http://flask.pocoo.org/) with Google App Engine Standard. -Before running or deploying this application, install the dependencies using -[pip](http://pip.readthedocs.io/en/stable/): - - pip install -t lib -r requirements.txt - For more information, see the [App Engine Standard README](../../README.md) diff --git a/appengine/standard/flask/hello_world/app.yaml b/appengine/standard/flask/hello_world/app.yaml index f041d384c05..c8d42601d93 100644 --- a/appengine/standard/flask/hello_world/app.yaml +++ b/appengine/standard/flask/hello_world/app.yaml @@ -5,3 +5,8 @@ threadsafe: true handlers: - url: /.* script: main.app + +# Using flask as a built-in third-party library +libraries: +- name: flask + version: latest diff --git a/appengine/standard/flask/hello_world/appengine_config.py b/appengine/standard/flask/hello_world/appengine_config.py deleted file mode 100644 index c903d9a0ac5..00000000000 --- a/appengine/standard/flask/hello_world/appengine_config.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from google.appengine.ext import vendor - -# Add any libraries installed in the "lib" folder. -vendor.add('lib') diff --git a/appengine/standard/flask/hello_world/requirements.txt b/appengine/standard/flask/hello_world/requirements.txt deleted file mode 100644 index 52ab1a39019..00000000000 --- a/appengine/standard/flask/hello_world/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Flask==0.12.2 diff --git a/testing/requirements.txt b/testing/requirements.txt index e51938b0d19..8ba942671a9 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -15,3 +15,4 @@ webapp2==2.5.2 google-api-python-client==1.6.4 google-cloud-core==0.28.0 gcp-devrel-py-tools==0.0.11 +flask==0.12 From 9179823a52d20d1247703d1a0a358b65c1d813c7 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 7 Nov 2017 16:00:39 -0800 Subject: [PATCH 07/10] Revert "Merge branch 'master' of https://github.com/michaelawyu/python-docs-samples" This reverts commit 1614c7d3ef33630a8ab095792b27fc25fd91f0ad, reversing changes made to cd1dbfd25997a154a8a85cc754cc2a85b18a63c4. --- dataproc/dataproc_e2e_test.py | 1 - dataproc/submit_job_to_cluster.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py index 0a45d080122..b83b2b6de26 100644 --- a/dataproc/dataproc_e2e_test.py +++ b/dataproc/dataproc_e2e_test.py @@ -25,7 +25,6 @@ CLUSTER_NAME = 'testcluster3' ZONE = 'us-central1-b' - def test_e2e(): output = submit_job_to_cluster.main( PROJECT, ZONE, CLUSTER_NAME, BUCKET) diff --git a/dataproc/submit_job_to_cluster.py b/dataproc/submit_job_to_cluster.py index 1815078202f..ab833f02fcb 100644 --- a/dataproc/submit_job_to_cluster.py +++ b/dataproc/submit_job_to_cluster.py @@ -82,7 +82,7 @@ def create_cluster(dataproc, project, zone, region, cluster_name): 'machineTypeUri': 'n1-standard-1' }, 'workerConfig': { - 'numInstances': 2, + 'numInstances' : 2, 'machineTypeUri': 'n1-standard-1' } } From 4e29f7174e003aca099870d06f13ee342fe50ef7 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 7 Nov 2017 16:00:44 -0800 Subject: [PATCH 08/10] Revert "Fixed b/65407087" This reverts commit cd1dbfd25997a154a8a85cc754cc2a85b18a63c4. --- appengine/standard/flask/hello_world/README.md | 5 +++++ appengine/standard/flask/hello_world/app.yaml | 5 ----- .../flask/hello_world/appengine_config.py | 18 ++++++++++++++++++ .../flask/hello_world/requirements.txt | 1 + testing/requirements.txt | 1 - 5 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 appengine/standard/flask/hello_world/appengine_config.py create mode 100644 appengine/standard/flask/hello_world/requirements.txt diff --git a/appengine/standard/flask/hello_world/README.md b/appengine/standard/flask/hello_world/README.md index e2749c87e56..04ec9402ab1 100644 --- a/appengine/standard/flask/hello_world/README.md +++ b/appengine/standard/flask/hello_world/README.md @@ -3,4 +3,9 @@ This sample shows how to use [Flask](http://flask.pocoo.org/) with Google App Engine Standard. +Before running or deploying this application, install the dependencies using +[pip](http://pip.readthedocs.io/en/stable/): + + pip install -t lib -r requirements.txt + For more information, see the [App Engine Standard README](../../README.md) diff --git a/appengine/standard/flask/hello_world/app.yaml b/appengine/standard/flask/hello_world/app.yaml index c8d42601d93..f041d384c05 100644 --- a/appengine/standard/flask/hello_world/app.yaml +++ b/appengine/standard/flask/hello_world/app.yaml @@ -5,8 +5,3 @@ threadsafe: true handlers: - url: /.* script: main.app - -# Using flask as a built-in third-party library -libraries: -- name: flask - version: latest diff --git a/appengine/standard/flask/hello_world/appengine_config.py b/appengine/standard/flask/hello_world/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/flask/hello_world/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/flask/hello_world/requirements.txt b/appengine/standard/flask/hello_world/requirements.txt new file mode 100644 index 00000000000..52ab1a39019 --- /dev/null +++ b/appengine/standard/flask/hello_world/requirements.txt @@ -0,0 +1 @@ +Flask==0.12.2 diff --git a/testing/requirements.txt b/testing/requirements.txt index 8ba942671a9..e51938b0d19 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -15,4 +15,3 @@ webapp2==2.5.2 google-api-python-client==1.6.4 google-cloud-core==0.28.0 gcp-devrel-py-tools==0.0.11 -flask==0.12 From 0d16b4e93744a4c4a53045981f0dfaac85e5f7e9 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 7 Nov 2017 16:05:42 -0800 Subject: [PATCH 09/10] Fixed Lint Error --- dataproc/submit_job_to_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataproc/submit_job_to_cluster.py b/dataproc/submit_job_to_cluster.py index ab833f02fcb..1815078202f 100644 --- a/dataproc/submit_job_to_cluster.py +++ b/dataproc/submit_job_to_cluster.py @@ -82,7 +82,7 @@ def create_cluster(dataproc, project, zone, region, cluster_name): 'machineTypeUri': 'n1-standard-1' }, 'workerConfig': { - 'numInstances' : 2, + 'numInstances': 2, 'machineTypeUri': 'n1-standard-1' } } From b085a9949bbb0d1173f08e5eb5e3e38b9cd85e1e Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 7 Nov 2017 16:11:15 -0800 Subject: [PATCH 10/10] Fixed Lint Error --- dataproc/dataproc_e2e_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py index b83b2b6de26..0a45d080122 100644 --- a/dataproc/dataproc_e2e_test.py +++ b/dataproc/dataproc_e2e_test.py @@ -25,6 +25,7 @@ CLUSTER_NAME = 'testcluster3' ZONE = 'us-central1-b' + def test_e2e(): output = submit_job_to_cluster.main( PROJECT, ZONE, CLUSTER_NAME, BUCKET)