From e4cea3b05dbca3fdcc8bccefe87458fd11b1196d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 20 Aug 2018 17:04:37 -0700 Subject: [PATCH 1/2] Composer: Move trigger response DAG to GitHub. Sample originally was posted at https://cloud.google.com/composer/docs/how-to/using/triggering-with-gcf#wzxhzdk15gcs_response_dagpywzxhzdk16 --- composer/workflows/trigger_response_dag.py | 45 +++++++++++++++++++ .../workflows/trigger_response_dag_test.py | 23 ++++++++++ 2 files changed, 68 insertions(+) create mode 100644 composer/workflows/trigger_response_dag.py create mode 100644 composer/workflows/trigger_response_dag_test.py diff --git a/composer/workflows/trigger_response_dag.py b/composer/workflows/trigger_response_dag.py new file mode 100644 index 00000000000..dd933c54404 --- /dev/null +++ b/composer/workflows/trigger_response_dag.py @@ -0,0 +1,45 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DAG running in response to a Cloud Storage bucket change.""" + +# [START composer_trigger_response_dag] +import datetime + +import airflow +from airflow.operators import bash_operator + + +default_args = { + 'owner': 'Composer Example', + 'depends_on_past': False, + 'email': [''], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': datetime.timedelta(minutes=5), + 'start_date': datetime.datetime(2017, 1, 1), +} + +with airflow.DAG( + 'composer_sample_trigger_response_dag', + default_args=default_args, + # Not scheduled, trigger only + schedule_interval=None) as dag: + + # Print the dag_run's configuration, which includes information about the + # Cloud Storage object change. + print_gcs_info = bash_operator.BashOperator( + task_id='print_gcs_info', bash_command='echo {{ dag_run.conf }}') +# [END composer_trigger_response_dag] diff --git a/composer/workflows/trigger_response_dag_test.py b/composer/workflows/trigger_response_dag_test.py new file mode 100644 index 00000000000..441c5771d06 --- /dev/null +++ b/composer/workflows/trigger_response_dag_test.py @@ -0,0 +1,23 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import trigger_response_dag # noqa From 9252acf4b40eb027b68c1457d7c29e008e13da84 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 21 Aug 2018 11:27:57 -0700 Subject: [PATCH 2/2] Composer: Add snippet to get the client ID for a composer env. --- composer/rest/get_client_id.py | 70 +++++++++++++++++++++++++++++ composer/rest/get_client_id_test.py | 28 ++++++++++++ composer/rest/requirements.txt | 3 +- 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 composer/rest/get_client_id.py create mode 100644 composer/rest/get_client_id_test.py diff --git a/composer/rest/get_client_id.py b/composer/rest/get_client_id.py new file mode 100644 index 00000000000..789a1a9c093 --- /dev/null +++ b/composer/rest/get_client_id.py @@ -0,0 +1,70 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Get the client ID associated with a Cloud Composer environment.""" + +import argparse + + +def get_client_id(project_id, location, composer_environment): + # [START composer_get_environment_client_id] + import google.auth + import google.auth.transport.requests + import requests + import six.moves.urllib.parse + + # Authenticate with Google Cloud. + # See: https://cloud.google.com/docs/authentication/getting-started + credentials, _ = google.auth.default( + scopes=['https://www.googleapis.com/auth/cloud-platform']) + authed_session = google.auth.transport.requests.AuthorizedSession( + credentials) + + # project_id = 'YOUR_PROJECT_ID' + # location = 'us-central1' + # composer_environment = 'YOUR_COMPOSER_ENVIRONMENT_NAME' + + environment_url = ( + 'https://composer.googleapis.com/v1beta1/projects/{}/locations/{}' + '/environments/{}').format(project_id, location, composer_environment) + composer_response = authed_session.request('GET', environment_url) + environment_data = composer_response.json() + airflow_uri = environment_data['config']['airflowUri'] + + # The Composer environment response does not include the IAP client ID. + # Make a second, unauthenticated HTTP request to the web server to get the + # redirect URI. + redirect_response = requests.get(airflow_uri, allow_redirects=False) + redirect_location = redirect_response.headers['location'] + + # Extract the client_id query parameter from the redirect. + parsed = six.moves.urllib.parse.urlparse(redirect_location) + query_string = six.moves.urllib.parse.parse_qs(parsed.query) + print(query_string['client_id'][0]) + # [END composer_get_environment_client_id] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project_id', help='Your Project ID.') + parser.add_argument( + 'location', help='Region of the Cloud Composer environent.') + parser.add_argument( + 'composer_environment', help='Name of the Cloud Composer environent.') + + args = parser.parse_args() + get_client_id( + args.project_id, args.location, args.composer_environment) diff --git a/composer/rest/get_client_id_test.py b/composer/rest/get_client_id_test.py new file mode 100644 index 00000000000..52ac8c5f380 --- /dev/null +++ b/composer/rest/get_client_id_test.py @@ -0,0 +1,28 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .get_client_id import get_client_id + + +PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] +COMPOSER_LOCATION = os.environ['COMPOSER_LOCATION'] +COMPOSER_ENVIRONMENT = os.environ['COMPOSER_ENVIRONMENT'] + + +def test_get_client_id(capsys): + get_client_id(PROJECT, COMPOSER_LOCATION, COMPOSER_ENVIRONMENT) + out, _ = capsys.readouterr() + assert '.apps.googleusercontent.com' in out diff --git a/composer/rest/requirements.txt b/composer/rest/requirements.txt index 05a09e88552..7a9184558c8 100644 --- a/composer/rest/requirements.txt +++ b/composer/rest/requirements.txt @@ -1,2 +1,3 @@ google-auth==1.4.1 -requests==2.18.4 \ No newline at end of file +requests==2.18.4 +six==1.11.0 \ No newline at end of file