TST: configure CircleCI (#232)

tswast · web-flow · commit 3b865c616fd7 · 2018-11-05T09:44:22.000-08:00
Use CircleCI for testing pandas-gbq. This configuration uses the
canonical Python and miniconda3 Docker images to reduce problems caused
by a unique configuration.

Also:

* Re-enables conda tests.
* Adds Python 3.7 tests.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,75 @@
+version: 2
+jobs:
+  # Pip
+  "pip-2.7-0.19.2":
+    docker:
+      - image: python:2.7
+    environment:
+      PYTHON: "2.7"
+      PANDAS: "0.19.2"
+    steps:
+      - checkout
+      - run: ci/config_auth.sh
+      - run: ci/run_pip.sh
+  "pip-3.5-0.18.1":
+    docker:
+      - image: python:3.5
+    environment:
+      PYTHON: "3.5"
+      PANDAS: "0.18.1"
+    steps:
+      - checkout
+      - run: ci/config_auth.sh
+      - run: ci/run_pip.sh
+  "pip-3.6-MASTER":
+    docker:
+      - image: python:3.6
+    environment:
+      PYTHON: "3.6"
+      PANDAS: "MASTER"
+    steps:
+      - checkout
+      - run: ci/config_auth.sh
+      - run: ci/run_pip.sh
+      # Coverage
+      - run: codecov
+  "pip-3.7-0.23.4":
+    docker:
+      - image: python:3.7
+    environment:
+      PYTHON: "3.7"
+      PANDAS: "0.23.4"
+    steps:
+      - checkout
+      - run: ci/config_auth.sh
+      - run: ci/run_pip.sh
+
+  # Conda
+  "conda-3.6-0.20.1":
+    docker:
+      - image: continuumio/miniconda3
+    environment:
+      PYTHON: "3.6"
+      PANDAS: "0.20.1"
+    steps:
+      - checkout
+      - run: ci/config_auth.sh
+      - run: ci/run_conda.sh
+
+  lint:
+    docker:
+      - image: python:3.6
+    steps:
+      - checkout
+      - run: pip install nox
+      - run: nox -s lint
+workflows:
+  version: 2
+  build:
+    jobs:
+      - "pip-2.7-0.19.2"
+      - "pip-3.5-0.18.1"
+      - "pip-3.6-MASTER"
+      - "pip-3.7-0.23.4"
+      - "conda-3.6-0.20.1"
+      - lint
diff --git a/.gitignore b/.gitignore
@@ -89,3 +89,4 @@ Thumbs.db
 # Credentials #
 ###############
 bigquery_credentials.dat
+ci/service_account.json
diff --git a/ci/config_auth.sh b/ci/config_auth.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+# Don't set -x, because we don't want to leak keys.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+# Write key to file if present.
+if [ ! -z "$SERVICE_ACCOUNT_KEY" ] ; then
+    echo "$SERVICE_ACCOUNT_KEY" | base64 --decode > "$DIR"/service_account.json
+fi
diff --git a/ci/requirements-2.7-0.19.2.pip b/ci/requirements-2.7-0.19.2.pip
@@ -1,5 +1 @@
-google-auth
-google-auth-oauthlib
-PyCrypto
 mock
-google-cloud-bigquery
diff --git a/ci/requirements-3.5-0.18.1.pip b/ci/requirements-3.5-0.18.1.pip
@@ -1,4 +1,3 @@
 google-auth==1.4.1
 google-auth-oauthlib==0.0.1
-google-cloud-bigquery==0.32.0
-pandas==0.18.1
+google-cloud-bigquery==0.32.0
diff --git a/ci/requirements-3.6-MASTER.pip b/ci/requirements-3.6-MASTER.pip
@@ -1,5 +1,3 @@
-google-auth
-google-auth-oauthlib
-git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core
-git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=core
-git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery
+git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core
+git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=core
+git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery
diff --git a/ci/requirements-3.7-0.23.4.pip b/ci/requirements-3.7-0.23.4.pip
diff --git a/ci/run_conda.sh b/ci/run_conda.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e -x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+# Install dependencies using Conda
+
+conda config --set always_yes yes --set changeps1 no
+conda config --add channels pandas
+conda config --add channels conda-forge
+conda update -q conda
+conda info -a
+conda create -q -n test-environment python=$PYTHON
+source activate test-environment
+if [[ "$PANDAS" == "MASTER" ]]; then
+  conda install -q numpy pytz python-dateutil;
+  PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com";
+  pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas;
+else
+  conda install -q pandas=$PANDAS;
+fi
+
+REQ="ci/requirements-${PYTHON}-${PANDAS}"
+conda install -q --file "$REQ.conda";
+python setup.py develop
+
+# Run the tests
+$DIR/run_tests.sh
diff --git a/ci/run_pip.sh b/ci/run_pip.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e -x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+# Install dependencies using Pip
+
+if [[ "$PANDAS" == "MASTER" ]]; then
+  PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com";
+  pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas;
+else
+  pip install pandas==$PANDAS
+fi
+
+# Install test requirements
+pip install coverage pytest pytest-cov flake8 codecov
+
+REQ="ci/requirements-${PYTHON}-${PANDAS}"
+pip install -r "$REQ.pip"
+pip install -e .
+
+$DIR/run_tests.sh
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e -x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+if [ -f "$DIR/service_account.json" ]; then
+    export GOOGLE_APPLICATION_CREDENTIALS="$DIR/service_account.json"
+fi
+
+# Install test requirements
+pip install coverage pytest pytest-cov flake8 codecov
+pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml tests
diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst
@@ -105,10 +105,10 @@ want to clone your fork to your machine::
 This creates the directory `pandas-gbq-yourname` and connects your repository to
 the upstream (main project) *pandas-gbq* repository.
 
-The testing suite will run automatically on Travis-CI once your pull request is submitted.
+The testing suite will run automatically on CircleCI once your pull request is submitted.
 However, if you wish to run the test suite on a branch prior to submitting the pull request,
-then Travis-CI needs to be hooked up to your GitHub repository.  Instructions for doing so
-are `here <http://about.travis-ci.org/docs/user/getting-started/>`__.
+then CircleCI needs to be hooked up to your GitHub repository.  Instructions for doing so
+are `here <https://circleci.com/docs/2.0/getting-started/>`__.
 
 Creating a branch
 -----------------
@@ -214,11 +214,13 @@ the more common ``PEP8`` issues:
   - we restrict line-length to 79 characters to promote readability
   - passing arguments should have spaces after commas, e.g. ``foo(arg1, arg2, kw1='bar')``
 
-Travis-CI will run the `flake8 <http://pypi.python.org/pypi/flake8>`_ tool
-and report any stylistic errors in your code. Therefore, it is helpful before
-submitting code to run the check yourself on the diff::
+CircleCI will run the `'black' code formatting tool
+<https://black.readthedocs.io/>`_ and report any stylistic errors in your
+code. Therefore, it is helpful before submitting code to run the formatter
+yourself::
 
-   git diff master | flake8 --diff
+    pip install black
+    black .
 
 Backwards Compatibility
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -287,10 +289,12 @@ directory.
 Running Google BigQuery Integration Tests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You will need to create a Google BigQuery private key in JSON format in
-order to run Google BigQuery integration tests on your local machine and
-on Travis-CI. The first step is to create a `service account
-<https://console.cloud.google.com/iam-admin/serviceaccounts/>`__.
+You will need to create a Google BigQuery private key in JSON format in order
+to run Google BigQuery integration tests on your local machine and on
+CircleCI. The first step is to create a `service account
+<https://console.cloud.google.com/iam-admin/serviceaccounts/>`__. Grant the
+service account permissions to run BigQuery queries and to create datasets
+and tables.
 
 To run the integration tests locally, set the following environment variables
 before running ``pytest``:
@@ -301,30 +305,30 @@ before running ``pytest``:
 
 Integration tests are skipped in pull requests because the credentials that
 are required for running Google BigQuery integration tests are
-`encrypted <https://docs.travis-ci.com/user/encrypting-files/>`__
-on Travis-CI and are only accessible from the pydata/pandas-gbq repository. The
-credentials won't be available on forks of pandas-gbq. Here are the steps to run
-gbq integration tests on a forked repository:
-
-#. Go to `Travis CI <https://travis-ci.org/>`__ and sign in with your GitHub
-   account.
-#. Click on the ``+`` icon next to the ``My Repositories`` list and enable
-   Travis builds for your fork.
-#. Click on the gear icon to edit your travis build, and add two environment
+`configured in the CircleCI web interface
+<https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-project>`_
+and are only accessible from the pydata/pandas-gbq repository. The
+credentials won't be available on forks of pandas-gbq. Here are the steps to
+run gbq integration tests on a forked repository:
+
+#. Go to `CircleCI <https://circleci.com/dashboard>`__ and sign in with your
+   GitHub account.
+#. Switch to your personal account in the top-left organization switcher.
+#. Use the "Add projects" tab to enable CircleCI for your fork.
+#. Click on the gear icon to edit your CircleCI build, and add two environment
    variables:
 
    - ``GBQ_PROJECT_ID`` with the value being the ID of your BigQuery project.
 
-   - ``SERVICE_ACCOUNT_KEY`` with the value being the *contents* of the JSON
-     key that you downloaded for your service account. Use single quotes around
-     your JSON key to ensure that it is treated as a string.
+   - ``SERVICE_ACCOUNT_KEY`` with the value being the base64-encoded
+   *contents* of the JSON key that you downloaded for your service account.
 
-   For both environment variables, keep the "Display value in build log" option
-   DISABLED. These variables contain sensitive data and you do not want their
-   contents being exposed in build logs.
+   Keep the contents of these variables confidential. These variables contain
+   sensitive data and you do not want their contents being exposed in build
+   logs.
 #. Your branch should be tested automatically once it is pushed. You can check
    the status by visiting your Travis branches page which exists at the
-   following location: https://travis-ci.org/your-user-name/pandas-gbq/branches .
+   following location: https://circleci.com/gh/your-username/pandas-gbq .
    Click on a build job for your branch.
 
 Documenting your code
diff --git a/tests/system/test_auth.py b/tests/system/test_auth.py
@@ -90,7 +90,7 @@ def test_get_application_default_credentials_returns_credentials():
 
 
 @pytest.mark.local_auth
-def test_get_user_account_credentials_bad_file_returns_credentials():
+def test_get_user_account_credentials_bad_file_returns_user_credentials():
     from google.auth.credentials import Credentials
 
     with mock.patch("__main__.open", side_effect=IOError()):
