remove need to clone core repo (open-telemetry#678)

Author: alrex

.flake8

@@ -19,6 +19,5 @@ exclude =
   exporter/opentelemetry-exporter-jaeger/build/*
   docs/examples/opentelemetry-example-app/src/opentelemetry_example_app/grpc/gen/
   docs/examples/opentelemetry-example-app/build/*
-  opentelemetry-python-core/
   opentelemetry-proto/build/*
   opentelemetry-proto/src/opentelemetry/proto/

.github/workflows/test.yml

@@ -28,12 +28,6 @@ jobs:
     steps:
       - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@v2
-      - name: Checkout Core Repo @ SHA - ${{ env.CORE_REPO_SHA }}
-        uses: actions/checkout@v2
-        with:
-          repository: open-telemetry/opentelemetry-python
-          ref: ${{ env.CORE_REPO_SHA }}
-          path: opentelemetry-python-core
       - name: Set up Python ${{ env[matrix.python-version] }}
         uses: actions/setup-python@v2
         with:
@@ -84,12 +78,6 @@ jobs:
     steps:
       - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@v2
-      - name: Checkout Core Repo @ SHA ${{ env.CORE_REPO_SHA }}
-        uses: actions/checkout@v2
-        with:
-          repository: open-telemetry/opentelemetry-python
-          ref: ${{ env.CORE_REPO_SHA }}
-          path: opentelemetry-python-core
       - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
@@ -109,6 +97,3 @@ jobs:
           key: v4-misc-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt', 'gen-requirements.txt', 'docs-requirements.txt') }}
       - name: run tox
         run: tox -e ${{ matrix.tox-environment }}
-      - name: Ensure generated code is up to date
-        if: matrix.tox-environment == 'generate'
-        run: git diff --exit-code || (echo 'Generated code is out of date, please run "tox -e generate" and commit the changes in this PR.' && exit 1)

.gitignore

@@ -23,8 +23,6 @@ lib64
 __pycache__
 venv*/
 .venv*/
-opentelemetry-python-core*/
-/opentelemetry-python-core
 
 # Installer logs
 pip-log.txt

.isort.cfg

@@ -14,6 +14,6 @@ profile=black
 ; docs: https://github.com/timothycrosley/isort#multi-line-output-modes
 multi_line_output=3
 skip=target
-skip_glob=**/gen/*,.venv*/*,venv*/*,opentelemetry-python-core/*,.tox/*
+skip_glob=**/gen/*,.venv*/*,venv*/*,.tox/*
 known_first_party=opentelemetry
 known_third_party=psutil,pytest,redis,redis_opentracing

CONTRIBUTING.md

@@ -156,16 +156,16 @@ For a deeper discussion, see: https://github.com/open-telemetry/opentelemetry-sp
 ## Running Tests Locally
 
 1. Go to your Contrib repo directory. `git clone [email protected]:open-telemetry/opentelemetry-python-contrib.git && cd opentelemetry-python-contrib`.
-2. Clone the [OpenTelemetry Python](https://github.com/open-telemetry/opentelemetry-python) Python Core repo to a folder named `opentelemetry-python-core`. `git clone https://github.com/open-telemetry/opentelemetry-python.git opentelemetry-python-core`.
-3. Make sure you have `tox` installed. `pip install tox`.
-4. Run `tox` without any arguments to run tests for all the packages. Read more about [tox](https://tox.readthedocs.io/en/latest/).
+2. Make sure you have `tox` installed. `pip install tox`.
+3. Run `tox` without any arguments to run tests for all the packages. Read more about [tox](https://tox.readthedocs.io/en/latest/).
 
 ### Testing against a different Core repo branch/commit
 
-1. Change directory to the repo that was cloned above. `cd opentelemetry-python-core`.
-2. Move the head of this repo to the SHA hash you want your tests to use. The current SHA hash can be found in `.github/workflows/test.yml` file under the `opentelemetry-python-contrib` directory. For example, currently it is `1a12fa0d681e37c1fda9cb8d46212ff3bbf6b76a`. So use `git fetch && git checkout <current SHA hash>`.
-3. Go back to the root directory. `cd ../`.
-4. With `tox` installed, run tests for a package. (e.g. `tox -e test-instrumentation-flask`.)
+Some of the tox targets install packages from the [OpenTelemetry Python Core Repository](https://github.com/open-telemetry/opentelemetry-python) via pip. The version of the packages installed defaults to the main branch in that repository when tox is run locally. It is possible to install packages tagged with a specific git commit hash by setting an environment variable before running tox as per the following example:
+
+CORE_REPO_SHA=c49ad57bfe35cfc69bfa863d74058ca9bec55fc3 tox
+
+The continuation integration overrides that environment variable with as per the configuration [here](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/.github/workflows/test.yml#L9).
 
 
 ## Style Guide

README.md

@@ -116,12 +116,8 @@ Maintainers ([@open-telemetry/python-maintainers](https://github.com/orgs/open-t
 1. Go to your Contrib repo directory. `cd ~/git/opentelemetry-python-contrib`.
 2. Create a virtual env in your Contrib repo directory. `python3 -m venv my_test_venv`.
 3. Activate your virtual env. `source my_test_venv/bin/activate`.
-4. Clone the [OpenTelemetry Python](https://github.com/open-telemetry/opentelemetry-python) Python Core repo to a folder named `opentelemetry-python-core`. `git clone [email protected]:open-telemetry/opentelemetry-python.git opentelemetry-python-core`.
-5. Change directory to the repo that was just cloned. `cd opentelemetry-python-core`.
-6. Move the head of this repo to the hash you want your tests to use. This is currently the SHA `47483865854c7adae7455f8441dab7f814f4ce2a` as seen in `.github/workflows/test.yml`. Use `git fetch && git checkout 47483865854c7adae7455f8441dab7f814f4ce2a`.
-7. Go back to the root directory. `cd ../`.
-8. Make sure you have `tox` installed. `pip install tox`.
-9. Run tests for a package. (e.g. `tox -e test-instrumentation-flask`.)
+4. Make sure you have `tox` installed. `pip install tox`.
+5. Run tests for a package. (e.g. `tox -e test-instrumentation-flask`.)
 
 ### Thanks to all the people who already contributed!
 

docs/conf.py

@@ -24,11 +24,7 @@
 
 settings.configure()
 
-source_dirs = [
-    os.path.abspath(
-        "../opentelemetry-python-core/opentelemetry-instrumentation/src/"
-    ),
-]
+source_dirs = []
 
 exp = "../exporter"
 exp_dirs = [
@@ -51,7 +47,7 @@
     if isdir(join(sdk_ext, f))
 ]
 
-sys.path[:0] = source_dirs + exp_dirs + instr_dirs + sdk_ext_dirs
+sys.path[:0] = exp_dirs + instr_dirs + sdk_ext_dirs
 
 # -- Project information -----------------------------------------------------
 

eachdist.ini

@@ -3,8 +3,6 @@
 [DEFAULT]
 ignore=
     _template
-    reference
-    opentelemetry-python-core
 
 sortfirst=
     util/opentelemetry-util-http

gen-requirements.txt

@@ -3,4 +3,4 @@ astor==0.8.1
 jinja2~=2.7
 isort
 black
-
+requests

scripts/generate_instrumentation_bootstrap.py

@@ -15,18 +15,17 @@
 # limitations under the License.
 
 import ast
+import filecmp
 import logging
 import os
 import subprocess
 import sys
+import tempfile
 
 import astor
 import pkg_resources
-from otel_packaging import (
-    get_instrumentation_packages,
-    root_path,
-    scripts_path,
-)
+import requests
+from otel_packaging import get_instrumentation_packages, scripts_path
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("instrumentation_list_generator")
@@ -50,16 +49,16 @@
 default_instrumentations = []
 """
 
+tmpdir = tempfile.TemporaryDirectory()  # pylint: disable=R1732
+gen_path = os.path.join(tmpdir.name, "new.py",)
 
-gen_path = os.path.join(
-    root_path,
-    "opentelemetry-python-core",
-    "opentelemetry-instrumentation",
-    "src",
-    "opentelemetry",
-    "instrumentation",
-    "bootstrap_gen.py",
-)
+current_path = os.path.join(tmpdir.name, "current.py",)
+
+core_repo = os.getenv("CORE_REPO_SHA", "main")
+url = f"https://raw.githubusercontent.com/open-telemetry/opentelemetry-python/{core_repo}/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py"
+r = requests.get(url, allow_redirects=True)
+with open(current_path, "wb") as output:
+    output.write(r.content)
 
 
 def main():
@@ -99,13 +98,23 @@ def main():
             "scripts/eachdist.py",
             "format",
             "--path",
-            "opentelemetry-python-core/opentelemetry-instrumentation",
+            tmpdir.name,
         ],
         check=True,
     )
 
     logger.info("generated %s", gen_path)
 
 
+def compare():
+    if not filecmp.cmp(current_path, gen_path):
+        logger.info(
+            'Generated code is out of date, please run "tox -e generate" and commit bootstrap_gen.py to core repo.'
+        )
+        os.replace(gen_path, "bootstrap_gen.py")
+        sys.exit(1)
+
+
 if __name__ == "__main__":
     main()
+    compare()

tox.ini

@@ -184,6 +184,12 @@ deps =
   ; FIXME: add coverage testing
   ; FIXME: add mypy testing
 
+setenv =
+  ; override CORE_REPO_SHA via env variable when testing other branches/commits than main
+  ; i.e: CORE_REPO_SHA=dde62cebffe519c35875af6d06fae053b3be65ec tox -e <env to test>
+  CORE_REPO_SHA={env:CORE_REPO_SHA:"main"}
+  CORE_REPO="git+https://github.com/open-telemetry/opentelemetry-python.git@{env:CORE_REPO_SHA}"
+
 changedir =
   test-instrumentation-aiohttp-client: instrumentation/opentelemetry-instrumentation-aiohttp-client/tests
   test-instrumentation-aiopg: instrumentation/opentelemetry-instrumentation-aiopg/tests
@@ -228,11 +234,11 @@ commands_pre =
   py3{6,7,8,9}: python -m pip install -U pip setuptools wheel
 ; Install common packages for all the tests. These are not needed in all the
 ; cases but it saves a lot of boilerplate in this file.
-  test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api[test]
-  test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions[test]
-  test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation[test]
-  test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk[test]
-  test: pip install {toxinidir}/opentelemetry-python-core/tests/util[test]
+  test: pip install "opentelemetry-api[test] @ {env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api"
+  test: pip install "opentelemetry-semantic-conventions[test] @ {env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions"
+  test: pip install "opentelemetry-instrumentation[test] @ {env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation"
+  test: pip install "opentelemetry-sdk[test] @ {env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk"
+  test: pip install "opentelemetry-test[test] @ {env:CORE_REPO}#egg=opentelemetry-test&subdirectory=tests/util"
 
   celery: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-celery[test]
 
@@ -299,7 +305,7 @@ commands_pre =
 
   sqlalchemy{11,14}: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlalchemy[test]
 
-  elasticsearch{2,5,6}: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation {toxinidir}/instrumentation/opentelemetry-instrumentation-elasticsearch[test]
+  elasticsearch{2,5,6}: pip install "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" {toxinidir}/instrumentation/opentelemetry-instrumentation-elasticsearch[test]
 
   httpx: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-httpx[test]
 
@@ -323,10 +329,10 @@ deps =
   pytest
 
 commands_pre =
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk"
   python -m pip install {toxinidir}/util/opentelemetry-util-http
 
 changedir = docs
@@ -348,11 +354,11 @@ deps =
   httpretty
 
 commands_pre =
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation[test]
-  python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk
-  python -m pip install {toxinidir}/opentelemetry-python-core/tests/util
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk"
+  python -m pip install "{env:CORE_REPO}#egg=opentelemetry-test&subdirectory=tests/util"
   python -m pip install -e {toxinidir}/util/opentelemetry-util-http[test]
   python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-wsgi[test]
   python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi[test]
@@ -417,11 +423,11 @@ changedir =
   tests/opentelemetry-docker-tests/tests
 
 commands_pre =
-  pip install -e {toxinidir}/opentelemetry-python-core/opentelemetry-api \
-              -e {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions \
-              -e {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation \
-              -e {toxinidir}/opentelemetry-python-core/opentelemetry-sdk \
-              -e {toxinidir}/opentelemetry-python-core/tests/util \
+  pip install "{env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api" \
+              "{env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions" \
+              "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" \
+              "{env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk" \
+              "{env:CORE_REPO}#egg=opentelemetry-test&subdirectory=tests/util" \
               -e {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncpg \
               -e {toxinidir}/instrumentation/opentelemetry-instrumentation-celery \
               -e {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi \
@@ -432,7 +438,7 @@ commands_pre =
               -e {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlalchemy \
               -e {toxinidir}/instrumentation/opentelemetry-instrumentation-aiopg \
               -e {toxinidir}/instrumentation/opentelemetry-instrumentation-redis \
-              -e {toxinidir}/opentelemetry-python-core/exporter/opentelemetry-exporter-opencensus
+              "{env:CORE_REPO}#egg=opentelemetry-exporter-opencensus&subdirectory=exporter/opentelemetry-exporter-opencensus"
   docker-compose up -d
   python check_availability.py