feat(django): pytest startup and test collection on Django 1.11 (#16105)

Author: josh

.travis.yml

@@ -149,6 +149,21 @@ matrix:
       name: 'Acceptance'
       env: TEST_SUITE=acceptance USE_SNUBA=1
 
+    # allowed to fail
+    - <<: *postgres_default
+      name: '[Django 1.11] Backend with migrations [Postgres] (1/2)'
+      env: DJANGO_VERSION=">=1.11,<1.12" TEST_SUITE=postgres DB=postgres TOTAL_TEST_GROUPS=2 TEST_GROUP=0 MIGRATIONS_TEST_MIGRATE=1
+
+    # allowed to fail
+    - <<: *postgres_default
+      name: '[Django 1.11] Backend with migrations [Postgres] (2/2)'
+      env: DJANGO_VERSION=">=1.11,<1.12" TEST_SUITE=postgres DB=postgres TOTAL_TEST_GROUPS=2 TEST_GROUP=1 MIGRATIONS_TEST_MIGRATE=1
+
+    # allowed to fail
+    - <<: *acceptance_default
+      name: '[Django 1.11] Acceptance'
+      env: DJANGO_VERSION=">=1.11,<1.12" TEST_SUITE=acceptance USE_SNUBA=1 PERCY_ENABLE=0
+
     - <<: *acceptance_default
       name: 'Plugins'
       env: TEST_SUITE=plugins DB=postgres PERCY_TOKEN=${PLUGIN_PERCY_TOKEN}
@@ -214,6 +229,29 @@ matrix:
       before_script:
         - psql -c 'create database sentry;' -U postgres
 
+    # allowed to fail
+    - python: 2.7
+      name: '[Django 1.11] Snuba Integration with migrations'
+      env: DJANGO_VERSION=">=1.11,<1.12" TEST_SUITE=snuba USE_SNUBA=1 SENTRY_ZOOKEEPER_HOSTS=localhost:2181 SENTRY_KAFKA_HOSTS=localhost:9092 MIGRATIONS_TEST_MIGRATE=1
+      services:
+        - docker
+        - memcached
+        - redis-server
+        - postgresql
+      before_install:
+        - *pip_install
+        - docker run -d --network host --name zookeeper -e ZOOKEEPER_CLIENT_PORT=2181 confluentinc/cp-zookeeper:4.1.0
+        - docker run -d --network host --name kafka -e KAFKA_ZOOKEEPER_CONNECT=localhost:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 confluentinc/cp-kafka:4.1.0
+        - docker run -d --network host --name clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:19.11
+        - docker run -d --network host --name snuba --env SNUBA_SETTINGS=test --env CLICKHOUSE_SERVER=localhost:9000 getsentry/snuba
+        - docker ps -a
+      install:
+        - python setup.py install_egg_info
+        - pip install -U -e ".[dev]"
+        - pip install confluent-kafka
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
     # Deploy 'storybook' (component & style guide) - allowed to fail
     - name: 'Storybook Deploy'
       language: node_js
@@ -235,6 +273,10 @@ matrix:
 
   allow_failures:
     - name: 'Storybook Deploy'
+    - name: '[Django 1.11] Backend with migrations [Postgres] (1/2)'
+    - name: '[Django 1.11] Backend with migrations [Postgres] (2/2)'
+    - name: '[Django 1.11] Acceptance'
+    - name: '[Django 1.11] Snuba Integration with migrations'
 
 notifications:
   webhooks:

src/sentry/db/postgres/base.py

@@ -3,6 +3,8 @@
 from six import string_types
 import psycopg2 as Database
 
+import django
+
 # Some of these imports are unused, but they are inherited from other engines
 # and should be available as part of the backend ``base.py`` namespace.
 from django.db.backends.postgresql_psycopg2.base import DatabaseWrapper
@@ -95,7 +97,18 @@ def _set_isolation_level(self, level):
     @auto_reconnect_connection
     def _cursor(self, *args, **kwargs):
         cursor = super(DatabaseWrapper, self)._cursor()
-        return CursorWrapper(self, cursor)
+        if django.VERSION[:2] < (1, 11):
+            return CursorWrapper(self, cursor)
+        return cursor
+
+    # We're overriding this internal method that's present in Django 1.11+, because
+    # things were shuffled around since 1.10 resulting in not constructing a django CursorWrapper
+    # with our CursorWrapper. We need to be passing our wrapped cursor to their wrapped cursor,
+    # not the other way around since then we'll lose things like __enter__ due to the way this
+    # wrapper is working (getattr on self.cursor).
+    def _prepare_cursor(self, cursor):
+        cursor = super(DatabaseWrapper, self)._prepare_cursor(CursorWrapper(self, cursor))
+        return cursor
 
     def close(self, reconnect=False):
         """

src/sentry/new_migrations/monkey/__init__.py

@@ -7,7 +7,7 @@
 
 from django.db.migrations import migration, executor, writer
 
-LAST_VERIFIED_DJANGO_VERSION = (1, 10)
+LAST_VERIFIED_DJANGO_VERSION = (1, 11)
 CHECK_MESSAGE = """Looks like you're trying to upgrade Django! Since we monkeypatch
 the Django migration library in several places, please verify that we have the latest
 code, and that the monkeypatching still works as expected. Currently the main things

src/sentry/runner/initializer.py

@@ -404,7 +404,8 @@ def __model_unpickle_compat(model_id, attrs=None, factory=None):
         attrs = [] if attrs is None else attrs
         factory = django.db.models.base.simple_class_factory if factory is None else factory
         return model_unpickle(model_id, attrs, factory)
-    elif VERSION[:2] == (1, 10):
+    # TODO(joshuarli): unverified on 1.11, but i'm doing this to unblock tests for now
+    elif VERSION[:2] in [(1, 10), (1, 11)]:
         return model_unpickle(model_id)
     else:
         raise NotImplementedError