Fix broken TLS interception & CacheResponsesPlugin because UID is no longer a UUID (#866)

* Fix broken TLS interception because uid is now no longer a UUID

* Give enough context to work id for them to be unique within a `proxy.py` instance

* Use --port=0 by default within `proxy.TestCase`

* Attempt to fix weird buildx issue

* Add makefile targets within workflow

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Order?

* Write scm file for make

* Fetch depth

* Quote patch

* Try with sudo?

* docker/buildx#850

* Remove sudo hack

* https://github.com/docker/buildx/issues/850\#issuecomment-973270625

* Add explicit deps

* Add `requirements-testing.txt` during linting phase

* Pin buildx to v0.7.1

* Pin buildx to v0.7.0

* Revert back unnecessary change to dockerignore

* Skip container within make workflow (because GHA lacks support for docker on macOS by default)

* Repurpose make into developer workflow

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

.github/workflows/test-library.yml

@@ -631,6 +631,34 @@ jobs:
         npm run build
         cd ..
 
+  developer:
+    runs-on: ${{ matrix.os }}-latest
+    name: Developer setup ${{ matrix.node }} @ ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu, macOS]
+        python: ['3.10']
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install Pip Dependencies
+      run: |
+        make lib-dep
+    - name: Run essentials
+      run: |
+        ./write-scm-version.sh
+        python3 check.py
+        make https-certificates
+        make sign-https-certificates
+        make ca-certificates
+        python3 -m proxy --version
+
   docker:
     # TODO: To build our docker container, we must wait for check,
     # so that we can use the same distribution available.
@@ -658,18 +686,20 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+    - name: Download all the dists
+      uses: actions/download-artifact@v2
+      with:
+        name: python-package-distributions
+        path: dist/
     - name: Set up Docker Buildx
       id: buildx
       uses: docker/setup-buildx-action@v1
       with:
+        # FIXME: See https://github.com/docker/buildx/issues/850#issuecomment-996408167
+        version: v0.7.0
         buildkitd-flags: --debug
         config: .github/buildkitd.toml
         install: true
-    - name: Download all the dists
-      uses: actions/download-artifact@v2
-      with:
-        name: python-package-distributions
-        path: dist/
     - name: Enable Multiarch    # This slows down arm build by 4-5x
       run: |
         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
@@ -699,6 +729,7 @@ jobs:
     - docker
     - dashboard
     - brew
+    - developer
 
     runs-on: Ubuntu-latest
 

Makefile

@@ -126,11 +126,11 @@ lib-release: lib-package
 
 lib-doc:
 	python -m tox -e build-docs && \
-	$(OPEN) .tox/build-docs/docs_out/index.html
+	$(OPEN) .tox/build-docs/docs_out/index.html || true
 
 lib-coverage:
 	pytest --cov=proxy --cov=tests --cov-report=html tests/ && \
-	$(OPEN) htmlcov/index.html
+	$(OPEN) htmlcov/index.html || true
 
 lib-profile:
 	ulimit -n 65536 && \
@@ -177,6 +177,11 @@ dashboard-clean:
 container: lib-package
 	$(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl)
 
+container-build:
+	docker build \
+		-t $(PROXYPY_CONTAINER_TAG) \
+		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
+
 # Usage:
 #
 # make container-buildx \
@@ -189,10 +194,5 @@ container-buildx:
 		-t $(PROXYPY_CONTAINER_TAG) \
 		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
 
-container-build:
-	docker build \
-		-t $(PROXYPY_CONTAINER_TAG) \
-		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
-
 container-run:
 	docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG)

README.md

@@ -1322,10 +1322,10 @@ import proxy
 
 if __name__ == '__main__':
   with proxy.Proxy([]) as p:
-    print(p.acceptors.flags.port)
+    print(p.flags.port)
 ```
 
-`acceptors.flags.port` will give you access to the random port allocated by the kernel.
+`flags.port` will give you access to the random port allocated by the kernel.
 
 ## Loading Plugins
 
@@ -1384,7 +1384,7 @@ Note that:
 
 1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and tear down `proxy.py`.
 2. `proxy.py` server will listen on a random available port on the system.
-   This random port is available as `self.PROXY.acceptors.flags.port` within your test cases.
+   This random port is available as `self.PROXY.flags.port` within your test cases.
 3. Only a single acceptor and worker is started by default (`--num-workers 1 --num-acceptors 1`) for faster setup and tear down.
 4. Most importantly, `proxy.TestCase` also ensures `proxy.py` server
    is up and running before proceeding with execution of tests. By default,
@@ -2073,7 +2073,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.3.2.dev190+ge60d80d.d20211124
+proxy.py v2.4.0rc2.dev21+g20b3eb1.d20211203
 
 options:
   -h, --help            show this help message and exit

check.py

@@ -60,11 +60,12 @@
 with open('README.md', 'rb+') as f:
     c = f.read()
     pre_flags, post_flags = c.split(b'# Flags')
-    help_text, post_changelog = post_flags.split(b'# Changelog')
     f.seek(0)
     f.write(
-        pre_flags + b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' + lib_help + b'```' +
-        b'\n\n# Changelog' + post_changelog,
+        pre_flags +
+        b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' +
+        lib_help +
+        b'```\n',
     )
 
 # Version is also hardcoded in README.md flags section

proxy/core/acceptor/acceptor.py

@@ -191,6 +191,7 @@ def _start_local(self) -> None:
         assert self.sock
         self._local_work_queue = NonBlockingQueue()
         self._local = LocalExecutor(
+            iid=self.idd,
             work_queue=self._local_work_queue,
             flags=self.flags,
             event_queue=self.event_queue,

proxy/core/acceptor/executors.py

@@ -175,6 +175,7 @@ def _start_worker(self, index: int) -> None:
         pipe = multiprocessing.Pipe()
         self.work_queues.append(pipe[0])
         w = RemoteExecutor(
+            iid=index,
             work_queue=pipe[1],
             flags=self.flags,
             event_queue=self.event_queue,

proxy/core/acceptor/threadless.py

@@ -59,11 +59,13 @@ class Threadless(ABC, Generic[T]):
 
     def __init__(
             self,
+            iid: str,
             work_queue: T,
             flags: argparse.Namespace,
             event_queue: Optional[EventQueue] = None,
     ) -> None:
         super().__init__()
+        self.iid = iid
         self.work_queue = work_queue
         self.flags = flags
         self.event_queue = event_queue
@@ -84,6 +86,7 @@ def __init__(
         ] = {}
         self.wait_timeout: float = DEFAULT_WAIT_FOR_TASKS_TIMEOUT
         self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT
+        self._total: int = 0
 
     @property
     @abstractmethod
@@ -122,14 +125,15 @@ def work_on_tcp_conn(
             fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6,
             type=socket.SOCK_STREAM,
         )
+        uid = '%s-%s-%s' % (self.iid, self._total, fileno)
         self.works[fileno] = self.flags.work_klass(
             TcpClientConnection(
                 conn=conn,
                 addr=addr,
             ),
             flags=self.flags,
             event_queue=self.event_queue,
-            uid=fileno,
+            uid=uid,
         )
         self.works[fileno].publish_event(
             event_name=eventNames.WORK_STARTED,
@@ -138,6 +142,7 @@ def work_on_tcp_conn(
         )
         try:
             self.works[fileno].initialize()
+            self._total += 1
         except Exception as e:
             logger.exception(
                 'Exception occurred during initialization',

proxy/core/acceptor/work.py

@@ -15,7 +15,7 @@
 import argparse
 
 from abc import ABC, abstractmethod
-from uuid import uuid4, UUID
+from uuid import uuid4
 from typing import Optional, Dict, Any
 
 from ..event import eventNames, EventQueue
@@ -31,10 +31,10 @@ def __init__(
             work: TcpClientConnection,
             flags: argparse.Namespace,
             event_queue: Optional[EventQueue] = None,
-            uid: Optional[UUID] = None,
+            uid: Optional[str] = None,
     ) -> None:
         # Work uuid
-        self.uid: UUID = uid if uid is not None else uuid4()
+        self.uid: str = uid if uid is not None else uuid4().hex
         self.flags = flags
         # Eventing core queue
         self.event_queue = event_queue
@@ -92,7 +92,7 @@ def publish_event(
             return
         assert self.event_queue
         self.event_queue.publish(
-            self.uid.hex,
+            self.uid,
             event_name,
             event_payload,
             publisher_id,

proxy/http/plugin.py

@@ -11,7 +11,6 @@
 import socket
 import argparse
 
-from uuid import UUID
 from abc import ABC, abstractmethod
 from typing import Tuple, List, Union, Optional
 
@@ -46,13 +45,13 @@ class HttpProtocolHandlerPlugin(ABC):
 
     def __init__(
             self,
-            uid: UUID,
+            uid: str,
             flags: argparse.Namespace,
             client: TcpClientConnection,
             request: HttpParser,
             event_queue: EventQueue,
     ):
-        self.uid: UUID = uid
+        self.uid: str = uid
         self.flags: argparse.Namespace = flags
         self.client: TcpClientConnection = client
         self.request: HttpParser = request

proxy/http/proxy/plugin.py

@@ -11,7 +11,6 @@
 import argparse
 
 from abc import ABC
-from uuid import UUID
 from typing import Any, Dict, List, Optional, Tuple
 
 from ..parser import HttpParser
@@ -28,7 +27,7 @@ class HttpProxyBasePlugin(ABC):
 
     def __init__(
             self,
-            uid: UUID,
+            uid: str,
             flags: argparse.Namespace,
             client: TcpClientConnection,
             event_queue: EventQueue,

proxy/http/proxy/server.py

@@ -307,11 +307,8 @@ async def read_from_descriptors(self, r: Readables) -> bool:
             # parse incoming response packet
             # only for non-https requests and when
             # tls interception is enabled
-            if not self.request.is_https_tunnel:
-                # See https://github.com/abhinavsingh/proxy.py/issues/127 for why
-                # currently response parsing is disabled when TLS interception is enabled.
-                #
-                # or self.tls_interception_enabled():
+            if not self.request.is_https_tunnel \
+                    or self.tls_interception_enabled():
                 if self.response.is_complete:
                     self.handle_pipeline_response(raw)
                 else:
@@ -733,7 +730,7 @@ def gen_ca_signed_certificate(
         ca_key_path = self.flags.ca_key_file
         ca_key_password = ''
         ca_crt_path = self.flags.ca_cert_file
-        serial = self.uid.int
+        serial = self.uid
 
         # Sign generated CSR
         if not os.path.isfile(cert_file_path):
@@ -903,7 +900,7 @@ def emit_request_complete(self) -> None:
             return
         assert self.request.port
         self.event_queue.publish(
-            request_id=self.uid.hex,
+            request_id=self.uid,
             event_name=eventNames.REQUEST_COMPLETE,
             event_payload={
                 'url': text_(self.request.path)
@@ -937,7 +934,7 @@ def emit_response_headers_complete(self) -> None:
         if not self.flags.enable_events:
             return
         self.event_queue.publish(
-            request_id=self.uid.hex,
+            request_id=self.uid,
             event_name=eventNames.RESPONSE_HEADERS_COMPLETE,
             event_payload={
                 'headers': {}
@@ -954,7 +951,7 @@ def emit_response_chunk_received(self, chunk_size: int) -> None:
         if not self.flags.enable_events:
             return
         self.event_queue.publish(
-            request_id=self.uid.hex,
+            request_id=self.uid,
             event_name=eventNames.RESPONSE_CHUNK_RECEIVED,
             event_payload={
                 'chunk_size': chunk_size,
@@ -967,7 +964,7 @@ def emit_response_complete(self) -> None:
         if not self.flags.enable_events:
             return
         self.event_queue.publish(
-            request_id=self.uid.hex,
+            request_id=self.uid,
             event_name=eventNames.RESPONSE_COMPLETE,
             event_payload={
                 'encoded_response_size': self.response.total_size,

proxy/http/server/plugin.py

@@ -10,7 +10,6 @@
 """
 import argparse
 
-from uuid import UUID
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -27,7 +26,7 @@ class HttpWebServerBasePlugin(ABC):
 
     def __init__(
             self,
-            uid: UUID,
+            uid: str,
             flags: argparse.Namespace,
             client: TcpClientConnection,
             event_queue: EventQueue,

proxy/plugin/cache/store/base.py

@@ -10,13 +10,13 @@
 """
 from abc import ABC, abstractmethod
 from typing import Optional
-from uuid import UUID
+
 from ....http.parser import HttpParser
 
 
 class CacheStore(ABC):
 
-    def __init__(self, uid: UUID) -> None:
+    def __init__(self, uid: str) -> None:
         self.uid = uid
 
     @abstractmethod