From 66ad44cd9b61dfbf4f56162f2ee3bee11dde6fe1 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 14 Oct 2022 02:54:59 +0800 Subject: [PATCH] Enhance developer experience for debugging tests and add related README ### Motivation When I ran `python3 ./tests/pulsar_tests.py`, I found some tests failed. However, when I went into the container that was started by `./build-support/pulsar-test-service-start.sh`, the logs are incomplete because the immediate flush is disabled by default. Finally I found the error is caused by the relative path in tests. The certs are under `tests/test-conf` directory, but `pulsar_tests.py` uses the `test-conf/` relative path. Since `tests/run-unit-tests.sh` first goes to the `tests` directory before running the tests, CI works well. But a developer might just want to run a specific test from any possible directory, e.g. ```bash python3 ./tests/pulsar_tests.py PulsarTest.test_tls_auth ``` Then it could fail and he cannot see complete logs. ### Modifications - Modify `immediateFlush` to true - Use absolute path to represent `test-conf/` in `pulsar_tests.py` - Add an initial README for quick start --- README.md | 84 +++++++++++++++++++ .../start-test-service-inside-container.sh | 3 + tests/pulsar_test.py | 23 +++-- 3 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d403cd --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ + + +# Pulsar Python client library + +## Requirements + +- Python >= 3.7 +- A C++ compiler that supports C++11 +- CMake >= 3.18 +- [Pulsar C++ client library](https://github.com/apache/pulsar-client-cpp) +- [Boost.Python](https://github.com/boostorg/python) + +## Install the Python wheel + +```bash +cmake -B build +cmake --build build -j8 +cp build/_pulsar.so . +./setup.py bdist_wheel +pip3 install dist/pulsar_client-*.whl --force-reinstall +rm _pulsar.so +``` + +> **NOTE** +> +> 1. Here a separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` is under the project directory. +> 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before. + +You can run `python3 -c 'import pulsar'` to see whether the wheel has been installed successfully. If it failed, check whether dependencies (e.g. `libpulsar.so`) are in the system path. If not, make sure the dependencies are in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS). + +Then you can run examples as a simple end-to-end test. + +```bash +# In terminal 1 +python3 ./examples/consumer.py +``` + +```bash +# In terminal 2 +python3 ./examples/producer.py +``` + +Before executing the commands above, you must ensure the Pulsar service is running. See [here](https://pulsar.apache.org/docs/getting-started-standalone) for quick start. + +## Unit tests + +Before running the unit tests, you must run a Pulsar service with all things set up: + +```bash +./build-support/pulsar-test-service-start.sh +``` + +The command above runs a Pulsar standalone in a Docker container. You can run `./build-support/pulsar-test-service-stop.sh` to stop it. + +Run all unit tests: + +```bash +./tests/run-unit-tests.sh +``` + +Run a single unit test (e.g. `PulsarTest.test_tls_auth`): + +```bash +python3 ./tests/pulsar_test.py 'PulsarTest.test_tls_auth' +``` diff --git a/build-support/start-test-service-inside-container.sh b/build-support/start-test-service-inside-container.sh index 0517f77..658be03 100755 --- a/build-support/start-test-service-inside-container.sh +++ b/build-support/start-test-service-inside-container.sh @@ -22,6 +22,9 @@ set -e -x export PULSAR_EXTRA_OPTS=-Dpulsar.auth.basic.conf=test-conf/.htpasswd +# Show logs immediately for debugging +sed -i.bak 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml + # Generate secret key and token mkdir -p data/tokens bin/pulsar tokens create-secret-key --output data/tokens/secret.key diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py index 314f6f6..71af279 100755 --- a/tests/pulsar_test.py +++ b/tests/pulsar_test.py @@ -49,6 +49,7 @@ from urllib.request import urlopen, Request TM = 10000 # Do not wait forever in tests +CERTS_DIR = os.path.dirname(__file__) + "/test-conf/" def doHttpPost(url, data): @@ -319,12 +320,11 @@ def test_message_properties(self): client.close() def test_tls_auth(self): - certs_dir = "test-conf/" client = Client( self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, - authentication=AuthenticationTLS(certs_dir + "client-cert.pem", certs_dir + "client-key.pem"), + authentication=AuthenticationTLS(CERTS_DIR + "client-cert.pem", CERTS_DIR + "client-key.pem"), ) topic = "my-python-topic-tls-auth-" + str(time.time()) @@ -342,13 +342,12 @@ def test_tls_auth(self): client.close() def test_tls_auth2(self): - certs_dir = "test-conf/" authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls" - authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir) + authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (CERTS_DIR, CERTS_DIR) client = Client( self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=Authentication(authPlugin, authParams), ) @@ -368,8 +367,8 @@ def test_tls_auth2(self): client.close() def test_encryption(self): - publicKeyPath = "test-conf/public-key.client-rsa.pem" - privateKeyPath = "test-conf/private-key.client-rsa.pem" + publicKeyPath = CERTS_DIR + "public-key.client-rsa.pem" + privateKeyPath = CERTS_DIR + "private-key.client-rsa.pem" crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath) client = Client(self.serviceUrl) topic = "my-python-test-end-to-end-encryption" @@ -400,13 +399,12 @@ def test_encryption(self): client.close() def test_tls_auth3(self): - certs_dir = "test-conf/" authPlugin = "tls" - authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir) + authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (CERTS_DIR, CERTS_DIR) client = Client( self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=Authentication(authPlugin, authParams), ) @@ -426,12 +424,11 @@ def test_tls_auth3(self): client.close() def test_auth_junk_params(self): - certs_dir = "test-conf/" authPlugin = "someoldjunk.so" authParams = "blah" client = Client( self.serviceUrlTls, - tls_trust_certs_file_path=certs_dir + "cacert.pem", + tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=Authentication(authPlugin, authParams), )