From 18741450ceafeac03c168973c05279150ac97819 Mon Sep 17 00:00:00 2001 From: "Pfifer, Justin" Date: Mon, 25 Feb 2019 08:03:42 -0800 Subject: [PATCH] Updated to KCL 2.1.2 with SDK 2.4.0 Updated the dependency script to handle classifiers Maven has an additional field call classifier which allow artifacts to vary. One of the KCL dependencies includes a classifier so the dependencies script has been updated to support that. --- scripts/build_deps.py | 48 ++++++++++++++++++++++++++++++++++++----- setup.py | 50 +++++++++++++++++++++---------------------- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/scripts/build_deps.py b/scripts/build_deps.py index 49d4b61..d4da5ca 100755 --- a/scripts/build_deps.py +++ b/scripts/build_deps.py @@ -13,7 +13,7 @@ # permissions and limitations under the License. """ Builds the dependency list used by setup.py from the maven dependency tree. This script must be run in the -amazon-kinesis-client directory, or where the pom.xml for the amazon-kinesis-client is available. +amazon-kinesis-client or amazon-kinesis-client-multilang directory, or where the pom.xml for the libraries are present. """ import subprocess from tempfile import mkstemp @@ -22,14 +22,52 @@ def format_dependency(line): - match = re.match(r'^[\\\s+|-]*(?P[^:]+):(?P[^:]+):[^:]+:(?P[^:\s]+)', line) + """ + This attempts to extract Maven dependencies and versions from a line of output from mvn dependency:tree + + An example line without specifiers: + + ``[INFO] +- software.amazon.kinesis:amazon-kinesis-client:jar:2.1.2:compile`` + + This fields in the line in order are: + 1. Group Id: software.amazon.kinesis + 2. Artifact Id: amazon-kinesis-client + 3. Packaging: jar (not used) + 4. Version: 2.1.2 + 5. Dependency type: compile (this will be runtime or compile) + + An example line with specifiers: + + ``[INFO] | | +- io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.32.Final:compile`` + + The fields in order are: + 1. Group Id: io.netty + 2. Artifact Id: netty-transport-native-epoll + 3. Packaging: jar (not used) + 4. Specifier: linux-x86_64 (not used) + 5. Version: 4.1.32.Final + 6. Dependency type: compile (this will be runtime or compile) + + :param str line: the line to extract version information from + :return: the version information needed to retrieve the jars from Maven Central + """ + match = re.match(r'^[\\\s+|-]*(?P[^\s]+)', line) assert match is not None - return "('{group_id}', '{artifact_id}', '{version}')".format(group_id=match.groupdict()['group_id'], - artifact_id=match.groupdict()['artifact_id'], - version=match.groupdict()['version']) + items = match.groupdict()['dep_line'].split(":") + version_idx = 3 + if len(items) > 5: + version_idx = 4 + + return "('{group_id}', '{artifact_id}', '{version}')".format(group_id=items[0], + artifact_id=items[1], + version=items[version_idx]) def build_deps(): + """ + Extracts all the dependencies from the pom.xml and formats them into a form usable for setup.py or other + multilang daemon implementations + """ (fh, filename) = mkstemp() close(fh) output_command = '-Doutput={temp}'.format(temp=filename) diff --git a/setup.py b/setup.py index 2f66e2b..99aba8b 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.0' +PACKAGE_VERSION = '2.0.1' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 @@ -58,15 +58,15 @@ ] REMOTE_MAVEN_PACKAGES = [ # (group id, artifact id, version), - ('software.amazon.kinesis', 'amazon-kinesis-client-multilang', '2.1.0'), - ('software.amazon.kinesis', 'amazon-kinesis-client', '2.1.0'), - ('software.amazon.awssdk', 'kinesis', '2.2.0'), - ('software.amazon.awssdk', 'aws-cbor-protocol', '2.2.0'), - ('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.7'), - ('software.amazon.awssdk', 'aws-json-protocol', '2.2.0'), - ('software.amazon.awssdk', 'dynamodb', '2.2.0'), - ('software.amazon.awssdk', 'cloudwatch', '2.2.0'), - ('software.amazon.awssdk', 'netty-nio-client', '2.2.0'), + ('software.amazon.kinesis', 'amazon-kinesis-client-multilang', '2.1.2'), + ('software.amazon.kinesis', 'amazon-kinesis-client', '2.1.2'), + ('software.amazon.awssdk', 'kinesis', '2.4.0'), + ('software.amazon.awssdk', 'aws-cbor-protocol', '2.4.0'), + ('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.8'), + ('software.amazon.awssdk', 'aws-json-protocol', '2.4.0'), + ('software.amazon.awssdk', 'dynamodb', '2.4.0'), + ('software.amazon.awssdk', 'cloudwatch', '2.4.0'), + ('software.amazon.awssdk', 'netty-nio-client', '2.4.0'), ('io.netty', 'netty-codec-http', '4.1.32.Final'), ('io.netty', 'netty-codec-http2', '4.1.32.Final'), ('io.netty', 'netty-codec', '4.1.32.Final'), @@ -90,22 +90,22 @@ ('org.apache.commons', 'commons-lang3', '3.8.1'), ('org.slf4j', 'slf4j-api', '1.7.25'), ('io.reactivex.rxjava2', 'rxjava', '2.1.14'), - ('software.amazon.awssdk', 'sts', '2.2.0'), - ('software.amazon.awssdk', 'aws-query-protocol', '2.2.0'), - ('software.amazon.awssdk', 'protocol-core', '2.2.0'), - ('software.amazon.awssdk', 'profiles', '2.2.0'), - ('software.amazon.awssdk', 'sdk-core', '2.2.0'), - ('com.fasterxml.jackson.core', 'jackson-core', '2.9.7'), - ('com.fasterxml.jackson.core', 'jackson-databind', '2.9.7'), - ('software.amazon.awssdk', 'auth', '2.2.0'), + ('software.amazon.awssdk', 'sts', '2.4.0'), + ('software.amazon.awssdk', 'aws-query-protocol', '2.4.0'), + ('software.amazon.awssdk', 'protocol-core', '2.4.0'), + ('software.amazon.awssdk', 'profiles', '2.4.0'), + ('software.amazon.awssdk', 'sdk-core', '2.4.0'), + ('com.fasterxml.jackson.core', 'jackson-core', '2.9.8'), + ('com.fasterxml.jackson.core', 'jackson-databind', '2.9.8'), + ('software.amazon.awssdk', 'auth', '2.4.0'), ('software.amazon', 'flow', '1.7'), - ('software.amazon.awssdk', 'http-client-spi', '2.2.0'), - ('software.amazon.awssdk', 'regions', '2.2.0'), - ('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.7'), - ('software.amazon.awssdk', 'annotations', '2.2.0'), - ('software.amazon.awssdk', 'utils', '2.2.0'), - ('software.amazon.awssdk', 'aws-core', '2.2.0'), - ('software.amazon.awssdk', 'apache-client', '2.2.0'), + ('software.amazon.awssdk', 'http-client-spi', '2.4.0'), + ('software.amazon.awssdk', 'regions', '2.4.0'), + ('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.0'), + ('software.amazon.awssdk', 'annotations', '2.4.0'), + ('software.amazon.awssdk', 'utils', '2.4.0'), + ('software.amazon.awssdk', 'aws-core', '2.4.0'), + ('software.amazon.awssdk', 'apache-client', '2.4.0'), ('org.apache.httpcomponents', 'httpclient', '4.5.6'), ('commons-codec', 'commons-codec', '1.10'), ('org.apache.httpcomponents', 'httpcore', '4.4.10'),