From 5d1547625a11690dc92d29b498c2a1b59598f95b Mon Sep 17 00:00:00 2001 From: Justin Black Date: Wed, 28 Sep 2022 15:29:07 -0700 Subject: [PATCH 1/6] Fixes the regex pattern used for regex extraction --- .../org/openapitools/codegen/languages/PythonClientCodegen.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java index 9ed711c6649..3ab3fb8045f 100644 --- a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java +++ b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java @@ -1895,7 +1895,7 @@ private String toExampleValueRecursive(String modelName, Schema schema, Object o RxGen does not support our ECMA dialect https://github.com/curious-odd-man/RgxGen/issues/56 So strip off the leading / and trailing / and turn on ignore case if we have it */ - Pattern valueExtractor = Pattern.compile("^/?(.+?)/?(.?)$"); + Pattern valueExtractor = Pattern.compile("^/?(.+?)/?([gim]?)$"); Matcher m = valueExtractor.matcher(pattern); RgxGen rgxGen = null; if (m.find()) { From d2bc8564e0cc22479b7d92e7ef53d86968a6985a Mon Sep 17 00:00:00 2001 From: Justin Black Date: Wed, 28 Sep 2022 16:21:54 -0700 Subject: [PATCH 2/6] Centralized regex handling --- .../languages/PythonClientCodegen.java | 109 +++++++++++------- 1 file changed, 66 insertions(+), 43 deletions(-) diff --git a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java index 3ab3fb8045f..2dc3726c2fc 100644 --- a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java +++ b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java @@ -1891,34 +1891,21 @@ private String toExampleValueRecursive(String modelName, Schema schema, Object o example = "2"; } else if (StringUtils.isNotBlank(schema.getPattern())) { String pattern = schema.getPattern(); + List results = getPatternAndModifiers(pattern); + String extractedPattern = (String) results.get(0); + List regexFlags = (List) results.get(1); /* RxGen does not support our ECMA dialect https://github.com/curious-odd-man/RgxGen/issues/56 So strip off the leading / and trailing / and turn on ignore case if we have it */ - Pattern valueExtractor = Pattern.compile("^/?(.+?)/?([gim]?)$"); - Matcher m = valueExtractor.matcher(pattern); RgxGen rgxGen = null; - if (m.find()) { - int groupCount = m.groupCount(); - if (groupCount == 1) { - // only pattern found - String isolatedPattern = m.group(1); - rgxGen = new RgxGen(isolatedPattern); - } else if (groupCount == 2) { - // patterns and flag found - String isolatedPattern = m.group(1); - String flags = m.group(2); - if (flags.contains("i")) { - rgxGen = new RgxGen(isolatedPattern); - RgxGenProperties properties = new RgxGenProperties(); - RgxGenOption.CASE_INSENSITIVE.setInProperties(properties, true); - rgxGen.setProperties(properties); - } else { - rgxGen = new RgxGen(isolatedPattern); - } - } + if (regexFlags.size() > 0 && regexFlags.contains("i")) { + rgxGen = new RgxGen(extractedPattern); + RgxGenProperties properties = new RgxGenProperties(); + RgxGenOption.CASE_INSENSITIVE.setInProperties(properties, true); + rgxGen.setProperties(properties); } else { - rgxGen = new RgxGen(pattern); + rgxGen = new RgxGen(extractedPattern); } // this seed makes it so if we have [a-z] we pick a @@ -2358,6 +2345,16 @@ protected void updatePropertyForString(CodegenProperty property, Schema p) { property.pattern = toRegularExpression(p.getPattern()); } + @Override + public String toRegularExpression(String pattern) { + if (pattern == null) { + return null; + } + List results = getPatternAndModifiers(pattern); + String extractedPattern = (String) results.get(0); + return extractedPattern; + } + protected void updatePropertyForNumber(CodegenProperty property, Schema p) { property.setIsNumber(true); // float and double differentiation is determined with format info @@ -2482,6 +2479,46 @@ public ModelsMap postProcessModels(ModelsMap objs) { return postProcessModelsEnum(objs); } + /** + * @param pattern the regex pattern + * @return List> + */ + private List getPatternAndModifiers(String pattern) { + /* + Notes: + RxGen does not support our ECMA dialect https://github.com/curious-odd-man/RgxGen/issues/56 + So strip off the leading / and trailing / and turn on ignore case if we have it + + json schema test cases omit the leading and trailing /s, so make sure that the regex allows that + */ + Pattern valueExtractor = Pattern.compile("^/?(.+?)/?([sim]?)$"); + Matcher m = valueExtractor.matcher(pattern); + if (m.find()) { + int groupCount = m.groupCount(); + if (groupCount == 1) { + // only pattern found + String isolatedPattern = m.group(1); + return Arrays.asList(isolatedPattern, null); + } else if (groupCount == 2) { + List modifiers = new ArrayList(); + // patterns and flag found + String isolatedPattern = m.group(1); + String flags = m.group(2); + if (flags.contains("s")) { + modifiers.add("DOTALL"); + } + if (flags.contains("i")) { + modifiers.add("IGNORECASE"); + } + if (flags.contains("m")) { + modifiers.add("MULTILINE"); + } + return Arrays.asList(isolatedPattern, modifiers); + } + } + return Arrays.asList(pattern, new ArrayList()); + } + /* * The OpenAPI pattern spec follows the Perl convention and style of modifiers. Python * does not support this in as natural a way so it needs to convert it. See @@ -2489,28 +2526,14 @@ public ModelsMap postProcessModels(ModelsMap objs) { */ public void postProcessPattern(String pattern, Map vendorExtensions) { if (pattern != null) { - int regexLength = pattern.length(); - String regex = pattern; - int i = pattern.lastIndexOf('/'); - if (regexLength >= 2 && pattern.charAt(0) == '/' && i != -1) { - // json schema tests do not include the leading and trailing slashes - // so I do not think that they are required - regex = pattern.substring(1, i); - } - regex = regex.replace("'", "\\'"); - List modifiers = new ArrayList(); - - if (i != -1) { - for (char c : pattern.substring(i).toCharArray()) { - if (regexModifiers.containsKey(c)) { - String modifier = regexModifiers.get(c); - modifiers.add(modifier); - } - } - } + List results = getPatternAndModifiers(pattern); + String extractedPattern = (String) results.get(0); + List modifiers = (List) results.get(1); - vendorExtensions.put("x-regex", regex); - vendorExtensions.put("x-modifiers", modifiers); + vendorExtensions.put("x-regex", extractedPattern); + if (modifiers.size() > 0) { + vendorExtensions.put("x-modifiers", modifiers); + } } } From 058ad604cfbdde88e3e5e0d06415014d1069d4ff Mon Sep 17 00:00:00 2001 From: Justin Black Date: Wed, 28 Sep 2022 16:59:16 -0700 Subject: [PATCH 3/6] Updates pattern handling in python-experimental --- .../openapitools/codegen/languages/PythonClientCodegen.java | 6 +++++- .../client/petstore/python/docs/apis/tags/FakeApi.md | 4 ++-- .../client/petstore/python/petstore_api/model/apple.py | 3 --- .../petstore/python/petstore_api/model/format_test.py | 6 ------ .../client/petstore/python/petstore_api/paths/fake/post.py | 3 --- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java index 2dc3726c2fc..0dd27048714 100644 --- a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java +++ b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java @@ -1485,7 +1485,11 @@ protected Object processTestExampleData(Object value) { @Override public CodegenModel fromModel(String name, Schema sc) { CodegenModel cm = super.fromModel(name, sc); - + if (sc.getPattern() != null) { + postProcessPattern(sc.getPattern(), cm.vendorExtensions); + String pattern = (String) cm.vendorExtensions.get("x-regex"); + cm.setPattern(pattern); + } if (cm.isNullable) { cm.setIsNull(true); cm.isNullable = false; diff --git a/samples/openapi3/client/petstore/python/docs/apis/tags/FakeApi.md b/samples/openapi3/client/petstore/python/docs/apis/tags/FakeApi.md index a368406d7b2..ad2384b1241 100644 --- a/samples/openapi3/client/petstore/python/docs/apis/tags/FakeApi.md +++ b/samples/openapi3/client/petstore/python/docs/apis/tags/FakeApi.md @@ -949,8 +949,8 @@ with petstore_api.ApiClient(configuration) as api_client: number=32.1, _float=3.14, double=67.8, - string="A", - pattern_without_delimiter="Aj", + string="a", + pattern_without_delimiter="AUR,rZ#UM/?R,Fp^l6$ARjbhJk C>", byte='YQ==', binary=open('/path/to/file', 'rb'), date="1970-01-01", diff --git a/samples/openapi3/client/petstore/python/petstore_api/model/apple.py b/samples/openapi3/client/petstore/python/petstore_api/model/apple.py index 638edc274dc..c28dc7d92b3 100644 --- a/samples/openapi3/client/petstore/python/petstore_api/model/apple.py +++ b/samples/openapi3/client/petstore/python/petstore_api/model/apple.py @@ -63,9 +63,6 @@ class origin( class MetaOapg: regex=[{ 'pattern': r'^[A-Z\s]*$', # noqa: E501 - 'flags': ( - re.IGNORECASE - ) }] __annotations__ = { "cultivar": cultivar, diff --git a/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py b/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py index b16b10a0488..a95920230c8 100644 --- a/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py +++ b/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py @@ -148,9 +148,6 @@ class string( class MetaOapg: regex=[{ 'pattern': r'[a-z]', # noqa: E501 - 'flags': ( - re.IGNORECASE - ) }] binary = schemas.BinarySchema dateTime = schemas.DateTimeSchema @@ -177,9 +174,6 @@ class pattern_with_digits_and_delimiter( class MetaOapg: regex=[{ 'pattern': r'^image_\d{1,3}$', # noqa: E501 - 'flags': ( - re.IGNORECASE - ) }] noneProp = schemas.NoneSchema __annotations__ = { diff --git a/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py b/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py index b732e5d6100..6443f046f53 100644 --- a/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py +++ b/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py @@ -107,9 +107,6 @@ class string( class MetaOapg: regex=[{ 'pattern': r'[a-z]', # noqa: E501 - 'flags': ( - re.IGNORECASE - ) }] From a4cf7a076cd4ad0396186b7abbe2c88c6aa886a0 Mon Sep 17 00:00:00 2001 From: Justin Black Date: Wed, 28 Sep 2022 17:22:04 -0700 Subject: [PATCH 4/6] Fixes python regex flags --- .../openapitools/codegen/languages/PythonClientCodegen.java | 4 +++- .../client/petstore/python/petstore_api/model/apple.py | 3 +++ .../petstore/python/petstore_api/model/format_test.py | 6 ++++++ .../client/petstore/python/petstore_api/paths/fake/post.py | 3 +++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java index 0dd27048714..e6183ec7e81 100644 --- a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java +++ b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java @@ -1041,7 +1041,9 @@ public CodegenProperty fromProperty(String name, Schema p, boolean required, boo cp.isNullable = false; cp.setHasMultipleTypes(true); } - postProcessPattern(cp.pattern, cp.vendorExtensions); + if (p.getPattern() != null) { + postProcessPattern(p.getPattern(), cp.vendorExtensions); + } // if we have a property that has a difficult name, either: // 1. name is reserved, like class int float // 2. name is invalid in python like '3rd' or 'Content-Type' diff --git a/samples/openapi3/client/petstore/python/petstore_api/model/apple.py b/samples/openapi3/client/petstore/python/petstore_api/model/apple.py index c28dc7d92b3..638edc274dc 100644 --- a/samples/openapi3/client/petstore/python/petstore_api/model/apple.py +++ b/samples/openapi3/client/petstore/python/petstore_api/model/apple.py @@ -63,6 +63,9 @@ class origin( class MetaOapg: regex=[{ 'pattern': r'^[A-Z\s]*$', # noqa: E501 + 'flags': ( + re.IGNORECASE + ) }] __annotations__ = { "cultivar": cultivar, diff --git a/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py b/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py index a95920230c8..b16b10a0488 100644 --- a/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py +++ b/samples/openapi3/client/petstore/python/petstore_api/model/format_test.py @@ -148,6 +148,9 @@ class string( class MetaOapg: regex=[{ 'pattern': r'[a-z]', # noqa: E501 + 'flags': ( + re.IGNORECASE + ) }] binary = schemas.BinarySchema dateTime = schemas.DateTimeSchema @@ -174,6 +177,9 @@ class pattern_with_digits_and_delimiter( class MetaOapg: regex=[{ 'pattern': r'^image_\d{1,3}$', # noqa: E501 + 'flags': ( + re.IGNORECASE + ) }] noneProp = schemas.NoneSchema __annotations__ = { diff --git a/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py b/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py index 6443f046f53..b732e5d6100 100644 --- a/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py +++ b/samples/openapi3/client/petstore/python/petstore_api/paths/fake/post.py @@ -107,6 +107,9 @@ class string( class MetaOapg: regex=[{ 'pattern': r'[a-z]', # noqa: E501 + 'flags': ( + re.IGNORECASE + ) }] From f5944d5068f5c548be9981c356a5ba8d08f5b02e Mon Sep 17 00:00:00 2001 From: Justin Black Date: Thu, 29 Sep 2022 11:33:46 -0700 Subject: [PATCH 5/6] Adds tow regex tests --- .../languages/PythonClientCodegen.java | 8 ++--- .../codegen/python/PythonClientTest.java | 33 +++++++++++++++++++ .../src/test/resources/3_0/11_regex.yaml | 27 +++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 modules/openapi-json-schema-generator/src/test/resources/3_0/11_regex.yaml diff --git a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java index e6183ec7e81..860aee83784 100644 --- a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java +++ b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java @@ -2497,7 +2497,7 @@ private List getPatternAndModifiers(String pattern) { json schema test cases omit the leading and trailing /s, so make sure that the regex allows that */ - Pattern valueExtractor = Pattern.compile("^/?(.+?)/?([sim]?)$"); + Pattern valueExtractor = Pattern.compile("^/?(.+?)/?([simu]{0,4})$"); Matcher m = valueExtractor.matcher(pattern); if (m.find()) { int groupCount = m.groupCount(); @@ -2511,13 +2511,13 @@ private List getPatternAndModifiers(String pattern) { String isolatedPattern = m.group(1); String flags = m.group(2); if (flags.contains("s")) { - modifiers.add("DOTALL"); + modifiers.add("s"); } if (flags.contains("i")) { - modifiers.add("IGNORECASE"); + modifiers.add("i"); } if (flags.contains("m")) { - modifiers.add("MULTILINE"); + modifiers.add("m"); } return Arrays.asList(isolatedPattern, modifiers); } diff --git a/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java b/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java index 82c86262dbf..a9efc0ee6d1 100644 --- a/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java +++ b/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java @@ -32,6 +32,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -164,4 +165,36 @@ public void testApisNotGenerated() throws Exception { Assert.assertFalse(Files.isDirectory(pathThatShouldNotExist)); output.deleteOnExit(); } + + @Test + public void testRegexWithoutTrailingSlashWorks() { + OpenAPI openAPI = TestUtils.parseFlattenSpec("src/test/resources/3_0/11_regex.yaml"); + PythonClientCodegen codegen = new PythonClientCodegen(); + codegen.setOpenAPI(openAPI); + + String modelName = "UUID"; + Schema schema = openAPI.getComponents().getSchemas().get(modelName); + + CodegenModel cm = codegen.fromModel(modelName, schema); + String expectedRegexPattern = "[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"; + Assert.assertEquals(cm.getPattern(), expectedRegexPattern); + Assert.assertEquals(cm.vendorExtensions.get("x-regex"), expectedRegexPattern); + Assert.assertFalse(cm.vendorExtensions.containsKey("x-modifiers")); + } + + @Test + public void testRegexWithMultipleFlagsWorks() { + OpenAPI openAPI = TestUtils.parseFlattenSpec("src/test/resources/3_0/11_regex.yaml"); + PythonClientCodegen codegen = new PythonClientCodegen(); + codegen.setOpenAPI(openAPI); + + String modelName = "StringWithRegexWithThreeFlags"; + Schema schema = openAPI.getComponents().getSchemas().get(modelName); + + CodegenModel cm = codegen.fromModel(modelName, schema); + String expectedRegexPattern = "a."; + Assert.assertEquals(cm.getPattern(), expectedRegexPattern); + Assert.assertEquals(cm.vendorExtensions.get("x-regex"), expectedRegexPattern); + Assert.assertEquals(cm.vendorExtensions.get("x-modifiers"), Arrays.asList("s", "i", "m")); + } } diff --git a/modules/openapi-json-schema-generator/src/test/resources/3_0/11_regex.yaml b/modules/openapi-json-schema-generator/src/test/resources/3_0/11_regex.yaml new file mode 100644 index 00000000000..1fd9eeadc93 --- /dev/null +++ b/modules/openapi-json-schema-generator/src/test/resources/3_0/11_regex.yaml @@ -0,0 +1,27 @@ +openapi: 3.0.3 +info: + title: Test + version: 1.0.0-SNAPSHOT +paths: + /test: + get: + tags: + - Test Resource + parameters: + - name: uuid + in: query + schema: + $ref: '#/components/schemas/UUID' + responses: + "200": + description: OK +components: + schemas: + UUID: + format: uuid + pattern: "[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}" + type: string + StringWithRegexWithThreeFlags: + format: uuid + pattern: "/a./sim" + type: string \ No newline at end of file From 9ef28e14544fea9c2317bc00afafc238d3e75dc0 Mon Sep 17 00:00:00 2001 From: Justin Black Date: Thu, 29 Sep 2022 11:50:21 -0700 Subject: [PATCH 6/6] Samples regenerated, test updated --- .../openapitools/codegen/languages/PythonClientCodegen.java | 6 +++--- .../org/openapitools/codegen/python/PythonClientTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java index 860aee83784..2c6b82ac411 100644 --- a/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java +++ b/modules/openapi-json-schema-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java @@ -2511,13 +2511,13 @@ private List getPatternAndModifiers(String pattern) { String isolatedPattern = m.group(1); String flags = m.group(2); if (flags.contains("s")) { - modifiers.add("s"); + modifiers.add("DOTALL"); } if (flags.contains("i")) { - modifiers.add("i"); + modifiers.add("IGNORECASE"); } if (flags.contains("m")) { - modifiers.add("m"); + modifiers.add("MULTILINE"); } return Arrays.asList(isolatedPattern, modifiers); } diff --git a/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java b/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java index a9efc0ee6d1..2b7aabae46a 100644 --- a/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java +++ b/modules/openapi-json-schema-generator/src/test/java/org/openapitools/codegen/python/PythonClientTest.java @@ -195,6 +195,6 @@ public void testRegexWithMultipleFlagsWorks() { String expectedRegexPattern = "a."; Assert.assertEquals(cm.getPattern(), expectedRegexPattern); Assert.assertEquals(cm.vendorExtensions.get("x-regex"), expectedRegexPattern); - Assert.assertEquals(cm.vendorExtensions.get("x-modifiers"), Arrays.asList("s", "i", "m")); + Assert.assertEquals(cm.vendorExtensions.get("x-modifiers"), Arrays.asList("DOTALL", "IGNORECASE", "MULTILINE")); } }