Skip to content
This repository was archived by the owner on May 14, 2025. It is now read-only.

Commit 439f26a

Browse files
author
Corneil du Plessis
authored
Add sanitizer to history output. (#4968)
Fixes - #4964
1 parent b69f45f commit 439f26a

File tree

5 files changed

+146
-34
lines changed

5 files changed

+146
-34
lines changed

‎spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,20 @@
1717
package org.springframework.cloud.dataflow.rest.util;
1818

1919
import java.util.ArrayList;
20+
import java.util.Arrays;
2021
import java.util.HashMap;
2122
import java.util.LinkedHashMap;
2223
import java.util.List;
2324
import java.util.Map;
2425
import java.util.regex.Pattern;
2526

27+
import com.fasterxml.jackson.core.JsonProcessingException;
28+
import com.fasterxml.jackson.core.type.TypeReference;
29+
import com.fasterxml.jackson.databind.ObjectMapper;
30+
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
31+
import org.slf4j.Logger;
32+
import org.slf4j.LoggerFactory;
33+
2634
import org.springframework.batch.core.JobParameter;
2735
import org.springframework.batch.core.JobParameters;
2836
import org.springframework.cloud.dataflow.core.DefinitionUtils;
@@ -39,8 +47,10 @@
3947
* @author Glenn Renfro
4048
* @author Gunnar Hillert
4149
* @author Ilayaperumal Gopinathan
50+
* @author Corneil du Plessis
4251
*/
4352
public class ArgumentSanitizer {
53+
private final static Logger logger = LoggerFactory.getLogger(ArgumentSanitizer.class);
4454

4555
private static final String[] REGEX_PARTS = {"*", "$", "^", "+"};
4656

@@ -204,4 +214,75 @@ public HttpHeaders sanitizeHeaders(HttpHeaders headers) {
204214
}
205215
return result;
206216
}
217+
/**
218+
* Will replace sensitive string value in the Map with '*****'
219+
*
220+
* @param input to be sanitized
221+
* @return the sanitized map.
222+
*/
223+
public Map<String, Object> sanitizeMap(Map<String, Object> input) {
224+
Map<String, Object> result = new HashMap<>();
225+
for (Map.Entry<String, Object> entry : input.entrySet()) {
226+
if (entry.getValue() instanceof String) {
227+
result.put(entry.getKey(), sanitize(entry.getKey(), (String) entry.getValue()));
228+
} else if (entry.getValue() instanceof Map) {
229+
Map<String, Object> map = (Map<String, Object>) entry.getValue();
230+
result.put(entry.getKey(), sanitizeMap(map));
231+
} else {
232+
result.put(entry.getKey(), entry.getValue());
233+
}
234+
}
235+
return result;
236+
}
237+
238+
/**
239+
* Will replace the sensitive string fields with '*****'
240+
*
241+
* @param input to be sanitized
242+
* @return The sanitized JSON string
243+
* @throws JsonProcessingException
244+
*/
245+
public String sanitizeJsonString(String input) throws JsonProcessingException {
246+
ObjectMapper mapper = new ObjectMapper();
247+
Map<String, Object> data = mapper.readValue(input, new TypeReference<Map<String, Object>>() {
248+
});
249+
return mapper.writeValueAsString(sanitizeMap(data));
250+
}
251+
252+
/**
253+
* Will replace the sensitive string fields with '*****'
254+
*
255+
* @param input to be sanitized
256+
* @return The sanitized YAML string
257+
* @throws JsonProcessingException
258+
*/
259+
public String sanitizeYamlString(String input) throws JsonProcessingException {
260+
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
261+
Map<String, Object> data = mapper.readValue(input, new TypeReference<Map<String, Object>>() {});
262+
return mapper.writeValueAsString(sanitizeMap(data));
263+
}
264+
/**
265+
* Will determine the type of data and treat as JSON or YAML to sanitize sensitive values.
266+
*
267+
* @param input to be sanitized
268+
* @return the sanitized string
269+
* @throws JsonProcessingException
270+
*/
271+
public String sanitizeJsonOrYamlString(String input) throws JsonProcessingException {
272+
273+
try { // Try parsing as JSON
274+
return sanitizeJsonString(input);
275+
} catch (Throwable x) {
276+
logger.debug("Cannot parse as JSON:" + x);
277+
}
278+
try {
279+
return sanitizeYamlString(input);
280+
} catch (Throwable x) {
281+
logger.debug("Cannot parse as YAML:" + x);
282+
}
283+
if (input.contains("\n")) {
284+
return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "\n"))), "\n");
285+
}
286+
return sanitize(input);
287+
}
207288
}

‎spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
import java.util.Arrays;
2020
import java.util.Collection;
2121
import java.util.Map;
22+
import java.util.stream.Collectors;
2223

24+
import com.fasterxml.jackson.core.JsonProcessingException;
2325
import org.slf4j.Logger;
2426
import org.slf4j.LoggerFactory;
2527

@@ -29,6 +31,7 @@
2931
import org.springframework.cloud.dataflow.rest.UpdateStreamRequest;
3032
import org.springframework.cloud.dataflow.rest.resource.DeploymentStateResource;
3133
import org.springframework.cloud.dataflow.rest.resource.StreamDeploymentResource;
34+
import org.springframework.cloud.dataflow.rest.util.ArgumentSanitizer;
3235
import org.springframework.cloud.dataflow.server.controller.support.ControllerUtils;
3336
import org.springframework.cloud.dataflow.server.repository.NoSuchStreamDefinitionException;
3437
import org.springframework.cloud.dataflow.server.repository.StreamDefinitionRepository;
@@ -142,7 +145,19 @@ public ResponseEntity<String> manifest(@PathVariable("name") String name,
142145
@RequestMapping(path = "/history/{name}", method = RequestMethod.GET)
143146
@ResponseStatus(HttpStatus.OK)
144147
public Collection<Release> history(@PathVariable("name") String releaseName) {
145-
return this.streamService.history(releaseName);
148+
ArgumentSanitizer sanitizer = new ArgumentSanitizer();
149+
return this.streamService.history(releaseName)
150+
.stream()
151+
.map(release -> {
152+
if (release.getConfigValues() != null && StringUtils.hasText(release.getConfigValues().getRaw())) {
153+
try {
154+
release.getConfigValues().setRaw(sanitizer.sanitizeJsonOrYamlString(release.getConfigValues().getRaw()));
155+
} catch (JsonProcessingException e) {
156+
logger.error("history:exception:" + e, e);
157+
}
158+
}
159+
return release;
160+
}).collect(Collectors.toList());
146161
}
147162

148163
@RequestMapping(path = "/platform/list", method = RequestMethod.GET)

‎spring-cloud-dataflow-server-core/src/test/java/org/springframework/cloud/dataflow/server/support/ArgumentSanitizerTest.java

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616

1717
package org.springframework.cloud.dataflow.server.support;
1818

19+
import java.io.IOException;
20+
import java.io.InputStreamReader;
21+
import java.io.Reader;
1922
import java.util.ArrayList;
2023
import java.util.Date;
2124
import java.util.LinkedHashMap;
@@ -30,10 +33,14 @@
3033
import org.springframework.batch.core.JobParameters;
3134
import org.springframework.cloud.dataflow.core.TaskDefinition;
3235
import org.springframework.cloud.dataflow.rest.util.ArgumentSanitizer;
36+
import org.springframework.core.io.DefaultResourceLoader;
37+
import org.springframework.core.io.Resource;
38+
import org.springframework.util.FileCopyUtils;
3339

3440
/**
3541
* @author Christian Tzolov
3642
* @author Ilayaperumal Gopinathan
43+
* @author Corneil du Plessis
3744
*/
3845
public class ArgumentSanitizerTest {
3946

@@ -151,38 +158,31 @@ public void testMultipartProperty() {
151158
Assert.assertEquals("--one.two.password=******", sanitizer.sanitize("--one.two.password=boza"));
152159
Assert.assertEquals("--one_two_password=******", sanitizer.sanitize("--one_two_password=boza"));
153160
}
161+
private String loadStringFromResource(String uri) throws IOException {
162+
Resource resource = new DefaultResourceLoader().getResource(uri);
163+
try (Reader reader = new InputStreamReader(resource.getInputStream())) {
164+
return FileCopyUtils.copyToString(reader);
165+
}
166+
}
167+
@Test
168+
public void testJsonData() throws IOException {
169+
String input = loadStringFromResource("classpath:sanitizer1.json");
170+
String output = sanitizer.sanitizeJsonOrYamlString(input);
171+
System.out.println("Read:" + input);
172+
System.out.println("Sanitized:" + output);
173+
Assert.assertTrue(output.contains("*****"));
174+
Assert.assertFalse(output.contains("54321"));
175+
176+
}
177+
178+
@Test
179+
public void testYamlData() throws IOException {
180+
String input = loadStringFromResource("classpath:sanitizer2.yaml");
181+
String output = sanitizer.sanitizeJsonOrYamlString(input);
182+
System.out.println("Read:" + input);
183+
System.out.println("Sanitized:" + output);
184+
Assert.assertTrue(output.contains("*****"));
185+
Assert.assertFalse(output.contains("54321"));
186+
}
154187

155-
// @Test
156-
// public void testHierarchicalPropertyNames() {
157-
// Assert.assertEquals("time --password='******' | log",
158-
// sanitizer.sanitizeStream(new StreamDefinition("stream", "time --password=bar | log")));
159-
// }
160-
//
161-
// @Test
162-
// public void testStreamPropertyOrder() {
163-
// Assert.assertEquals("time --some.password='******' --another-secret='******' | log",
164-
// sanitizer.sanitizeStream(new StreamDefinition("stream", "time --some.password=foobar --another-secret=kenny | log")));
165-
// }
166-
//
167-
// @Test
168-
// public void testStreamMatcherWithHyphenDotChar() {
169-
// Assert.assertEquals("twitterstream --twitter.credentials.access-token-secret='******' "
170-
// + "--twitter.credentials.access-token='******' --twitter.credentials.consumer-secret='******' "
171-
// + "--twitter.credentials.consumer-key='******' | "
172-
// + "filter --expression=#jsonPath(payload,'$.lang')=='en' | "
173-
// + "twitter-sentiment --vocabulary=https://dl.bintray.com/test --model-fetch=output/test "
174-
// + "--model=https://dl.bintray.com/test | field-value-counter --field-name=sentiment --name=sentiment",
175-
// sanitizer.sanitizeStream(new StreamDefinition("stream", "twitterstream "
176-
// + "--twitter.credentials.consumer-key=dadadfaf --twitter.credentials.consumer-secret=dadfdasfdads "
177-
// + "--twitter.credentials.access-token=58849055-dfdae "
178-
// + "--twitter.credentials.access-token-secret=deteegdssa4466 | filter --expression='#jsonPath(payload,''$.lang'')==''en''' | "
179-
// + "twitter-sentiment --vocabulary=https://dl.bintray.com/test --model-fetch=output/test --model=https://dl.bintray.com/test | "
180-
// + "field-value-counter --field-name=sentiment --name=sentiment")));
181-
// }
182-
//
183-
// @Test
184-
// public void testStreamSanitizeOriginalDsl() {
185-
// StreamDefinition streamDefinition = new StreamDefinition("test", "time --password='******' | log --password='******'", "time --password='******' | log");
186-
// Assert.assertEquals("time --password='******' | log", sanitizer.sanitizeOriginalStreamDsl(streamDefinition));
187-
// }
188188
}

‎spring-cloud-dataflow-server-core/src/test/resources/sanitizer1.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"password": "54321",
3+
"username": "user21",
4+
"server": "localhost",
5+
"aws": {
6+
"secretKey": "5432111",
7+
"keyId": "key12"
8+
}
9+
}

‎spring-cloud-dataflow-server-core/src/test/resources/sanitizer2.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
properties:
2+
password: '54321'
3+
username: 'user21'
4+
server: 'localhost'
5+
aws:
6+
secretKey: '5432111'
7+
keyId: 'key12'

0 commit comments

Comments
 (0)