diff --git a/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java b/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java index a320fc7b06..63a1bfe8c6 100644 --- a/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java +++ b/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java @@ -17,18 +17,27 @@ package org.springframework.cloud.dataflow.rest.util; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.springframework.batch.core.JobParameter; import org.springframework.batch.core.JobParameters; import org.springframework.cloud.dataflow.core.DefinitionUtils; import org.springframework.cloud.dataflow.core.TaskDefinition; import org.springframework.cloud.dataflow.core.dsl.TaskParser; import org.springframework.cloud.dataflow.core.dsl.graph.Graph; +import org.springframework.http.HttpHeaders; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; @@ -40,13 +49,20 @@ * @author Ilayaperumal Gopinathan */ public class ArgumentSanitizer { + private final static Logger logger = LoggerFactory.getLogger(ArgumentSanitizer.class); - private static final String[] REGEX_PARTS = { "*", "$", "^", "+" }; + private static final String[] REGEX_PARTS = {"*", "$", "^", "+"}; private static final String REDACTION_STRING = "******"; - private static final String[] KEYS_TO_SANITIZE = { "username", "password", "secret", "key", "token", ".*credentials.*", - "vcap_services", "url" }; + private static final String[] KEYS_TO_SANITIZE = {"username", "password", "secret", "key", "token", ".*credentials.*", + "vcap_services", "url"}; + + private final static TypeReference> mapTypeReference = new TypeReference>() {}; + + private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + + private final ObjectMapper jsonMapper = new ObjectMapper(); private Pattern[] keysToSanitize; @@ -80,6 +96,10 @@ private boolean isRegex(String value) { * @return the argument with a potentially sanitized value */ public String sanitize(String argument) { + // Oracle handles an empty string as a null. + if (argument == null) { + return ""; + } int indexOfFirstEqual = argument.indexOf("="); if (indexOfFirstEqual == -1) { return argument; @@ -95,7 +115,7 @@ public String sanitize(String argument) { /** * Replaces a potential secure value with "******". * - * @param key to check for sensitive words. + * @param key to check for sensitive words. * @param value the argument to cleanse. * @return the argument with a potentially sanitized value */ @@ -118,13 +138,12 @@ public String sanitize(String key, String value) { * @return the sanitized job parameters */ public JobParameters sanitizeJobParameters(JobParameters jobParameters) { - Map newJobParameters = new HashMap<>(); - jobParameters.getParameters().forEach( (key, jobParameter) -> { + Map newJobParameters = new HashMap<>(); + jobParameters.getParameters().forEach((key, jobParameter) -> { String updatedKey = !jobParameter.isIdentifying() ? "-" + key : key; if (jobParameter.getType().equals(JobParameter.ParameterType.STRING)) { newJobParameters.put(updatedKey, new JobParameter(this.sanitize(key, jobParameter.toString()))); - } - else { + } else { newJobParameters.put(updatedKey, jobParameter); } }); @@ -138,7 +157,7 @@ public JobParameters sanitizeJobParameters(JobParameters jobParameters) { * @return Task definition text that has sensitive data redacted. */ public String sanitizeTaskDsl(TaskDefinition taskDefinition) { - if(StringUtils.isEmpty(taskDefinition.getDslText())) { + if (StringUtils.isEmpty(taskDefinition.getDslText())) { return taskDefinition.getDslText(); } TaskParser taskParser = new TaskParser(taskDefinition.getTaskName(), taskDefinition.getDslText(), true, true); @@ -147,7 +166,7 @@ public String sanitizeTaskDsl(TaskDefinition taskDefinition) { if (node.properties != null) { node.properties.keySet().stream().forEach(key -> { node.properties.put(key, - DefinitionUtils.autoQuotes(sanitize(key, node.properties.get(key)))); + DefinitionUtils.autoQuotes(sanitize(key, node.properties.get(key)))); }); } }); @@ -157,13 +176,14 @@ public String sanitizeTaskDsl(TaskDefinition taskDefinition) { /** * For all sensitive properties (e.g. key names containing words like password, secret, * key, token) replace the value with '*****' string + * * @param properties to be sanitized * @return sanitized properties */ public Map sanitizeProperties(Map properties) { if (!CollectionUtils.isEmpty(properties)) { final Map sanitizedProperties = new LinkedHashMap<>(properties.size()); - for (Map.Entry property : properties.entrySet()) { + for (Map.Entry property : properties.entrySet()) { sanitizedProperties.put(property.getKey(), this.sanitize(property.getKey(), property.getValue())); } return sanitizedProperties; @@ -174,6 +194,7 @@ public Map sanitizeProperties(Map properties) { /** * For all sensitive arguments (e.g. key names containing words like password, secret, * key, token) replace the value with '*****' string + * * @param arguments to be sanitized * @return sanitized arguments */ @@ -187,4 +208,96 @@ public List sanitizeArguments(List arguments) { } return arguments; } + + public HttpHeaders sanitizeHeaders(HttpHeaders headers) { + HttpHeaders result = new HttpHeaders(); + for (Map.Entry> entry : headers.entrySet()) { + List values = entry.getValue(); + for (String value : values) { + result.add(entry.getKey(), sanitize(entry.getKey(), value)); + } + } + return result; + } + + /** + * Will replace sensitive string value in the Map with '*****' + * + * @param input to be sanitized + * @return the sanitized map. + */ + public Map sanitizeMap(Map input) { + Map result = new HashMap<>(); + for (Map.Entry entry : input.entrySet()) { + if (entry.getValue() instanceof String) { + result.put(entry.getKey(), sanitize(entry.getKey(), (String) entry.getValue())); + } else if (entry.getValue() instanceof Map) { + Map map = (Map) entry.getValue(); + result.put(entry.getKey(), sanitizeMap(map)); + } else { + result.put(entry.getKey(), entry.getValue()); + } + } + return result; + } + + /** + * Will replace the sensitive string fields with '*****' + * + * @param input to be sanitized + * @return The sanitized JSON string + * @throws JsonProcessingException + */ + public String sanitizeJsonString(String input) throws JsonProcessingException { + if (input == null) { + return null; + } + Map data = jsonMapper.readValue(input, mapTypeReference); + return jsonMapper.writeValueAsString(sanitizeMap(data)); + } + + /** + * Will replace the sensitive string fields with '*****' + * + * @param input to be sanitized + * @return The sanitized YAML string + * @throws JsonProcessingException + */ + public String sanitizeYamlString(String input) throws JsonProcessingException { + if (input == null) { + return null; + } + Map data = yamlMapper.readValue(input, mapTypeReference); + return yamlMapper.writeValueAsString(sanitizeMap(data)); + } + + /** + * Will determine the type of data and treat as JSON or YAML to sanitize sensitive values. + * + * @param input to be sanitized + * @return the sanitized string + * @throws JsonProcessingException + */ + public String sanitizeJsonOrYamlString(String input) { + if (input == null) { + return null; + } + try { // Try parsing as JSON + return sanitizeJsonString(input); + } catch (Throwable x) { + logger.trace("Cannot parse as JSON:" + x); + } + try { + return sanitizeYamlString(input); + } catch (Throwable x) { + logger.trace("Cannot parse as YAML:" + x); + } + if (input.contains("\n")) { + return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "\n"))), "\n"); + } + if (input.contains("--")) { + return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "--"))), "--"); + } + return sanitize(input); + } } diff --git a/spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java b/spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java index f57ec71239..5c2aa1cd4d 100644 --- a/spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java +++ b/spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +30,7 @@ import org.springframework.cloud.dataflow.rest.UpdateStreamRequest; import org.springframework.cloud.dataflow.rest.resource.DeploymentStateResource; import org.springframework.cloud.dataflow.rest.resource.StreamDeploymentResource; +import org.springframework.cloud.dataflow.rest.util.ArgumentSanitizer; import org.springframework.cloud.dataflow.server.controller.support.ControllerUtils; import org.springframework.cloud.dataflow.server.repository.NoSuchStreamDefinitionException; import org.springframework.cloud.dataflow.server.repository.StreamDefinitionRepository; @@ -80,6 +82,8 @@ public class StreamDeploymentController { */ private final StreamDefinitionRepository repository; + private final ArgumentSanitizer sanitizer = new ArgumentSanitizer(); + /** * Construct a new UpdatableStreamDeploymentController, given a * {@link StreamDeploymentController} and {@link StreamService} @@ -142,7 +146,17 @@ public ResponseEntity manifest(@PathVariable("name") String name, @RequestMapping(path = "/history/{name}", method = RequestMethod.GET) @ResponseStatus(HttpStatus.OK) public Collection history(@PathVariable("name") String releaseName) { - return this.streamService.history(releaseName); + return this.streamService.history(releaseName) + .stream() + .map(this::sanitizeRelease) + .collect(Collectors.toList()); + } + + private Release sanitizeRelease(Release release) { + if (release.getConfigValues() != null && StringUtils.hasText(release.getConfigValues().getRaw())) { + release.getConfigValues().setRaw(sanitizer.sanitizeJsonOrYamlString(release.getConfigValues().getRaw())); + } + return release; } @RequestMapping(path = "/platform/list", method = RequestMethod.GET)