Sanitize stream history data

Backport sanitizer (#4976)

Author: Corneil du Plessis

spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java

@@ -17,18 +17,27 @@
 package org.springframework.cloud.dataflow.rest.util;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.springframework.batch.core.JobParameter;
 import org.springframework.batch.core.JobParameters;
 import org.springframework.cloud.dataflow.core.DefinitionUtils;
 import org.springframework.cloud.dataflow.core.TaskDefinition;
 import org.springframework.cloud.dataflow.core.dsl.TaskParser;
 import org.springframework.cloud.dataflow.core.dsl.graph.Graph;
+import org.springframework.http.HttpHeaders;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
@@ -40,13 +49,20 @@
  * @author Ilayaperumal Gopinathan
  */
 public class ArgumentSanitizer {
+	private final static Logger logger = LoggerFactory.getLogger(ArgumentSanitizer.class);
 
-	private static final String[] REGEX_PARTS = { "*", "$", "^", "+" };
+	private static final String[] REGEX_PARTS = {"*", "$", "^", "+"};
 
 	private static final String REDACTION_STRING = "******";
 
-	private static final String[] KEYS_TO_SANITIZE = { "username", "password", "secret", "key", "token", ".*credentials.*",
-			"vcap_services", "url" };
+	private static final String[] KEYS_TO_SANITIZE = {"username", "password", "secret", "key", "token", ".*credentials.*",
+		"vcap_services", "url"};
+
+	private final static TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() {};
+
+	private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
+
+	private final ObjectMapper jsonMapper = new ObjectMapper();
 
 	private Pattern[] keysToSanitize;
 
@@ -80,6 +96,10 @@ private boolean isRegex(String value) {
 	 * @return the argument with a potentially sanitized value
 	 */
 	public String sanitize(String argument) {
+		// Oracle handles an empty string as a null.
+		if (argument == null) {
+			return "";
+		}
 		int indexOfFirstEqual = argument.indexOf("=");
 		if (indexOfFirstEqual == -1) {
 			return argument;
@@ -95,7 +115,7 @@ public String sanitize(String argument) {
 	/**
 	 * Replaces a potential secure value with "******".
 	 *
-	 * @param key to check for sensitive words.
+	 * @param key   to check for sensitive words.
 	 * @param value the argument to cleanse.
 	 * @return the argument with a potentially sanitized value
 	 */
@@ -118,13 +138,12 @@ public String sanitize(String key, String value) {
 	 * @return the sanitized job parameters
 	 */
 	public JobParameters sanitizeJobParameters(JobParameters jobParameters) {
-		Map<String,JobParameter> newJobParameters = new HashMap<>();
-		jobParameters.getParameters().forEach( (key, jobParameter) -> {
+		Map<String, JobParameter> newJobParameters = new HashMap<>();
+		jobParameters.getParameters().forEach((key, jobParameter) -> {
 			String updatedKey = !jobParameter.isIdentifying() ? "-" + key : key;
 			if (jobParameter.getType().equals(JobParameter.ParameterType.STRING)) {
 				newJobParameters.put(updatedKey, new JobParameter(this.sanitize(key, jobParameter.toString())));
-			}
-			else {
+			} else {
 				newJobParameters.put(updatedKey, jobParameter);
 			}
 		});
@@ -138,7 +157,7 @@ public JobParameters sanitizeJobParameters(JobParameters jobParameters) {
 	 * @return Task definition text that has sensitive data redacted.
 	 */
 	public String sanitizeTaskDsl(TaskDefinition taskDefinition) {
-		if(StringUtils.isEmpty(taskDefinition.getDslText())) {
+		if (StringUtils.isEmpty(taskDefinition.getDslText())) {
 			return taskDefinition.getDslText();
 		}
 		TaskParser taskParser = new TaskParser(taskDefinition.getTaskName(), taskDefinition.getDslText(), true, true);
@@ -147,7 +166,7 @@ public String sanitizeTaskDsl(TaskDefinition taskDefinition) {
 			if (node.properties != null) {
 				node.properties.keySet().stream().forEach(key -> {
 					node.properties.put(key,
-							DefinitionUtils.autoQuotes(sanitize(key, node.properties.get(key))));
+						DefinitionUtils.autoQuotes(sanitize(key, node.properties.get(key))));
 				});
 			}
 		});
@@ -157,13 +176,14 @@ public String sanitizeTaskDsl(TaskDefinition taskDefinition) {
 	/**
 	 * For all sensitive properties (e.g. key names containing words like password, secret,
 	 * key, token) replace the value with '*****' string
+	 *
 	 * @param properties to be sanitized
 	 * @return sanitized properties
 	 */
 	public Map<String, String> sanitizeProperties(Map<String, String> properties) {
 		if (!CollectionUtils.isEmpty(properties)) {
 			final Map<String, String> sanitizedProperties = new LinkedHashMap<>(properties.size());
-			for (Map.Entry<String, String > property : properties.entrySet()) {
+			for (Map.Entry<String, String> property : properties.entrySet()) {
 				sanitizedProperties.put(property.getKey(), this.sanitize(property.getKey(), property.getValue()));
 			}
 			return sanitizedProperties;
@@ -174,6 +194,7 @@ public Map<String, String> sanitizeProperties(Map<String, String> properties) {
 	/**
 	 * For all sensitive arguments (e.g. key names containing words like password, secret,
 	 * key, token) replace the value with '*****' string
+	 *
 	 * @param arguments to be sanitized
 	 * @return sanitized arguments
 	 */
@@ -187,4 +208,96 @@ public List<String> sanitizeArguments(List<String> arguments) {
 		}
 		return arguments;
 	}
+
+	public HttpHeaders sanitizeHeaders(HttpHeaders headers) {
+		HttpHeaders result = new HttpHeaders();
+		for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
+			List<String> values = entry.getValue();
+			for (String value : values) {
+				result.add(entry.getKey(), sanitize(entry.getKey(), value));
+			}
+		}
+		return result;
+	}
+
+	/**
+	 * Will replace sensitive string value in the Map with '*****'
+	 *
+	 * @param input to be sanitized
+	 * @return the sanitized map.
+	 */
+	public Map<String, Object> sanitizeMap(Map<String, Object> input) {
+		Map<String, Object> result = new HashMap<>();
+		for (Map.Entry<String, Object> entry : input.entrySet()) {
+			if (entry.getValue() instanceof String) {
+				result.put(entry.getKey(), sanitize(entry.getKey(), (String) entry.getValue()));
+			} else if (entry.getValue() instanceof Map) {
+				Map<String, Object> map = (Map<String, Object>) entry.getValue();
+				result.put(entry.getKey(), sanitizeMap(map));
+			} else {
+				result.put(entry.getKey(), entry.getValue());
+			}
+		}
+		return result;
+	}
+
+	/**
+	 * Will replace the sensitive string fields with '*****'
+	 *
+	 * @param input to be sanitized
+	 * @return The sanitized JSON string
+	 * @throws JsonProcessingException
+	 */
+	public String sanitizeJsonString(String input) throws JsonProcessingException {
+		if (input == null) {
+			return null;
+		}
+		Map<String, Object> data = jsonMapper.readValue(input, mapTypeReference);
+		return jsonMapper.writeValueAsString(sanitizeMap(data));
+	}
+
+	/**
+	 * Will replace the sensitive string fields with '*****'
+	 *
+	 * @param input to be sanitized
+	 * @return The sanitized YAML string
+	 * @throws JsonProcessingException
+	 */
+	public String sanitizeYamlString(String input) throws JsonProcessingException {
+		if (input == null) {
+			return null;
+		}
+		Map<String, Object> data = yamlMapper.readValue(input, mapTypeReference);
+		return yamlMapper.writeValueAsString(sanitizeMap(data));
+	}
+
+	/**
+	 * Will determine the type of data and treat as JSON or YAML to sanitize sensitive values.
+	 *
+	 * @param input to be sanitized
+	 * @return the sanitized string
+	 * @throws JsonProcessingException
+	 */
+	public String sanitizeJsonOrYamlString(String input) {
+		if (input == null) {
+			return null;
+		}
+		try { // Try parsing as JSON
+			return sanitizeJsonString(input);
+		} catch (Throwable x) {
+			logger.trace("Cannot parse as JSON:" + x);
+		}
+		try {
+			return sanitizeYamlString(input);
+		} catch (Throwable x) {
+			logger.trace("Cannot parse as YAML:" + x);
+		}
+		if (input.contains("\n")) {
+			return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "\n"))), "\n");
+		}
+		if (input.contains("--")) {
+			return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "--"))), "--");
+		}
+		return sanitize(input);
+	}
 }

spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java

@@ -19,6 +19,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,6 +30,7 @@
 import org.springframework.cloud.dataflow.rest.UpdateStreamRequest;
 import org.springframework.cloud.dataflow.rest.resource.DeploymentStateResource;
 import org.springframework.cloud.dataflow.rest.resource.StreamDeploymentResource;
+import org.springframework.cloud.dataflow.rest.util.ArgumentSanitizer;
 import org.springframework.cloud.dataflow.server.controller.support.ControllerUtils;
 import org.springframework.cloud.dataflow.server.repository.NoSuchStreamDefinitionException;
 import org.springframework.cloud.dataflow.server.repository.StreamDefinitionRepository;
@@ -80,6 +82,8 @@ public class StreamDeploymentController {
 	 */
 	private final StreamDefinitionRepository repository;
 
+	private final ArgumentSanitizer sanitizer = new ArgumentSanitizer();
+
 	/**
 	 * Construct a new UpdatableStreamDeploymentController, given a
 	 * {@link StreamDeploymentController} and {@link StreamService}
@@ -142,7 +146,17 @@ public ResponseEntity<String> manifest(@PathVariable("name") String name,
 	@RequestMapping(path = "/history/{name}", method = RequestMethod.GET)
 	@ResponseStatus(HttpStatus.OK)
 	public Collection<Release> history(@PathVariable("name") String releaseName) {
-		return this.streamService.history(releaseName);
+		return this.streamService.history(releaseName)
+			.stream()
+			.map(this::sanitizeRelease)
+			.collect(Collectors.toList());
+	}
+
+	private Release sanitizeRelease(Release release) {
+		if (release.getConfigValues() != null && StringUtils.hasText(release.getConfigValues().getRaw())) {
+			release.getConfigValues().setRaw(sanitizer.sanitizeJsonOrYamlString(release.getConfigValues().getRaw()));
+		}
+		return release;
 	}
 
 	@RequestMapping(path = "/platform/list", method = RequestMethod.GET)