Skip to content
This repository was archived by the owner on May 14, 2025. It is now read-only.

Backport sanitizer #4976

Merged
merged 1 commit into from
Jul 1, 2022
Merged

Backport sanitizer #4976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 124 additions & 11 deletions ...esource/src/main/java/org/springframework/cloud/dataflow/rest/util/ArgumentSanitizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,27 @@
package org.springframework.cloud.dataflow.rest.util;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.cloud.dataflow.core.DefinitionUtils;
import org.springframework.cloud.dataflow.core.TaskDefinition;
import org.springframework.cloud.dataflow.core.dsl.TaskParser;
import org.springframework.cloud.dataflow.core.dsl.graph.Graph;
import org.springframework.http.HttpHeaders;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

Expand All @@ -40,13 +49,20 @@
* @author Ilayaperumal Gopinathan
*/
public class ArgumentSanitizer {
private final static Logger logger = LoggerFactory.getLogger(ArgumentSanitizer.class);

private static final String[] REGEX_PARTS = { "*", "$", "^", "+" };
private static final String[] REGEX_PARTS = {"*", "$", "^", "+"};

private static final String REDACTION_STRING = "******";

private static final String[] KEYS_TO_SANITIZE = { "username", "password", "secret", "key", "token", ".*credentials.*",
"vcap_services", "url" };
private static final String[] KEYS_TO_SANITIZE = {"username", "password", "secret", "key", "token", ".*credentials.*",
"vcap_services", "url"};

private final static TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() {};

private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());

private final ObjectMapper jsonMapper = new ObjectMapper();

private Pattern[] keysToSanitize;

Expand Down Expand Up @@ -80,6 +96,10 @@ private boolean isRegex(String value) {
* @return the argument with a potentially sanitized value
*/
public String sanitize(String argument) {
// Oracle handles an empty string as a null.
if (argument == null) {
return "";
}
int indexOfFirstEqual = argument.indexOf("=");
if (indexOfFirstEqual == -1) {
return argument;
Expand All @@ -95,7 +115,7 @@ public String sanitize(String argument) {
/**
* Replaces a potential secure value with "******".
*
* @param key to check for sensitive words.
* @param key to check for sensitive words.
* @param value the argument to cleanse.
* @return the argument with a potentially sanitized value
*/
Expand All @@ -118,13 +138,12 @@ public String sanitize(String key, String value) {
* @return the sanitized job parameters
*/
public JobParameters sanitizeJobParameters(JobParameters jobParameters) {
Map<String,JobParameter> newJobParameters = new HashMap<>();
jobParameters.getParameters().forEach( (key, jobParameter) -> {
Map<String, JobParameter> newJobParameters = new HashMap<>();
jobParameters.getParameters().forEach((key, jobParameter) -> {
String updatedKey = !jobParameter.isIdentifying() ? "-" + key : key;
if (jobParameter.getType().equals(JobParameter.ParameterType.STRING)) {
newJobParameters.put(updatedKey, new JobParameter(this.sanitize(key, jobParameter.toString())));
}
else {
} else {
newJobParameters.put(updatedKey, jobParameter);
}
});
Expand All @@ -138,7 +157,7 @@ public JobParameters sanitizeJobParameters(JobParameters jobParameters) {
* @return Task definition text that has sensitive data redacted.
*/
public String sanitizeTaskDsl(TaskDefinition taskDefinition) {
if(StringUtils.isEmpty(taskDefinition.getDslText())) {
if (StringUtils.isEmpty(taskDefinition.getDslText())) {
return taskDefinition.getDslText();
}
TaskParser taskParser = new TaskParser(taskDefinition.getTaskName(), taskDefinition.getDslText(), true, true);
Expand All @@ -147,7 +166,7 @@ public String sanitizeTaskDsl(TaskDefinition taskDefinition) {
if (node.properties != null) {
node.properties.keySet().stream().forEach(key -> {
node.properties.put(key,
DefinitionUtils.autoQuotes(sanitize(key, node.properties.get(key))));
DefinitionUtils.autoQuotes(sanitize(key, node.properties.get(key))));
});
}
});
Expand All @@ -157,13 +176,14 @@ public String sanitizeTaskDsl(TaskDefinition taskDefinition) {
/**
* For all sensitive properties (e.g. key names containing words like password, secret,
* key, token) replace the value with '*****' string
*
* @param properties to be sanitized
* @return sanitized properties
*/
public Map<String, String> sanitizeProperties(Map<String, String> properties) {
if (!CollectionUtils.isEmpty(properties)) {
final Map<String, String> sanitizedProperties = new LinkedHashMap<>(properties.size());
for (Map.Entry<String, String > property : properties.entrySet()) {
for (Map.Entry<String, String> property : properties.entrySet()) {
sanitizedProperties.put(property.getKey(), this.sanitize(property.getKey(), property.getValue()));
}
return sanitizedProperties;
Expand All @@ -174,6 +194,7 @@ public Map<String, String> sanitizeProperties(Map<String, String> properties) {
/**
* For all sensitive arguments (e.g. key names containing words like password, secret,
* key, token) replace the value with '*****' string
*
* @param arguments to be sanitized
* @return sanitized arguments
*/
Expand All @@ -187,4 +208,96 @@ public List<String> sanitizeArguments(List<String> arguments) {
}
return arguments;
}

public HttpHeaders sanitizeHeaders(HttpHeaders headers) {
HttpHeaders result = new HttpHeaders();
for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
List<String> values = entry.getValue();
for (String value : values) {
result.add(entry.getKey(), sanitize(entry.getKey(), value));
}
}
return result;
}

/**
* Will replace sensitive string value in the Map with '*****'
*
* @param input to be sanitized
* @return the sanitized map.
*/
public Map<String, Object> sanitizeMap(Map<String, Object> input) {
Map<String, Object> result = new HashMap<>();
for (Map.Entry<String, Object> entry : input.entrySet()) {
if (entry.getValue() instanceof String) {
result.put(entry.getKey(), sanitize(entry.getKey(), (String) entry.getValue()));
} else if (entry.getValue() instanceof Map) {
Map<String, Object> map = (Map<String, Object>) entry.getValue();
result.put(entry.getKey(), sanitizeMap(map));
} else {
result.put(entry.getKey(), entry.getValue());
}
}
return result;
}

/**
* Will replace the sensitive string fields with '*****'
*
* @param input to be sanitized
* @return The sanitized JSON string
* @throws JsonProcessingException
*/
public String sanitizeJsonString(String input) throws JsonProcessingException {
if (input == null) {
return null;
}
Map<String, Object> data = jsonMapper.readValue(input, mapTypeReference);
return jsonMapper.writeValueAsString(sanitizeMap(data));
}

/**
* Will replace the sensitive string fields with '*****'
*
* @param input to be sanitized
* @return The sanitized YAML string
* @throws JsonProcessingException
*/
public String sanitizeYamlString(String input) throws JsonProcessingException {
if (input == null) {
return null;
}
Map<String, Object> data = yamlMapper.readValue(input, mapTypeReference);
return yamlMapper.writeValueAsString(sanitizeMap(data));
}

/**
* Will determine the type of data and treat as JSON or YAML to sanitize sensitive values.
*
* @param input to be sanitized
* @return the sanitized string
* @throws JsonProcessingException
*/
public String sanitizeJsonOrYamlString(String input) {
if (input == null) {
return null;
}
try { // Try parsing as JSON
return sanitizeJsonString(input);
} catch (Throwable x) {
logger.trace("Cannot parse as JSON:" + x);
}
try {
return sanitizeYamlString(input);
} catch (Throwable x) {
logger.trace("Cannot parse as YAML:" + x);
}
if (input.contains("\n")) {
return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "\n"))), "\n");
}
if (input.contains("--")) {
return StringUtils.collectionToDelimitedString(sanitizeArguments(Arrays.asList(StringUtils.split(input, "--"))), "--");
}
return sanitize(input);
}
}
16 changes: 15 additions & 1 deletion ...java/org/springframework/cloud/dataflow/server/controller/StreamDeploymentController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -29,6 +30,7 @@
import org.springframework.cloud.dataflow.rest.UpdateStreamRequest;
import org.springframework.cloud.dataflow.rest.resource.DeploymentStateResource;
import org.springframework.cloud.dataflow.rest.resource.StreamDeploymentResource;
import org.springframework.cloud.dataflow.rest.util.ArgumentSanitizer;
import org.springframework.cloud.dataflow.server.controller.support.ControllerUtils;
import org.springframework.cloud.dataflow.server.repository.NoSuchStreamDefinitionException;
import org.springframework.cloud.dataflow.server.repository.StreamDefinitionRepository;
Expand Down Expand Up @@ -80,6 +82,8 @@ public class StreamDeploymentController {
*/
private final StreamDefinitionRepository repository;

private final ArgumentSanitizer sanitizer = new ArgumentSanitizer();

/**
* Construct a new UpdatableStreamDeploymentController, given a
* {@link StreamDeploymentController} and {@link StreamService}
Expand Down Expand Up @@ -142,7 +146,17 @@ public ResponseEntity<String> manifest(@PathVariable("name") String name,
@RequestMapping(path = "/history/{name}", method = RequestMethod.GET)
@ResponseStatus(HttpStatus.OK)
public Collection<Release> history(@PathVariable("name") String releaseName) {
return this.streamService.history(releaseName);
return this.streamService.history(releaseName)
.stream()
.map(this::sanitizeRelease)
.collect(Collectors.toList());
}

private Release sanitizeRelease(Release release) {
if (release.getConfigValues() != null && StringUtils.hasText(release.getConfigValues().getRaw())) {
release.getConfigValues().setRaw(sanitizer.sanitizeJsonOrYamlString(release.getConfigValues().getRaw()));
}
return release;
}

@RequestMapping(path = "/platform/list", method = RequestMethod.GET)
Expand Down