Skip to content

[ML] Use unique named pipes for data frame analytics #70918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 26, 2021
Merged

[ML] Use unique named pipes for data frame analytics #70918

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ...main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;

public class NativeAnalyticsProcessFactory implements AnalyticsProcessFactory<AnalyticsResult> {
Expand All @@ -45,6 +46,7 @@ public class NativeAnalyticsProcessFactory implements AnalyticsProcessFactory<An
private final NamedXContentRegistry namedXContentRegistry;
private final ResultsPersisterService resultsPersisterService;
private final DataFrameAnalyticsAuditor auditor;
private final AtomicLong counter;
private volatile Duration processConnectTimeout;

public NativeAnalyticsProcessFactory(Environment env,
Expand All @@ -58,6 +60,7 @@ public NativeAnalyticsProcessFactory(Environment env,
this.namedXContentRegistry = Objects.requireNonNull(namedXContentRegistry);
this.auditor = auditor;
this.resultsPersisterService = resultsPersisterService;
this.counter = new AtomicLong(0);
setProcessConnectTimeout(MachineLearning.PROCESS_CONNECT_TIMEOUT.get(env.settings()));
clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.PROCESS_CONNECT_TIMEOUT,
this::setProcessConnectTimeout);
Expand All @@ -73,8 +76,11 @@ public NativeAnalyticsProcess createAnalyticsProcess(DataFrameAnalyticsConfig co
Consumer<String> onProcessCrash) {
String jobId = config.getId();
List<Path> filesToDelete = new ArrayList<>();
// When the stop API is called the process is killed. As it may take some time for the OS (especially Windows)
// to delete the named pipes, we use a unique identifier to avoid reusing an older named pipe if the task
// gets restarted immediately after stopping.
ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, processConnectTimeout, AnalyticsBuilder.ANALYTICS, jobId,
null, false, true, true, hasState, config.getAnalysis().persistsState());
counter.incrementAndGet(), false, true, true, hasState, config.getAnalysis().persistsState());

// The extra 2 are for the checksum and the control field
int numberOfFields = analyticsProcessConfig.cols() + 2;
Expand Down