diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 440ea8461986d..70a4b6382d7e8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -448,6 +448,7 @@ public Set getRoles() { public static final Setting USE_AUTO_MACHINE_MEMORY_PERCENT = Setting.boolSetting( "xpack.ml.use_auto_machine_memory_percent", false, + Property.Dynamic, Property.NodeScope); public static final Setting MAX_LAZY_ML_NODES = Setting.intSetting("xpack.ml.max_lazy_ml_nodes", 0, 0, Property.Dynamic, Property.NodeScope); @@ -497,6 +498,7 @@ public Set getRoles() { public static final Setting MAX_ML_NODE_SIZE = Setting.byteSizeSetting( "xpack.ml.max_ml_node_size", ByteSizeValue.ZERO, + Property.Dynamic, Property.NodeScope); private static final Logger logger = LogManager.getLogger(MachineLearning.class); @@ -579,7 +581,11 @@ public Settings additionalSettings() { // This is not used in v7 and higher, but users are still prevented from setting it directly to avoid confusion disallowMlNodeAttributes(mlEnabledNodeAttrName); } else { - disallowMlNodeAttributes(mlEnabledNodeAttrName, maxOpenJobsPerNodeNodeAttrName, machineMemoryAttrName); + disallowMlNodeAttributes(mlEnabledNodeAttrName, + maxOpenJobsPerNodeNodeAttrName, + machineMemoryAttrName, + jvmSizeAttrName + ); } return additionalSettings.build(); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java index 4f093e976c01e..310b4a1f2cdc5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java @@ -72,12 +72,12 @@ public class MlAutoscalingDeciderService implements AutoscalingDeciderService, private final NodeLoadDetector nodeLoadDetector; private final MlMemoryTracker mlMemoryTracker; private final Supplier timeSupplier; - private final boolean useAuto; private volatile boolean isMaster; private volatile boolean running; private volatile int maxMachineMemoryPercent; private volatile int maxOpenJobs; + private volatile boolean useAuto; private volatile long lastTimeToScale; private volatile long scaleDownDetected; @@ -99,6 +99,7 @@ public MlAutoscalingDeciderService(MlMemoryTracker memoryTracker, Settings setti clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, this::setMaxMachineMemoryPercent); clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs); + clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT, this::setUseAuto); clusterService.addLocalNodeMasterListener(this); clusterService.addLifecycleListener(new LifecycleListener() { @Override @@ -206,6 +207,10 @@ void setMaxOpenJobs(int maxOpenJobs) { this.maxOpenJobs = maxOpenJobs; } + void setUseAuto(boolean useAuto) { + this.useAuto = useAuto; + } + @Override public void onMaster() { isMaster = true; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java index b20f0cd3294b4..d90d466b80e3f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksExecutor; @@ -60,7 +61,6 @@ public static List verifyIndicesPrimaryShardsAreActive(ClusterState clus return unavailableIndices; } - protected final boolean useAutoMemoryPercentage; protected final MlMemoryTracker memoryTracker; protected final IndexNameExpressionResolver expressionResolver; @@ -68,8 +68,9 @@ public static List verifyIndicesPrimaryShardsAreActive(ClusterState clus protected volatile int maxConcurrentJobAllocations; protected volatile int maxMachineMemoryPercent; protected volatile int maxLazyMLNodes; + protected volatile boolean useAutoMemoryPercentage; + protected volatile long maxNodeMemory; protected volatile int maxOpenJobs; - protected final long maxNodeMemory; protected AbstractJobPersistentTasksExecutor(String taskName, String executor, @@ -92,6 +93,8 @@ protected AbstractJobPersistentTasksExecutor(String taskName, .addSettingsUpdateConsumer(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, this::setMaxMachineMemoryPercent); clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs); + clusterService.getClusterSettings().addSettingsUpdateConsumer(USE_AUTO_MACHINE_MEMORY_PERCENT, this::setUseAutoMemoryPercentage); + clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_ML_NODE_SIZE, this::setMaxNodeSize); } protected abstract String[] indicesOfInterest(Params params); @@ -136,6 +139,14 @@ void setMaxOpenJobs(int maxOpenJobs) { this.maxOpenJobs = maxOpenJobs; } + void setUseAutoMemoryPercentage(boolean useAutoMemoryPercentage) { + this.useAutoMemoryPercentage = useAutoMemoryPercentage; + } + + void setMaxNodeSize(ByteSizeValue maxNodeSize) { + this.maxNodeMemory = maxNodeSize.getBytes(); + } + public Optional checkRequiredIndices(String jobId, ClusterState clusterState, String... indicesOfInterest) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java index 7b3ed55e63acd..f49eb74ace686 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java @@ -153,6 +153,7 @@ private static TaskExecutor createTaskExecutor() { MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_ML_NODE_SIZE, MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE)); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java index 38a1b4a1f5a5f..7f15d88458e69 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java @@ -67,7 +67,9 @@ public void setup() { timeSupplier = System::currentTimeMillis; ClusterSettings cSettings = new ClusterSettings( Settings.EMPTY, - Set.of(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearning.MAX_OPEN_JOBS_PER_NODE)); + Set.of(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_OPEN_JOBS_PER_NODE, + MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT)); when(clusterService.getClusterSettings()).thenReturn(cSettings); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java index fe657a72374fb..73f7da9106a4a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java @@ -110,8 +110,13 @@ public void testValidate_givenValidJob() { public void testGetAssignment_GivenJobThatRequiresMigration() { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, - Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, - MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT) + Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, + MachineLearning.MAX_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_LAZY_ML_NODES, + MachineLearning.MAX_ML_NODE_SIZE, + MachineLearning.MAX_OPEN_JOBS_PER_NODE, + MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT + ) ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); @@ -125,8 +130,13 @@ public void testGetAssignment_GivenJobThatRequiresMigration() { public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 1).build(); ClusterSettings clusterSettings = new ClusterSettings(settings, - Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, - MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT) + Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, + MachineLearning.MAX_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_LAZY_ML_NODES, + MachineLearning.MAX_ML_NODE_SIZE, + MachineLearning.MAX_OPEN_JOBS_PER_NODE, + MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT + ) ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); @@ -150,8 +160,13 @@ public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() { Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 0).build(); ClusterSettings clusterSettings = new ClusterSettings(settings, - Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, - MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT) + Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, + MachineLearning.MAX_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_LAZY_ML_NODES, + MachineLearning.MAX_ML_NODE_SIZE, + MachineLearning.MAX_OPEN_JOBS_PER_NODE, + MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT + ) ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings);