Skip to content

Commit 3aed349

Browse files
danielmitterdorferdanielmitterdorfer
authored
Bootstrap profiling indices at startup (#95666)
With this commit we bootstrap all indices and related structures that are required for Elastic Universal Profiling.
1 parent ebaefc6 commit 3aed349

File tree

35 files changed

+1728
-105
lines changed

35 files changed

+1728
-105
lines changed

‎docs/changelog/95666.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 95666
2+
summary: Bootstrap profiling indices at startup
3+
area: Indices APIs
4+
type: enhancement
5+
issues: []

‎docs/reference/snapshot-restore/restore-snapshot.asciidoc

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ snapshot as well as each feature's indices.
232232

233233
To restore a specific feature state from the snapshot, specify the
234234
`feature_name` from the response in the restore snapshot API's
235-
<<restore-snapshot-api-feature-states,`feature_states`>> parameter.
235+
<<restore-snapshot-api-feature-states,`feature_states`>> parameter.
236236

237237
NOTE: When you restore a feature state, {es} closes and overwrites the feature's existing indices.
238238

@@ -247,7 +247,7 @@ realm>> to ensure you'll still be able to access your cluster.
247247
POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore
248248
{
249249
"feature_states": [ "geoip" ],
250-
"include_global_state": false, <1>
250+
"include_global_state": false, <1>
251251
"indices": "-*" <2>
252252
}
253253
----
@@ -256,7 +256,7 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore
256256
// TEST[s/_restore/_restore?wait_for_completion=true/]
257257
// TEST[s/"feature_states": \[ "geoip" \],//]
258258

259-
<1> Exclude the cluster state from the restore operation.
259+
<1> Exclude the cluster state from the restore operation.
260260
<2> Exclude the other indices and data streams in the snapshot from the restore operation.
261261

262262
[discrete]
@@ -365,6 +365,27 @@ POST _watcher/_start
365365
. {blank}
366366
+
367367
--
368+
* Universal Profiling
369+
+
370+
Check if Universal Profiling index template management is enabled:
371+
+
372+
[source,console]
373+
----
374+
GET /_cluster/settings?filter_path=**.xpack.profiling.templates.enabled&include_defaults=true
375+
----
376+
+
377+
If the value is `true`, disable Universal Profiling index template management:
378+
+
379+
[source,console]
380+
----
381+
PUT _cluster/settings
382+
{
383+
"persistent": {
384+
"xpack.profiling.templates.enabled": false
385+
}
386+
}
387+
----
388+
368389
[[restore-create-file-realm-user]]
369390
If you use {es} security features, log in to a node host, navigate to the {es}
370391
installation directory, and add a user with the `superuser` role to the file
@@ -480,6 +501,21 @@ POST _watcher/_start
480501
----
481502
--
482503

504+
* Universal Profiling
505+
+
506+
If the value was `true` initially, enable Universal Profiling index template management again, otherwise skip this step:
507+
+
508+
[source,console]
509+
----
510+
PUT _cluster/settings
511+
{
512+
"persistent": {
513+
"xpack.profiling.templates.enabled": true
514+
}
515+
}
516+
----
517+
//TEST[s/true/false/]
518+
483519
. If wanted, reset the `action.destructive_requires_name` cluster setting.
484520
+
485521
[source,console]

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -621,6 +621,7 @@ protected Set<String> preserveILMPolicyIds() {
621621
"ml-size-based-ilm-policy",
622622
"logs",
623623
"metrics",
624+
"profiling",
624625
"synthetics",
625626
"7-days-default",
626627
"30-days-default",
@@ -1814,11 +1815,15 @@ protected static boolean isXPackTemplate(String name) {
18141815
if (name.startsWith("behavioral_analytics-")) {
18151816
return true;
18161817
}
1818+
if (name.startsWith("profiling-")) {
1819+
return true;
1820+
}
18171821
switch (name) {
18181822
case ".watches":
18191823
case "security_audit_log":
18201824
case ".slm-history":
18211825
case ".async-search":
1826+
case ".profiling-ilm-lock": // TODO: Remove after switch to K/V indices
18221827
case "saml-service-provider":
18231828
case "logs":
18241829
case "logs-settings":

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion(
186186
public static final String TRANSFORM_ORIGIN = "transform";
187187
public static final String ASYNC_SEARCH_ORIGIN = "async_search";
188188
public static final String IDP_ORIGIN = "idp";
189+
public static final String PROFILING_ORIGIN = "profiling";
189190
public static final String STACK_ORIGIN = "stack";
190191
public static final String SEARCHABLE_SNAPSHOTS_ORIGIN = "searchable_snapshots";
191192
public static final String LOGSTASH_MANAGEMENT_ORIGIN = "logstash_management";

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,13 @@ public Iterator<Setting<?>> settings() {
9393
Setting.Property.NodeScope
9494
);
9595

96+
/** Setting for enabling or disabling universal profiling. Defaults to true. */
97+
public static final Setting<Boolean> PROFILING_ENABLED = Setting.boolSetting(
98+
"xpack.profiling.enabled",
99+
true,
100+
Setting.Property.NodeScope
101+
);
102+
96103
/** Setting for enabling or disabling enterprise search. Defaults to true. */
97104
public static final Setting<Boolean> ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting(
98105
"xpack.ent_search.enabled",
@@ -318,6 +325,7 @@ public static List<Setting<?>> getAllSettings() {
318325
settings.add(SECURITY_ENABLED);
319326
settings.add(GRAPH_ENABLED);
320327
settings.add(MACHINE_LEARNING_ENABLED);
328+
settings.add(PROFILING_ENABLED);
321329
settings.add(ENTERPRISE_SEARCH_ENABLED);
322330
settings.add(AUDIT_ENABLED);
323331
settings.add(WATCHER_ENABLED);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils;
1717
import org.elasticsearch.xpack.core.ilm.LifecycleType;
1818
import org.elasticsearch.xpack.core.ilm.RolloverAction;
19+
import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
1920
import org.elasticsearch.xpack.core.ilm.ShrinkAction;
2021
import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
2122

@@ -36,6 +37,7 @@ public class LifecyclePolicyConfig {
3637
(p) -> TimeseriesLifecycleType.INSTANCE
3738
),
3839
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
40+
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SetPriorityAction.NAME), SetPriorityAction::parse),
3941
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
4042
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse),
4143
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse)

‎x-pack/plugin/profiler/src/internalClusterTest/resources/events.json renamed to ‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
11
{
2-
"settings": {
3-
"index": {
4-
"number_of_shards": "4",
5-
"max_result_window": 150000,
6-
"refresh_interval": "10s",
7-
"sort": {
8-
"field": [
9-
"service.name",
10-
"@timestamp",
11-
"orchestrator.resource.name",
12-
"container.name",
13-
"process.thread.name",
14-
"host.id"
15-
]
16-
}
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"number_of_shards": 4,
6+
"number_of_replicas": 0,
7+
"auto_expand_replicas": "0-1",
8+
"max_result_window": 150000,
9+
"refresh_interval": "10s",
10+
"sort": {
11+
"field": [
12+
"service.name",
13+
"@timestamp",
14+
"orchestrator.resource.name",
15+
"container.name",
16+
"process.thread.name",
17+
"host.id"
18+
]
19+
}
20+
},
21+
"codec": "best_compression"
1722
},
18-
"codec": "best_compression"
19-
},
20-
"mappings": {
21-
"_doc": {
23+
"mappings": {
2224
"_source": {
2325
"enabled": false
2426
},
@@ -60,9 +62,6 @@
6062
"host.ip": {
6163
"type": "ip"
6264
},
63-
"host.ipstring": {
64-
"type": "keyword"
65-
},
6665
"host.name": {
6766
"type": "keyword"
6867
},
@@ -74,5 +73,6 @@
7473
}
7574
}
7675
}
77-
}
76+
},
77+
"version": ${xpack.profiling.template.version}
7878
}

‎x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json renamed to ‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
{
2-
"settings": {
3-
"index": {
4-
"refresh_interval": "10s"
5-
}
6-
},
7-
"mappings": {
8-
"_doc": {
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"number_of_replicas": 0,
6+
"auto_expand_replicas": "0-1",
7+
"refresh_interval": "10s"
8+
}
9+
},
10+
"mappings": {
911
"_source": {
1012
"mode": "synthetic"
1113
},
@@ -25,8 +27,14 @@
2527
"@timestamp": {
2628
"type": "date",
2729
"format": "epoch_second"
30+
},
31+
"Symbolization.lastprocessed": {
32+
"type": "date",
33+
"format": "epoch_second",
34+
"index": false
2835
}
2936
}
3037
}
31-
}
38+
},
39+
"version": ${xpack.profiling.template.version}
3240
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"lifecycle": {
6+
"name": "profiling"
7+
}
8+
}
9+
}
10+
},
11+
"version": ${xpack.profiling.template.version}
12+
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"number_of_replicas": 0,
6+
"auto_expand_replicas": "0-1",
7+
"refresh_interval": "10s",
8+
"sort": {
9+
"field": [
10+
"project.id",
11+
"@timestamp",
12+
"host.id"
13+
]
14+
}
15+
},
16+
"codec": "best_compression"
17+
},
18+
"mappings": {
19+
"_source": {
20+
"enabled": false
21+
},
22+
"properties": {
23+
"ecs.version": {
24+
"type": "keyword",
25+
"index": true
26+
},
27+
"project.id": {
28+
"type": "keyword"
29+
},
30+
"host.id": {
31+
"type": "keyword"
32+
},
33+
"@timestamp": {
34+
"type": "date",
35+
"format": "epoch_second"
36+
},
37+
"Elasticsearch.cluster.id": {
38+
"type": "keyword"
39+
}
40+
}
41+
}
42+
},
43+
"version": ${xpack.profiling.template.version}
44+
}

‎x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json renamed to ‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
{
2-
"settings": {
3-
"index": {
4-
"number_of_shards": "16",
5-
"refresh_interval": "10s"
6-
}
7-
},
8-
"mappings": {
9-
"_doc": {
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"number_of_shards": 16,
6+
"number_of_replicas": 0,
7+
"auto_expand_replicas": "0-1",
8+
"refresh_interval": "10s"
9+
}
10+
},
11+
"mappings": {
1012
"_source": {
1113
"enabled": true
1214
},
@@ -34,8 +36,15 @@
3436
"index": false,
3537
"doc_values": false,
3638
"store": false
39+
},
40+
"Stackframe.function.offset": {
41+
"type": "integer",
42+
"index": false,
43+
"doc_values": false,
44+
"store": false
3745
}
3846
}
3947
}
40-
}
48+
},
49+
"version": ${xpack.profiling.template.version}
4150
}

‎x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json renamed to ‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
11
{
2-
"settings": {
3-
"index": {
4-
"number_of_shards": "16",
5-
"refresh_interval": "10s",
6-
"sort": {
7-
"field": [
8-
"Stacktrace.frame.ids"
9-
]
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"number_of_shards": 16,
6+
"number_of_replicas": 0,
7+
"auto_expand_replicas": "0-1",
8+
"refresh_interval": "10s",
9+
"sort": {
10+
"field": [
11+
"Stacktrace.frame.ids"
12+
]
13+
}
1014
}
11-
}
12-
},
13-
"mappings": {
14-
"_doc": {
15+
},
16+
"mappings": {
1517
"_source": {
1618
"mode": "synthetic"
1719
},
@@ -27,12 +29,9 @@
2729
"Stacktrace.frame.types": {
2830
"type": "keyword",
2931
"index": false
30-
},
31-
"@timestamp": {
32-
"type": "date",
33-
"format": "epoch_second"
3432
}
3533
}
3634
}
37-
}
35+
},
36+
"version": ${xpack.profiling.template.version}
3837
}

0 commit comments

Comments
 (0)