Skip to content

Commit 11521bc

Browse files
jakelandisjakelandis
committed
Fix minor watcher bug, unmute test, add additional debug logging
This commit unmutes the org.elasticsearch.smoketest.SmokeTestWatcherWithSecurityIT test suite, fixes a bug [1] that was introduced while the test was muted, and adds some additional debug logging, and enables debug for the ES instance used in this Watcher test. The bug fixed here is minor and unlikely to happen. It requires ES to be started with ILM disabled, Watcher enabled, and Watcher explicitly stopped and restarted. Due to validation Watcher does not fully start and can result in a partially started state. This is an unlikely scenerio outside of the testing framework. Optimistically closing the following Fixes elastic#35361 Fixes elastic#30777 Fixes elastic#35361 Fixes elastic#33291 Fixes elastic#29893 If this does not fully fix the issue, there will now be better debug logging.
1 parent 46d25e6 commit 11521bc

File tree

6 files changed

+191
-7
lines changed

6 files changed

+191
-7
lines changed

‎x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
*/
66
package org.elasticsearch.xpack.watcher;
77

8+
import org.apache.log4j.LogManager;
9+
import org.apache.log4j.Logger;
810
import org.elasticsearch.cluster.ClusterChangedEvent;
911
import org.elasticsearch.cluster.ClusterState;
1012
import org.elasticsearch.cluster.ClusterStateListener;
@@ -39,6 +41,7 @@ public class WatcherLifeCycleService implements ClusterStateListener {
3941
private final AtomicReference<List<ShardRouting>> previousShardRoutings = new AtomicReference<>(Collections.emptyList());
4042
private volatile boolean shutDown = false; // indicates that the node has been shutdown and we should never start watcher after this.
4143
private volatile WatcherService watcherService;
44+
private static final Logger logger = LogManager.getLogger(WatcherLifeCycleService.class);
4245

4346
WatcherLifeCycleService(ClusterService clusterService, WatcherService watcherService) {
4447
this.watcherService = watcherService;
@@ -147,6 +150,7 @@ public void clusterChanged(ClusterChangedEvent event) {
147150
watcherService.start(event.state(), () -> this.state.set(WatcherState.STARTED));
148151
}
149152
} else {
153+
logger.debug("Watcher failed validation, will not start watcher and setting state to stopped");
150154
clearAllocationIds();
151155
this.state.set(WatcherState.STOPPED);
152156
}

‎x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ protected String getOrigin() {
7979
}
8080

8181
public static boolean validate(ClusterState state) {
82-
return state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) &&
82+
return (state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) ||
83+
state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) &&
8384
state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) &&
8485
state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME);
8586
}

‎x-pack/qa/smoke-test-watcher-with-security/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ integTestCluster {
2525
keystoreSetting 'xpack.notification.email.account._email.smtp.secure_password', '_passwd'
2626
setting 'xpack.license.self_generated.type', 'trial'
2727
extraConfigFile 'roles.yml', 'roles.yml'
28+
extraConfigFile 'log4j2.properties', 'custom-log4j2.properties'
2829
setupCommand 'setupTestAdminUser',
2930
'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
3031
setupCommand 'setupXpackUserForTests',

‎x-pack/qa/smoke-test-watcher-with-security/custom-log4j2.properties

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
2+
status = error
3+
4+
# log action execution errors for easier debugging
5+
logger.action.name = org.elasticsearch.action
6+
logger.action.level = debug
7+
8+
9+
# bump up watcher logging
10+
logger.watcher.name = org.elasticsearch.xpack.watcher
11+
logger.watcher.level = debug
12+
13+
logger.core-watcher.name = org.elasticsearch.xpack.core.watcher
14+
logger.core-watcher.level = debug
15+
16+
appender.console.type = Console
17+
appender.console.name = console
18+
appender.console.layout.type = PatternLayout
19+
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
20+
21+
######## Server JSON ############################
22+
appender.rolling.type = RollingFile
23+
appender.rolling.name = rolling
24+
appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json
25+
appender.rolling.layout.type = ESJsonLayout
26+
appender.rolling.layout.type_name = server
27+
28+
appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz
29+
appender.rolling.policies.type = Policies
30+
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
31+
appender.rolling.policies.time.interval = 1
32+
appender.rolling.policies.time.modulate = true
33+
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
34+
appender.rolling.policies.size.size = 128MB
35+
appender.rolling.strategy.type = DefaultRolloverStrategy
36+
appender.rolling.strategy.fileIndex = nomax
37+
appender.rolling.strategy.action.type = Delete
38+
appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
39+
appender.rolling.strategy.action.condition.type = IfFileName
40+
appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
41+
appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
42+
appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB
43+
################################################
44+
######## Server - old style pattern ###########
45+
appender.rolling_old.type = RollingFile
46+
appender.rolling_old.name = rolling_old
47+
appender.rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
48+
appender.rolling_old.layout.type = PatternLayout
49+
appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
50+
51+
appender.rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
52+
appender.rolling_old.policies.type = Policies
53+
appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy
54+
appender.rolling_old.policies.time.interval = 1
55+
appender.rolling_old.policies.time.modulate = true
56+
appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy
57+
appender.rolling_old.policies.size.size = 128MB
58+
appender.rolling_old.strategy.type = DefaultRolloverStrategy
59+
appender.rolling_old.strategy.fileIndex = nomax
60+
appender.rolling_old.strategy.action.type = Delete
61+
appender.rolling_old.strategy.action.basepath = ${sys:es.logs.base_path}
62+
appender.rolling_old.strategy.action.condition.type = IfFileName
63+
appender.rolling_old.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
64+
appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
65+
appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB
66+
################################################
67+
68+
rootLogger.level = info
69+
rootLogger.appenderRef.console.ref = console
70+
rootLogger.appenderRef.rolling.ref = rolling
71+
rootLogger.appenderRef.rolling_old.ref = rolling_old
72+
73+
######## Deprecation JSON #######################
74+
appender.deprecation_rolling.type = RollingFile
75+
appender.deprecation_rolling.name = deprecation_rolling
76+
appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json
77+
appender.deprecation_rolling.layout.type = ESJsonLayout
78+
appender.deprecation_rolling.layout.type_name = deprecation
79+
80+
appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz
81+
appender.deprecation_rolling.policies.type = Policies
82+
appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
83+
appender.deprecation_rolling.policies.size.size = 1GB
84+
appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
85+
appender.deprecation_rolling.strategy.max = 4
86+
#################################################
87+
######## Deprecation - old style pattern #######
88+
appender.deprecation_rolling_old.type = RollingFile
89+
appender.deprecation_rolling_old.name = deprecation_rolling_old
90+
appender.deprecation_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
91+
appender.deprecation_rolling_old.layout.type = PatternLayout
92+
appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
93+
94+
appender.deprecation_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
95+
_deprecation-%i.log.gz
96+
appender.deprecation_rolling_old.policies.type = Policies
97+
appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
98+
appender.deprecation_rolling_old.policies.size.size = 1GB
99+
appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy
100+
appender.deprecation_rolling_old.strategy.max = 4
101+
#################################################
102+
logger.deprecation.name = org.elasticsearch.deprecation
103+
logger.deprecation.level = warn
104+
logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
105+
logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old
106+
logger.deprecation.additivity = false
107+
108+
######## Search slowlog JSON ####################
109+
appender.index_search_slowlog_rolling.type = RollingFile
110+
appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
111+
appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
112+
.cluster_name}_index_search_slowlog.json
113+
appender.index_search_slowlog_rolling.layout.type = ESJsonLayout
114+
appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog
115+
116+
appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
117+
.cluster_name}_index_search_slowlog-%i.json.gz
118+
appender.index_search_slowlog_rolling.policies.type = Policies
119+
appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
120+
appender.index_search_slowlog_rolling.policies.size.size = 1GB
121+
appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy
122+
appender.index_search_slowlog_rolling.strategy.max = 4
123+
#################################################
124+
######## Search slowlog - old style pattern ####
125+
appender.index_search_slowlog_rolling_old.type = RollingFile
126+
appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old
127+
appender.index_search_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
128+
_index_search_slowlog.log
129+
appender.index_search_slowlog_rolling_old.layout.type = PatternLayout
130+
appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
131+
132+
appender.index_search_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
133+
_index_search_slowlog-%i.log.gz
134+
appender.index_search_slowlog_rolling_old.policies.type = Policies
135+
appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
136+
appender.index_search_slowlog_rolling_old.policies.size.size = 1GB
137+
appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy
138+
appender.index_search_slowlog_rolling_old.strategy.max = 4
139+
#################################################
140+
logger.index_search_slowlog_rolling.name = index.search.slowlog
141+
logger.index_search_slowlog_rolling.level = trace
142+
logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
143+
logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old
144+
logger.index_search_slowlog_rolling.additivity = false
145+
146+
######## Indexing slowlog JSON ##################
147+
appender.index_indexing_slowlog_rolling.type = RollingFile
148+
appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
149+
appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
150+
_index_indexing_slowlog.json
151+
appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout
152+
appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog
153+
154+
appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
155+
_index_indexing_slowlog-%i.json.gz
156+
appender.index_indexing_slowlog_rolling.policies.type = Policies
157+
appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
158+
appender.index_indexing_slowlog_rolling.policies.size.size = 1GB
159+
appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy
160+
appender.index_indexing_slowlog_rolling.strategy.max = 4
161+
#################################################
162+
######## Indexing slowlog - old style pattern ##
163+
appender.index_indexing_slowlog_rolling_old.type = RollingFile
164+
appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old
165+
appender.index_indexing_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
166+
_index_indexing_slowlog.log
167+
appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout
168+
appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
169+
170+
appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
171+
_index_indexing_slowlog-%i.log.gz
172+
appender.index_indexing_slowlog_rolling_old.policies.type = Policies
173+
appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
174+
appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB
175+
appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy
176+
appender.index_indexing_slowlog_rolling_old.strategy.max = 4
177+
#################################################
178+
179+
logger.index_indexing_slowlog.name = index.indexing.slowlog.index
180+
logger.index_indexing_slowlog.level = trace
181+
logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
182+
logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old
183+
logger.index_indexing_slowlog.additivity = false

‎x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import static org.hamcrest.Matchers.hasEntry;
3333
import static org.hamcrest.Matchers.is;
3434

35-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35361")
3635
public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
3736

3837
private static final String TEST_ADMIN_USERNAME = "test_admin";
@@ -83,7 +82,7 @@ public void startWatcher() throws Exception {
8382
});
8483

8584
assertBusy(() -> {
86-
for (String template : XPackRestTestConstants.TEMPLATE_NAMES) {
85+
for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
8786
assertOK(adminClient().performRequest(new Request("HEAD", "_template/" + template)));
8887
}
8988
});

‎x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,9 @@ public final class XPackRestTestConstants {
1111

1212
// Watcher constants:
1313
public static final String INDEX_TEMPLATE_VERSION = "9";
14-
public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
1514
public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
1615
public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";
1716
public static final String WATCHES_TEMPLATE_NAME = ".watches";
18-
public static final String[] TEMPLATE_NAMES = new String[] {
19-
HISTORY_TEMPLATE_NAME, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME
20-
};
2117
public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] {
2218
HISTORY_TEMPLATE_NAME_NO_ILM, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME
2319
};

0 commit comments

Comments
 (0)