|
35 | 35 | import org.elasticsearch.client.ml.job.results.OverallBucket;
|
36 | 36 | import org.elasticsearch.core.TimeValue;
|
37 | 37 | import org.elasticsearch.common.xcontent.XContentType;
|
| 38 | +import org.elasticsearch.test.XContentTestUtils; |
38 | 39 | import org.junit.After;
|
39 | 40 | import org.junit.Before;
|
40 | 41 |
|
41 | 42 | import java.io.IOException;
|
42 | 43 | import java.util.Arrays;
|
43 | 44 | import java.util.Date;
|
| 45 | +import java.util.HashMap; |
44 | 46 | import java.util.List;
|
| 47 | +import java.util.Map; |
45 | 48 |
|
46 | 49 | import static org.hamcrest.Matchers.closeTo;
|
47 | 50 | import static org.hamcrest.Matchers.equalTo;
|
@@ -138,55 +141,133 @@ private void addCategoriesIndexRequests(BulkRequest bulkRequest) {
|
138 | 141 | }
|
139 | 142 | }
|
140 | 143 |
|
141 |
| - private void addModelSnapshotIndexRequests(BulkRequest bulkRequest) { |
| 144 | + private void addModelSnapshotIndexRequests(BulkRequest bulkRequest) throws IOException { |
142 | 145 | // Index a number of model snapshots, one of which contains the new model_size_stats fields
|
143 | 146 | // 'model_bytes_exceeded' and 'model_bytes_memory_limit' that were introduced in 7.2.0.
|
144 | 147 | // We want to verify that we can parse the snapshots whether or not these fields are present.
|
145 | 148 | {
|
146 | 149 | IndexRequest indexRequest = new IndexRequest(RESULTS_INDEX);
|
147 |
| - indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541587919000, " + |
148 |
| - "\"description\":\"State persisted due to job close at 2018-11-07T10:51:59+0000\", \"snapshot_id\":\"1541587919\"," + |
149 |
| - "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + |
150 |
| - "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"model_bytes_exceeded\":10762, \"model_bytes_memory_limit\":40960," + |
151 |
| - "\"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," + |
152 |
| - "\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541587919000," + |
153 |
| - " \"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000," + |
154 |
| - " \"retain\":false }", XContentType.JSON); |
| 150 | + Map<String, Object> modelSizeStats = new HashMap<>(); |
| 151 | + modelSizeStats.put("job_id", JOB_ID); |
| 152 | + modelSizeStats.put("result_type", "model_size_stats"); |
| 153 | + modelSizeStats.put("model_bytes", 51722); |
| 154 | + modelSizeStats.put("peak_model_bytes", 61322); |
| 155 | + modelSizeStats.put("model_bytes_exceeded", 10762); |
| 156 | + modelSizeStats.put("model_bytes_memory_limit", 40960); |
| 157 | + modelSizeStats.put("total_by_field_count", 3); |
| 158 | + modelSizeStats.put("total_over_field_count", 0); |
| 159 | + modelSizeStats.put("total_partition_field_count", 2); |
| 160 | + modelSizeStats.put("bucket_allocation_failures_count", 0); |
| 161 | + modelSizeStats.put("memory_status", "ok"); |
| 162 | + modelSizeStats.put("log_time", 1541587919000L); |
| 163 | + modelSizeStats.put("timestamp", 1519930800000L); |
| 164 | + |
| 165 | + Map<String, Object> source = new HashMap<>(); |
| 166 | + source.put("job_id", JOB_ID); |
| 167 | + source.put("timestamp", 1541587919000L); |
| 168 | + source.put("description", "State persisted due to job close at 2018-11-07T10:51:59+0000"); |
| 169 | + source.put("snapshot_id", "1541587919"); |
| 170 | + source.put("snapshot_doc_count", 1); |
| 171 | + source.put("latest_record_time_stamp", 1519931700000L); |
| 172 | + source.put("latest_result_time_stamp", 1519930800000L); |
| 173 | + source.put("retain", false); |
| 174 | + source.put("model_size_stats", modelSizeStats); |
| 175 | + |
| 176 | + indexRequest.source(XContentTestUtils.convertToXContent(source, XContentType.JSON), XContentType.JSON); |
155 | 177 | bulkRequest.add(indexRequest);
|
156 | 178 | }
|
157 | 179 | // Also index one that contains 'memory_assignment_basis', which was added in 7.11
|
158 | 180 | {
|
159 | 181 | IndexRequest indexRequest = new IndexRequest(RESULTS_INDEX);
|
160 |
| - indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541587929000, " + |
161 |
| - "\"description\":\"State persisted due to job close at 2018-11-07T10:52:09+0000\", \"snapshot_id\":\"1541587929\"," + |
162 |
| - "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + |
163 |
| - "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"model_bytes_exceeded\":10762, \"model_bytes_memory_limit\":40960," + |
164 |
| - "\"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," + |
165 |
| - "\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"assignment_memory_basis\":\"model_memory_limit\"," + |
166 |
| - " \"log_time\":1541587929000, \"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," + |
167 |
| - "\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON); |
| 182 | + |
| 183 | + Map<String, Object> modelSizeStats = new HashMap<>(); |
| 184 | + modelSizeStats.put("job_id", JOB_ID); |
| 185 | + modelSizeStats.put("result_type", "model_size_stats"); |
| 186 | + modelSizeStats.put("model_bytes", 51722); |
| 187 | + modelSizeStats.put("peak_model_bytes", 61322); |
| 188 | + modelSizeStats.put("model_bytes_exceeded", 10762); |
| 189 | + modelSizeStats.put("model_bytes_memory_limit", 40960); |
| 190 | + modelSizeStats.put("total_by_field_count", 3); |
| 191 | + modelSizeStats.put("total_over_field_count", 0); |
| 192 | + modelSizeStats.put("total_partition_field_count", 2); |
| 193 | + modelSizeStats.put("bucket_allocation_failures_count", 0); |
| 194 | + modelSizeStats.put("memory_status", "ok"); |
| 195 | + modelSizeStats.put("assignment_memory_basis", "model_memory_limit"); |
| 196 | + modelSizeStats.put("log_time", 1541587929000L); |
| 197 | + modelSizeStats.put("timestamp", 1519930800000L); |
| 198 | + |
| 199 | + Map<String, Object> source = new HashMap<>(); |
| 200 | + source.put("job_id", JOB_ID); |
| 201 | + source.put("timestamp", 1541587929000L); |
| 202 | + source.put("description", "State persisted due to job close at 2018-11-07T10:52:09+0000"); |
| 203 | + source.put("snapshot_id", "1541587929"); |
| 204 | + source.put("snapshot_doc_count", 1); |
| 205 | + source.put("latest_record_time_stamp", 1519931700000L); |
| 206 | + source.put("latest_result_time_stamp", 1519930800000L); |
| 207 | + source.put("retain", false); |
| 208 | + source.put("model_size_stats", modelSizeStats); |
| 209 | + |
| 210 | + indexRequest.source(XContentTestUtils.convertToXContent(source, XContentType.JSON), XContentType.JSON); |
168 | 211 | bulkRequest.add(indexRequest);
|
169 | 212 | }
|
170 | 213 | {
|
171 | 214 | IndexRequest indexRequest = new IndexRequest(RESULTS_INDEX);
|
172 |
| - indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541588919000, " + |
173 |
| - "\"description\":\"State persisted due to job close at 2018-11-07T11:08:39+0000\", \"snapshot_id\":\"1541588919\"," + |
174 |
| - "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + |
175 |
| - "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"total_by_field_count\":3, \"total_over_field_count\":0," + |
176 |
| - "\"total_partition_field_count\":2,\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\"," + |
177 |
| - "\"log_time\":1541588919000,\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," + |
178 |
| - "\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON); |
| 215 | + |
| 216 | + Map<String, Object> modelSizeStats = new HashMap<>(); |
| 217 | + modelSizeStats.put("job_id", JOB_ID); |
| 218 | + modelSizeStats.put("result_type", "model_size_stats"); |
| 219 | + modelSizeStats.put("model_bytes", 51722); |
| 220 | + modelSizeStats.put("peak_model_bytes", 61322); |
| 221 | + modelSizeStats.put("total_by_field_count", 3); |
| 222 | + modelSizeStats.put("total_over_field_count", 0); |
| 223 | + modelSizeStats.put("total_partition_field_count", 2); |
| 224 | + modelSizeStats.put("bucket_allocation_failures_count", 0); |
| 225 | + modelSizeStats.put("memory_status", "ok"); |
| 226 | + modelSizeStats.put("log_time", 1541588919000L); |
| 227 | + modelSizeStats.put("timestamp", 1519930800000L); |
| 228 | + |
| 229 | + Map<String, Object> source = new HashMap<>(); |
| 230 | + source.put("job_id", JOB_ID); |
| 231 | + source.put("timestamp", 1541588919000L); |
| 232 | + source.put("description", "State persisted due to job close at 2018-11-07T11:08:39+0000"); |
| 233 | + source.put("snapshot_id", "1541588919"); |
| 234 | + source.put("snapshot_doc_count", 1); |
| 235 | + source.put("latest_record_time_stamp", 1519931700000L); |
| 236 | + source.put("latest_result_time_stamp", 1519930800000L); |
| 237 | + source.put("retain", false); |
| 238 | + source.put("model_size_stats", modelSizeStats); |
| 239 | + |
| 240 | + indexRequest.source(XContentTestUtils.convertToXContent(source, XContentType.JSON), XContentType.JSON); |
179 | 241 | bulkRequest.add(indexRequest);
|
180 | 242 | }
|
181 | 243 | {
|
182 | 244 | IndexRequest indexRequest = new IndexRequest(RESULTS_INDEX);
|
183 |
| - indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541589919000, " + |
184 |
| - "\"description\":\"State persisted due to job close at 2018-11-07T11:25:19+0000\", \"snapshot_id\":\"1541589919\"," + |
185 |
| - "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + |
186 |
| - "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"total_by_field_count\":3, \"total_over_field_count\":0," + |
187 |
| - "\"total_partition_field_count\":2,\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\"," + |
188 |
| - "\"log_time\":1541589919000,\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," + |
189 |
| - "\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON); |
| 245 | + |
| 246 | + Map<String, Object> modelSizeStats = new HashMap<>(); |
| 247 | + modelSizeStats.put("job_id", JOB_ID); |
| 248 | + modelSizeStats.put("result_type", "model_size_stats"); |
| 249 | + modelSizeStats.put("model_bytes", 51722); |
| 250 | + modelSizeStats.put("peak_model_bytes", 61322); |
| 251 | + modelSizeStats.put("total_by_field_count", 3); |
| 252 | + modelSizeStats.put("total_over_field_count", 0); |
| 253 | + modelSizeStats.put("total_partition_field_count", 2); |
| 254 | + modelSizeStats.put("bucket_allocation_failures_count", 0); |
| 255 | + modelSizeStats.put("memory_status", "ok"); |
| 256 | + modelSizeStats.put("log_time", 1541589919000L); |
| 257 | + modelSizeStats.put("timestamp", 1519930800000L); |
| 258 | + |
| 259 | + Map<String, Object> source = new HashMap<>(); |
| 260 | + source.put("job_id", JOB_ID); |
| 261 | + source.put("timestamp", 1541589919000L); |
| 262 | + source.put("description", "State persisted due to job close at 2018-11-07T11:25:19+0000"); |
| 263 | + source.put("snapshot_id", "1541589919"); |
| 264 | + source.put("snapshot_doc_count", 1); |
| 265 | + source.put("latest_record_time_stamp", 1519931700000L); |
| 266 | + source.put("latest_result_time_stamp", 1519930800000L); |
| 267 | + source.put("retain", false); |
| 268 | + source.put("model_size_stats", modelSizeStats); |
| 269 | + |
| 270 | + indexRequest.source(XContentTestUtils.convertToXContent(source, XContentType.JSON), XContentType.JSON); |
190 | 271 | bulkRequest.add(indexRequest);
|
191 | 272 | }
|
192 | 273 | }
|
|
0 commit comments