Merge pull request #338 from toddferg/master

Adding a watch for large shard monitoring

Author: toddferg

Alerting/Sample Watches/large_shard_watch/README.md

@@ -0,0 +1,12 @@
+# Monitoring for Large Shards
+
+## Description
+
+This is a watch that creates a helper index (large_shards), and it uses it to alert one time (per shard) based off the size of the shards defined in the metadata.
+
+It queries the cat/shards api call to get the information first, and then ingests it into large-shards
+
+
+# Configuration
+
+* Metadata is where the threshold_in_bytes is set.

Alerting/Sample Watches/large_shard_watch/watch.json

@@ -0,0 +1,127 @@
+{
+  "trigger": {
+    "schedule": {
+      "interval": "1h"
+    }
+  },
+  "input": {
+    "chain": {
+      "inputs": [
+        {
+          "first": {
+            "http": {
+              "request": {
+                "scheme": "https",
+                "host": "localhost",
+                "port": 9243,
+                "method": "get",
+                "path": "/_cat/shards",
+                "params": {
+                  "format": "json",
+                  "bytes": "b"
+                },
+                "headers": {},
+                "auth": {
+                  "basic": {
+                    "username": "elastic",
+                    "password": "changeme"
+                  }
+                }
+              }
+            }
+          }
+        },
+        {
+          "second": {
+            "search": {
+              "request": {
+                "indices": [
+                  "large*"
+                ],
+                "types": [
+                  "_doc"
+                ],
+                "body": {
+                  "size": 400,
+                  "query": {
+                    "match_all": {}
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  },
+  "condition": {
+    "always": {}
+  },
+  "transform": {
+    "script": {
+      "source": """
+        HashMap final = new HashMap();
+        def docs = [];
+if (ctx.payload.second.hits.total > 0) {
+  ctx.payload.second.hits.hits.stream()
+  .forEach(e -> {
+    final.put(e._source.indexName+"-"+e._source.shardNumber, e._source.shardSizeInBytes);
+  });
+}
+List result = ctx.payload.first.data.stream()
+  .forEach(s -> {
+    def size = s.store == null ? 0 : Long.parseLong(s.store);
+    if(size > ctx.metadata.threshold_in_bytes && !final.containsKey(s.index+"-"+ s.shard)) {
+      docs.add([
+        '_id' : s.index+ "-"+ s.shard,
+        'shardSizeInBytes': size,
+        'indexName': s.index,
+        'shardNumber': s.shard,
+        'node': s.node
+      ]);
+    }
+    return true;
+  });
+return ['_doc' : docs];
+        """,
+      "lang": "painless"
+    }
+  },
+  "actions": {
+    "index_payload": {
+      "condition": {
+        "script": {
+          "source": "return ctx.payload._doc.size() > 0",
+          "lang": "painless"
+        }
+      },
+      "index": {
+        "index": "large-shards",
+        "doc_type": "_doc"
+      }
+    },
+    "logging": {
+      "condition": {
+        "script": {
+          "source": "return ctx.payload._doc.size() > 0",
+          "lang": "painless"
+        }
+      },
+      "logging": {
+        "level": "info",
+        "text": """
+The following shards are bigger than {{ctx.metadata.threshold_in_bytes}} bytes:
+{{#ctx.payload._doc}}
+Node [{{node}}] Shard #{{shardNumber}} Index [{{indexName}}] Size [{{shardSizeInBytes}}]
+{{/ctx.payload._doc}}
+"""
+      }
+    }
+  },
+  "metadata": {
+    "threshold_in_bytes": 3221225400,
+    "xpack": {
+      "type": "json"
+    }
+  }
+}

README.md

@@ -67,6 +67,7 @@ X-Pack lets you set up watches (or rules) to detect and alert on changes in your
 - [Trending hashtag on twitter](https://github.com/elastic/examples/tree/master/Alerting/Sample%20Watches/twitter_trends)
 - [Unexpected account activity](https://github.com/elastic/examples/tree/master/Alerting/Sample%20Watches/unexpected_account_activity)
 - [Watch history dashboard](https://github.com/elastic/examples/tree/master/Alerting/watcher_dashboard)
+- [Alert on Large Shards](https://github.com/elastic/examples/tree/master/Alerting/Sample%20Watches/large_shard_watch)
 
 #### Machine learning