From 55471abc1223c40298c0d0fe3883a6b64ec3ead7 Mon Sep 17 00:00:00 2001 From: lcawl Date: Wed, 20 Jun 2018 11:03:02 -0700 Subject: [PATCH 1/2] [DOCS] Move monitoring to docs --- docs/reference/index.asciidoc | 2 +- x-pack/docs/en/monitoring/collectors.asciidoc | 149 --------------- .../configuring-monitoring.asciidoc | 1 + x-pack/docs/en/monitoring/exporters.asciidoc | 171 ------------------ .../docs/en/monitoring/http-export.asciidoc | 116 ------------ x-pack/docs/en/monitoring/index.asciidoc | 45 ----- x-pack/docs/en/monitoring/indices.asciidoc | 1 + .../docs/en/monitoring/local-export.asciidoc | 82 --------- .../docs/en/monitoring/pause-export.asciidoc | 35 ---- 9 files changed, 3 insertions(+), 599 deletions(-) delete mode 100644 x-pack/docs/en/monitoring/collectors.asciidoc delete mode 100644 x-pack/docs/en/monitoring/exporters.asciidoc delete mode 100644 x-pack/docs/en/monitoring/http-export.asciidoc delete mode 100644 x-pack/docs/en/monitoring/index.asciidoc delete mode 100644 x-pack/docs/en/monitoring/local-export.asciidoc delete mode 100644 x-pack/docs/en/monitoring/pause-export.asciidoc diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index adbdc01db1e72..ef8e345303137 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -59,7 +59,7 @@ include::ingest.asciidoc[] include::{xes-repo-dir}/sql/index.asciidoc[] -include::{xes-repo-dir}/monitoring/index.asciidoc[] +include::monitoring/index.asciidoc[] include::{xes-repo-dir}/rollup/index.asciidoc[] diff --git a/x-pack/docs/en/monitoring/collectors.asciidoc b/x-pack/docs/en/monitoring/collectors.asciidoc deleted file mode 100644 index 336f204b5eefb..0000000000000 --- a/x-pack/docs/en/monitoring/collectors.asciidoc +++ /dev/null @@ -1,149 +0,0 @@ -[role="xpack"] -[[es-monitoring-collectors]] -== Collectors - -Collectors, as their name implies, collect things. Each collector runs once for -each collection interval to obtain data from the public APIs in {es} and {xpack} -that it chooses to monitor. When the data collection is finished, the data is -handed in bulk to the <> to be sent to the -monitoring clusters. Regardless of the number of exporters, each collector only -runs once per collection interval. - -There is only one collector per data type gathered. In other words, for any -monitoring document that is created, it comes from a single collector rather -than being merged from multiple collectors. {monitoring} for {es} currently has -a few collectors because the goal is to minimize overlap between them for -optimal performance. - -Each collector can create zero or more monitoring documents. For example, -the `index_stats` collector collects all index statistics at the same time to -avoid many unnecessary calls. - -[options="header"] -|======================= -| Collector | Data Types | Description -| Cluster Stats | `cluster_stats` -| Gathers details about the cluster state, including parts of -the actual cluster state (for example `GET /_cluster/state`) and statistics -about it (for example, `GET /_cluster/stats`). This produces a single document -type. In versions prior to X-Pack 5.5, this was actually three separate collectors -that resulted in three separate types: `cluster_stats`, `cluster_state`, and -`cluster_info`. In 5.5 and later, all three are combined into `cluster_stats`. -+ -This only runs on the _elected_ master node and the data collected -(`cluster_stats`) largely controls the UI. When this data is not present, it -indicates either a misconfiguration on the elected master node, timeouts related -to the collection of the data, or issues with storing the data. Only a single -document is produced per collection. -| Index Stats | `indices_stats`, `index_stats` -| Gathers details about the indices in the cluster, both in summary and -individually. This creates many documents that represent parts of the index -statistics output (for example, `GET /_stats`). -+ -This information only needs to be collected once, so it is collected on the -_elected_ master node. The most common failure for this collector relates to an -extreme number of indices -- and therefore time to gather them -- resulting in -timeouts. One summary `indices_stats` document is produced per collection and one -`index_stats` document is produced per index, per collection. -| Index Recovery | `index_recovery` -| Gathers details about index recovery in the cluster. Index recovery represents -the assignment of _shards_ at the cluster level. If an index is not recovered, -it is not usable. This also corresponds to shard restoration via snapshots. -+ -This information only needs to be collected once, so it is collected on the -_elected_ master node. The most common failure for this collector relates to an -extreme number of shards -- and therefore time to gather them -- resulting in -timeouts. This creates a single document that contains all recoveries by default, -which can be quite large, but it gives the most accurate picture of recovery in -the production cluster. -| Shards | `shards` -| Gathers details about all _allocated_ shards for all indices, particularly -including what node the shard is allocated to. -+ -This information only needs to be collected once, so it is collected on the -_elected_ master node. The collector uses the local cluster state to get the -routing table without any network timeout issues unlike most other collectors. -Each shard is represented by a separate monitoring document. -| Jobs | `job_stats` -| Gathers details about all machine learning job statistics (for example, -`GET /_xpack/ml/anomaly_detectors/_stats`). -+ -This information only needs to be collected once, so it is collected on the -_elected_ master node. However, for the master node to be able to perform the -collection, the master node must have `xpack.ml.enabled` set to true (default) -and a license level that supports {ml}. -| Node Stats | `node_stats` -| Gathers details about the running node, such as memory utilization and CPU -usage (for example, `GET /_nodes/_local/stats`). -+ -This runs on _every_ node with {monitoring} enabled. One common failure -results in the timeout of the node stats request due to too many segment files. -As a result, the collector spends too much time waiting for the file system -stats to be calculated until it finally times out. A single `node_stats` -document is created per collection. This is collected per node to help to -discover issues with nodes communicating with each other, but not with the -monitoring cluster (for example, intermittent network issues or memory pressure). -|======================= - -{monitoring} uses a single threaded scheduler to run the collection of {es} -monitoring data by all of the appropriate collectors on each node. This -scheduler is managed locally by each node and its interval is controlled by -specifying the `xpack.monitoring.collection.interval`, which defaults to 10 -seconds (`10s`), at either the node or cluster level. - -Fundamentally, each collector works on the same principle. Per collection -interval, each collector is checked to see whether it should run and then the -appropriate collectors run. The failure of an individual collector does not -impact any other collector. - -Once collection has completed, all of the monitoring data is passed to the -exporters to route the monitoring data to the monitoring clusters. - -If gaps exist in the monitoring charts in {kib}, it is typically because either -a collector failed or the monitoring cluster did not receive the data (for -example, it was being restarted). In the event that a collector fails, a logged -error should exist on the node that attempted to perform the collection. - -NOTE: Collection is currently done serially, rather than in parallel, to avoid - extra overhead on the elected master node. The downside to this approach - is that collectors might observe a different version of the cluster state - within the same collection period. In practice, this does not make a - significant difference and running the collectors in parallel would not - prevent such a possibility. - -For more information about the configuration options for the collectors, see -<>. - -[float] -[[es-monitoring-stack]] -=== Collecting data from across the Elastic Stack - -{monitoring} in {es} also receives monitoring data from other parts of the -Elastic Stack. In this way, it serves as an unscheduled monitoring data -collector for the stack. - -By default, data collection is disabled. {es} monitoring data is not -collected and all monitoring data from other sources such as {kib}, Beats, and -Logstash is ignored. You must set `xpack.monitoring.collection.enabled` to `true` -to enable the collection of monitoring data. See <>. - -Once data is received, it is forwarded to the exporters -to be routed to the monitoring cluster like all monitoring data. - -WARNING: Because this stack-level "collector" lives outside of the collection -interval of {monitoring} for {es}, it is not impacted by the -`xpack.monitoring.collection.interval` setting. Therefore, data is passed to the -exporters whenever it is received. This behavior can result in indices for {kib}, -Logstash, or Beats being created somewhat unexpectedly. - -While the monitoring data is collected and processed, some production cluster -metadata is added to incoming documents. This metadata enables {kib} to link the -monitoring data to the appropriate cluster. If this linkage is unimportant to -the infrastructure that you're monitoring, it might be simpler to configure -Logstash and Beats to report monitoring data directly to the monitoring cluster. -This scenario also prevents the production cluster from adding extra overhead -related to monitoring data, which can be very useful when there are a large -number of Logstash nodes or Beats. - -For more information about typical monitoring architectures, see -{xpack-ref}/how-monitoring-works.html[How Monitoring Works]. diff --git a/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc b/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc index 1712c88380b7a..e705100e05e67 100644 --- a/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc +++ b/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold"] [[configuring-monitoring]] == Configuring Monitoring in {es} ++++ diff --git a/x-pack/docs/en/monitoring/exporters.asciidoc b/x-pack/docs/en/monitoring/exporters.asciidoc deleted file mode 100644 index e7727f1e97af0..0000000000000 --- a/x-pack/docs/en/monitoring/exporters.asciidoc +++ /dev/null @@ -1,171 +0,0 @@ -[role="xpack"] -[[es-monitoring-exporters]] -== Exporters - -The purpose of exporters is to take data collected from any Elastic Stack -source and route it to the monitoring cluster. It is possible to configure -more than one exporter, but the general and default setup is to use a single -exporter. - -There are two types of exporters in {es}: - -`local`:: -The default exporter used by {monitoring} for {es}. This exporter routes data -back into the _same_ cluster. See <>. - -`http`:: -The preferred exporter, which you can use to route data into any supported -{es} cluster accessible via HTTP. Production environments should always use a -separate monitoring cluster. See <>. - -Both exporters serve the same purpose: to set up the monitoring cluster and route -monitoring data. However, they perform these tasks in very different ways. Even -though things happen differently, both exporters are capable of sending all of -the same data. - -Exporters are configurable at both the node and cluster level. Cluster-wide -settings, which are updated with the -<>, take precedence over -settings in the `elasticsearch.yml` file on each node. When you update an -exporter, it is completely replaced by the updated version of the exporter. - -IMPORTANT: It is critical that all nodes share the same setup. Otherwise, -monitoring data might be routed in different ways or to different places. - -When the exporters route monitoring data into the monitoring cluster, they use -`_bulk` indexing for optimal performance. All monitoring data is forwarded in -bulk to all enabled exporters on the same node. From there, the exporters -serialize the monitoring data and send a bulk request to the monitoring cluster. -There is no queuing--in memory or persisted to disk--so any failure during the -export results in the loss of that batch of monitoring data. This design limits -the impact on {es} and the assumption is that the next pass will succeed. - -Routing monitoring data involves indexing it into the appropriate monitoring -indices. Once the data is indexed, it exists in a monitoring index that, by -default, is named with a daily index pattern. For {es} monitoring data, this is -an index that matches `.monitoring-es-6-*`. From there, the data lives inside -the monitoring cluster and must be curated or cleaned up as necessary. If you do -not curate the monitoring data, it eventually fills up the nodes and the cluster -might fail due to lack of disk space. - -TIP: You are strongly recommended to manage the curation of indices and -particularly the monitoring indices. To do so, you can take advantage of the -<> or -{curator-ref-current}/index.html[Elastic Curator]. - -//TO-DO: Add information about index lifecycle management https://github.com/elastic/x-pack-elasticsearch/issues/2814 - -When using cluster alerts, {watcher} creates daily `.watcher_history*` indices. -These are not managed by {monitoring} and they are not curated automatically. It -is therefore critical that you curate these indices to avoid an undesirable and -unexpected increase in the number of shards and indices and eventually the -amount of disk usage. If you are using a `local` exporter, you can set the -`xpack.watcher.history.cleaner_service.enabled` setting to `true` and curate the -`.watcher_history*` indices by using the -<>. See <>. - -There is also a disk watermark (known as the flood stage -watermark), which protects clusters from running out of disk space. When this -feature is triggered, it makes all indices (including monitoring indices) -read-only until the issue is fixed and a user manually makes the index writeable -again. While an active monitoring index is read-only, it will naturally fail to -write (index) new data and will continuously log errors that indicate the write -failure. For more information, see -{ref}/disk-allocator.html[Disk-based Shard Allocation]. - -[float] -[[es-monitoring-default-exporter]] -=== Default exporters - -If a node or cluster does not explicitly define an {monitoring} exporter, the -following default exporter is used: - -[source,yaml] ---------------------------------------------------- -xpack.monitoring.exporters.default_local: <1> - type: local ---------------------------------------------------- -<1> The exporter name uniquely defines the exporter, but it is otherwise unused. - When you specify your own exporters, you do not need to explicitly overwrite - or reference `default_local`. - -If another exporter is already defined, the default exporter is _not_ created. -When you define a new exporter, if the default exporter exists, it is -automatically removed. - -[float] -[[es-monitoring-templates]] -=== Exporter templates and ingest pipelines - -Before exporters can route monitoring data, they must set up certain {es} -resources. These resources include templates and ingest pipelines. The -following table lists the templates that are required before an exporter can -route monitoring data: - -[options="header"] -|======================= -| Template | Purpose -| `.monitoring-alerts` | All cluster alerts for monitoring data. -| `.monitoring-beats` | All Beats monitoring data. -| `.monitoring-es` | All {es} monitoring data. -| `.monitoring-kibana` | All {kib} monitoring data. -| `.monitoring-logstash` | All Logstash monitoring data. -|======================= - -The templates are ordinary {es} templates that control the default settings and -mappings for the monitoring indices. - -By default, monitoring indices are created daily (for example, -`.monitoring-es-6-2017.08.26`). You can change the default date suffix for -monitoring indices with the `index.name.time_format` setting. You can use this -setting to control how frequently monitoring indices are created by a specific -`http` exporter. You cannot use this setting with `local` exporters. For more -information, see <>. - -WARNING: Some users create their own templates that match _all_ index patterns, -which therefore impact the monitoring indices that get created. It is critical -that you do not disable `_source` storage for the monitoring indices. If you do, -{monitoring} for {kib} does not work and you cannot visualize monitoring data -for your cluster. - -The following table lists the ingest pipelines that are required before an -exporter can route monitoring data: - -[options="header"] -|======================= -| Pipeline | Purpose -| `xpack_monitoring_2` | Upgrades X-Pack monitoring data coming from X-Pack -5.0 - 5.4 to be compatible with the format used in {monitoring} 5.5. -| `xpack_monitoring_6` | A placeholder pipeline that is empty. -|======================= - -Exporters handle the setup of these resources before ever sending data. If -resource setup fails (for example, due to security permissions), no data is sent -and warnings are logged. - -NOTE: Empty pipelines are evaluated on the coordinating node during indexing and -they are ignored without any extra effort. This inherently makes them a safe, -no-op operation. - -For monitoring clusters that have disabled `node.ingest` on all nodes, it is -possible to disable the use of the ingest pipeline feature. However, doing so -blocks its purpose, which is to upgrade older monitoring data as our mappings -improve over time. Beginning in 6.0, the ingest pipeline feature is a -requirement on the monitoring cluster; you must have `node.ingest` enabled on at -least one node. - -WARNING: Once any node running 5.5 or later has set up the templates and ingest -pipeline on a monitoring cluster, you must use {kib} 5.5 or later to view all -subsequent data on the monitoring cluster. The easiest way to determine -whether this update has occurred is by checking for the presence of indices -matching `.monitoring-es-6-*` (or more concretely the existence of the -new pipeline). Versions prior to 5.5 used `.monitoring-es-2-*`. - -Each resource that is created by an {monitoring} exporter has a `version` field, -which is used to determine whether the resource should be replaced. The `version` -field value represents the latest version of {monitoring} that changed the -resource. If a resource is edited by someone or something external to -{monitoring}, those changes are lost the next time an automatic update occurs. - -include::local-export.asciidoc[] -include::http-export.asciidoc[] diff --git a/x-pack/docs/en/monitoring/http-export.asciidoc b/x-pack/docs/en/monitoring/http-export.asciidoc deleted file mode 100644 index db1dbe2a29c5b..0000000000000 --- a/x-pack/docs/en/monitoring/http-export.asciidoc +++ /dev/null @@ -1,116 +0,0 @@ -[role="xpack"] -[[http-exporter]] -=== HTTP Exporters - -The `http` exporter is the preferred exporter in {monitoring} because it enables -the use of a separate monitoring cluster. As a secondary benefit, it avoids -using a production cluster node as a coordinating node for indexing monitoring -data because all requests are HTTP requests to the monitoring cluster. - -The `http` exporter uses the low-level {es} REST Client, which enables it to -send its data to any {es} cluster it can access through the network. Its requests -make use of the <> parameter to -reduce bandwidth whenever possible, which helps to ensure that communications -between the production and monitoring clusters are as lightweight as possible. - -The `http` exporter supports a number of settings that control how it -communicates over HTTP to remote clusters. In most cases, it is not -necessary to explicitly configure these settings. For detailed -descriptions, see <>. - -[source,yaml] ----------------------------------- -xpack.monitoring.exporters: - my_local: <1> - type: local - my_remote: <2> - type: http - host: [ "10.1.2.3:9200", ... ] <3> - auth: <4> - username: my_username - password: changeme - connection: - timeout: 6s - read_timeout: 60s - ssl: ... <5> - proxy: - base_path: /some/base/path <6> - headers: <7> - My-Proxy-Header: abc123 - My-Other-Thing: [ def456, ... ] - index.name.time_format: YYYY-MM <8> - ----------------------------------- -<1> A `local` exporter defined explicitly whose arbitrary name is `my_local`. -<2> An `http` exporter defined whose arbitrary name is `my_remote`. This name -uniquely defines the exporter but is otherwise unused. -<3> `host` is a required setting for `http` exporters. It must specify the HTTP -port rather than the transport port. The default port value is `9200`. -<4> User authentication for those using {security} or some other - form of user authentication protecting the cluster. -<5> See <> for all TLS/SSL settings. If not supplied, -the default node-level TLS/SSL settings are used. -<6> Optional base path to prefix any outgoing request with in order to - work with proxies. -<7> Arbitrary key/value pairs to define as headers to send with every request. - The array-based key/value format sends one header per value. -<8> A mechanism for changing the date suffix used by default. - -NOTE: The `http` exporter accepts an array of `hosts` and it will round robin -through the list. It is a good idea to take advantage of that feature when the -monitoring cluster contains more than one node. - -Unlike the `local` exporter, _every_ node that uses the `http` exporter attempts -to check and create the resources that it needs. The `http` exporter avoids -re-checking the resources unless something triggers it to perform the checks -again. These triggers include: - -* The production cluster's node restarts. -* A connection failure to the monitoring cluster. -* The license on the production cluster changes. -* The `http` exporter is dynamically updated (and it is therefore replaced). - -The easiest way to trigger a check is to disable, then re-enable the exporter. - -WARNING: This resource management behavior can create a hole for users that -delete monitoring resources. Since the `http` exporter does not re-check its -resources unless one of the triggers occurs, this can result in malformed index -mappings. - -Unlike the `local` exporter, the `http` exporter is inherently routing requests -outside of the cluster. This situation means that the exporter must provide a -username and password when the monitoring cluster requires one (or other -appropriate security configurations, such as TLS/SSL settings). - -IMPORTANT: When discussing security relative to the `http` exporter, it is -critical to remember that all users are managed on the monitoring cluster. This -is particularly important to remember when you move from development -environments to production environments, where you often have dedicated -monitoring clusters. - -For more information about the configuration options for the `http` exporter, -see <>. - -[float] -[[http-exporter-dns]] -==== Using DNS Hosts in HTTP Exporters - -{monitoring} runs inside of the the JVM security manager. When the JVM has the -security manager enabled, the JVM changes the duration so that it caches DNS -lookups indefinitely (for example, the mapping of a DNS hostname to an IP -address). For this reason, if you are in an environment where the DNS response -might change from time-to-time (for example, talking to any load balanced cloud -provider), you are strongly discouraged from using DNS hostnames. - -Alternatively, you can set the JVM security property `networkaddress.cache.ttl`, -which accepts values in seconds. This property must be set for the node's JVM that -uses {monitoring} for {es} when using DNS that can change IP addresses. If you -do not apply this setting, the connection consistently fails after the IP -address changes. - -IMPORTANT: JVM security properties are different than system properties. They -cannot be set at startup via `-D` system property settings and instead they must -be set in code before the security manager has been setup _or_, more -appropriately, in the `$JAVA_HOME/lib/security/java.security` file. - -Restarting the node (and therefore the JVM) results in its cache being flushed. diff --git a/x-pack/docs/en/monitoring/index.asciidoc b/x-pack/docs/en/monitoring/index.asciidoc deleted file mode 100644 index 6b8ecc5038ea0..0000000000000 --- a/x-pack/docs/en/monitoring/index.asciidoc +++ /dev/null @@ -1,45 +0,0 @@ -[role="xpack"] -[[es-monitoring]] -= Monitoring {es} - -[partintro] --- -{monitoring} enables you to easily monitor the health of your {es} cluster. The -monitoring metrics are collected from each node and stored in {es} indices. - -Each {es} node is considered unique based on its persistent UUID, which is -written on first start to its <> directory, which -defaults to `./data`. - -All settings associated with {monitoring} in {es} must be set in either the -`elasticsearch.yml` file for each node or, where possible, in the dynamic -cluster settings. For more information, see <>. - -[[es-monitoring-overview]] -{es} is also at the core of {monitoring} across the Elastic Stack. In all cases, -{monitoring} documents are just ordinary JSON documents built by monitoring each -Elastic Stack component at some collection interval, then indexing those -documents into the monitoring cluster. Each component in the stack is -responsible for monitoring itself and then forwarding those documents to {es} -for both routing and indexing (storage). - -The routing and indexing processes in {es} are handled by what are called -<> and -<>. In the past, collectors and exporters -were considered to be part of a monitoring "agent", but that term is generally -not used anymore. - -You can view monitoring data from {kib} where it’s easy to spot issues at a -glance or delve into the system behavior over time to diagnose operational -issues. In addition to the built-in status warnings, you can also set up custom -alerts based on the data in the monitoring indices. - -For an introduction to monitoring your Elastic stack, including Beats, Logstash, -and {kib}, see {xpack-ref}/xpack-monitoring.html[Monitoring the Elastic Stack]. - --- - -include::collectors.asciidoc[] -include::exporters.asciidoc[] -include::pause-export.asciidoc[] - diff --git a/x-pack/docs/en/monitoring/indices.asciidoc b/x-pack/docs/en/monitoring/indices.asciidoc index efa9836daa2e9..a27d91d423e8d 100644 --- a/x-pack/docs/en/monitoring/indices.asciidoc +++ b/x-pack/docs/en/monitoring/indices.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="basic"] [[config-monitoring-indices]] === Configuring Indices for Monitoring diff --git a/x-pack/docs/en/monitoring/local-export.asciidoc b/x-pack/docs/en/monitoring/local-export.asciidoc deleted file mode 100644 index 12d0ab5ea9f81..0000000000000 --- a/x-pack/docs/en/monitoring/local-export.asciidoc +++ /dev/null @@ -1,82 +0,0 @@ -[role="xpack"] -[[local-exporter]] -=== Local Exporters - -The `local` exporter is the default exporter in {monitoring}. It routes data -back into the same (local) cluster. In other words, it uses the production -cluster as the monitoring cluster. For example: - -[source,yaml] ---------------------------------------------------- -xpack.monitoring.exporters.my_local_exporter: <1> - type: local ---------------------------------------------------- -<1> The exporter name uniquely defines the exporter, but it is otherwise unused. - -This exporter exists to provide a convenient option when hardware is simply not -available. It is also a way for developers to get an idea of what their actions -do for pre-production clusters when they do not have the time or resources to -provide a separate monitoring cluster. However, this exporter has disadvantages -that impact the local cluster: - -* All indexing impacts the local cluster and the nodes that hold the monitoring -indices' shards. -* Most collectors run on the elected master node. Therefore most indexing occurs -with the elected master node as the coordinating node, which is a bad practice. -* Any usage of {monitoring} for {kib} uses the local cluster's resources for -searches and aggregations, which means that they might not be available for -non-monitoring tasks. -* If the local cluster goes down, the monitoring cluster has inherently gone -down with it (and vice versa), which generally defeats the purpose of monitoring. - -For the `local` exporter, all setup occurs only on the elected master node. This -means that if you do not see any monitoring templates or ingest pipelines, the -elected master node is having issues or it is not configured in the same way. -Unlike the `http` exporter, the `local` exporter has the advantage of accessing -the monitoring cluster's up-to-date cluster state. It can therefore always check -that the templates and ingest pipelines exist without a performance penalty. If -the elected master node encounters errors while trying to create the monitoring -resources, it logs errors, ignores that collection, and tries again after the -next collection. - -The elected master node is the only node to set up resources for the `local` -exporter. Therefore all other nodes wait for the resources to be set up before -indexing any monitoring data from their own collectors. Each of these nodes logs -a message indicating that they are waiting for the resources to be set up. - -One benefit of the `local` exporter is that it lives within the cluster and -therefore no extra configuration is required when the cluster is secured with -{security}. All operations, including indexing operations, that occur from a -`local` exporter make use of the internal transport mechanisms within {es}. This -behavior enables the exporter to be used without providing any user credentials -when {security} is enabled. - -For more information about the configuration options for the `local` exporter, -see <>. - -[[local-exporter-cleaner]] -==== Cleaner Service - -One feature of the `local` exporter, which is not present in the `http` exporter, -is a cleaner service. The cleaner service runs once per day at 01:00 AM UTC on -the elected master node. - -The role of the cleaner service is to clean, or curate, the monitoring indices -that are older than a configurable amount of time (the default is `7d`). This -cleaner exists as part of the `local` exporter as a safety mechanism. The `http` -exporter does not make use of it because it could enable a single misconfigured -node to prematurely curate data from other production clusters that share the -same monitoring cluster. - -In a dedicated monitoring cluster, the cleaning service can be used without -having to also monitor the monitoring cluster. For example: - -[source,yaml] ---------------------------------------------------- -xpack.monitoring.collection.enabled: false <1> -xpack.monitoring.history.duration: 3d <2> ---------------------------------------------------- -<1> Disable the collection of data on the monitoring cluster. -<2> Lower the default history duration from `7d` to `3d`. The minimum value is -`1d`. This setting can be modified only when using a Gold or higher level -license. For the Basic license level, it uses the default of 7 days. diff --git a/x-pack/docs/en/monitoring/pause-export.asciidoc b/x-pack/docs/en/monitoring/pause-export.asciidoc deleted file mode 100644 index d26799c6892c3..0000000000000 --- a/x-pack/docs/en/monitoring/pause-export.asciidoc +++ /dev/null @@ -1,35 +0,0 @@ -[role="xpack"] -[[pause-export]] -== Pausing Data Collection - -To stop generating {monitoring} data in {es}, disable data collection: - -[source,yaml] ---------------------------------------------------- -xpack.monitoring.collection.enabled: false ---------------------------------------------------- - -When this setting is `false`, {es} monitoring data is not collected and all -monitoring data from other sources such as {kib}, Beats, and Logstash is ignored. - -You can update this setting by using the -{ref}/cluster-update-settings.html[Cluster Update Settings API]. - -If you want to separately disable a specific exporter, you can specify the -`enabled` setting (which defaults to `true`) per exporter. For example: - -[source,yaml] ---------------------------------------------------- -xpack.monitoring.exporters.my_http_exporter: - type: http - host: ["10.1.2.3:9200", "10.1.2.4:9200"] - enabled: false <1> ---------------------------------------------------- -<1> Disable the named exporter. If the same name as an existing exporter is not - used, then this will create a completely new exporter that is completely - ignored. This value can be set dynamically by using cluster settings. - -NOTE: Defining a disabled exporter prevents the default exporter from being - created. - -To re-start data collection, re-enable these settings. \ No newline at end of file From 1abde647dce0e82463da529ced71d22a6f96ebee Mon Sep 17 00:00:00 2001 From: lcawl Date: Wed, 20 Jun 2018 11:03:23 -0700 Subject: [PATCH 2/2] [DOCS] More monitoring docs --- docs/reference/monitoring/collectors.asciidoc | 150 +++++++++++++++ docs/reference/monitoring/exporters.asciidoc | 172 ++++++++++++++++++ .../reference/monitoring/http-export.asciidoc | 117 ++++++++++++ docs/reference/monitoring/index.asciidoc | 46 +++++ .../monitoring/local-export.asciidoc | 83 +++++++++ .../monitoring/pause-export.asciidoc | 36 ++++ 6 files changed, 604 insertions(+) create mode 100644 docs/reference/monitoring/collectors.asciidoc create mode 100644 docs/reference/monitoring/exporters.asciidoc create mode 100644 docs/reference/monitoring/http-export.asciidoc create mode 100644 docs/reference/monitoring/index.asciidoc create mode 100644 docs/reference/monitoring/local-export.asciidoc create mode 100644 docs/reference/monitoring/pause-export.asciidoc diff --git a/docs/reference/monitoring/collectors.asciidoc b/docs/reference/monitoring/collectors.asciidoc new file mode 100644 index 0000000000000..bd48d1287006a --- /dev/null +++ b/docs/reference/monitoring/collectors.asciidoc @@ -0,0 +1,150 @@ +[role="xpack"] +[testenv="basic"] +[[es-monitoring-collectors]] +== Collectors + +Collectors, as their name implies, collect things. Each collector runs once for +each collection interval to obtain data from the public APIs in {es} and {xpack} +that it chooses to monitor. When the data collection is finished, the data is +handed in bulk to the <> to be sent to the +monitoring clusters. Regardless of the number of exporters, each collector only +runs once per collection interval. + +There is only one collector per data type gathered. In other words, for any +monitoring document that is created, it comes from a single collector rather +than being merged from multiple collectors. {monitoring} for {es} currently has +a few collectors because the goal is to minimize overlap between them for +optimal performance. + +Each collector can create zero or more monitoring documents. For example, +the `index_stats` collector collects all index statistics at the same time to +avoid many unnecessary calls. + +[options="header"] +|======================= +| Collector | Data Types | Description +| Cluster Stats | `cluster_stats` +| Gathers details about the cluster state, including parts of +the actual cluster state (for example `GET /_cluster/state`) and statistics +about it (for example, `GET /_cluster/stats`). This produces a single document +type. In versions prior to X-Pack 5.5, this was actually three separate collectors +that resulted in three separate types: `cluster_stats`, `cluster_state`, and +`cluster_info`. In 5.5 and later, all three are combined into `cluster_stats`. ++ +This only runs on the _elected_ master node and the data collected +(`cluster_stats`) largely controls the UI. When this data is not present, it +indicates either a misconfiguration on the elected master node, timeouts related +to the collection of the data, or issues with storing the data. Only a single +document is produced per collection. +| Index Stats | `indices_stats`, `index_stats` +| Gathers details about the indices in the cluster, both in summary and +individually. This creates many documents that represent parts of the index +statistics output (for example, `GET /_stats`). ++ +This information only needs to be collected once, so it is collected on the +_elected_ master node. The most common failure for this collector relates to an +extreme number of indices -- and therefore time to gather them -- resulting in +timeouts. One summary `indices_stats` document is produced per collection and one +`index_stats` document is produced per index, per collection. +| Index Recovery | `index_recovery` +| Gathers details about index recovery in the cluster. Index recovery represents +the assignment of _shards_ at the cluster level. If an index is not recovered, +it is not usable. This also corresponds to shard restoration via snapshots. ++ +This information only needs to be collected once, so it is collected on the +_elected_ master node. The most common failure for this collector relates to an +extreme number of shards -- and therefore time to gather them -- resulting in +timeouts. This creates a single document that contains all recoveries by default, +which can be quite large, but it gives the most accurate picture of recovery in +the production cluster. +| Shards | `shards` +| Gathers details about all _allocated_ shards for all indices, particularly +including what node the shard is allocated to. ++ +This information only needs to be collected once, so it is collected on the +_elected_ master node. The collector uses the local cluster state to get the +routing table without any network timeout issues unlike most other collectors. +Each shard is represented by a separate monitoring document. +| Jobs | `job_stats` +| Gathers details about all machine learning job statistics (for example, +`GET /_xpack/ml/anomaly_detectors/_stats`). ++ +This information only needs to be collected once, so it is collected on the +_elected_ master node. However, for the master node to be able to perform the +collection, the master node must have `xpack.ml.enabled` set to true (default) +and a license level that supports {ml}. +| Node Stats | `node_stats` +| Gathers details about the running node, such as memory utilization and CPU +usage (for example, `GET /_nodes/_local/stats`). ++ +This runs on _every_ node with {monitoring} enabled. One common failure +results in the timeout of the node stats request due to too many segment files. +As a result, the collector spends too much time waiting for the file system +stats to be calculated until it finally times out. A single `node_stats` +document is created per collection. This is collected per node to help to +discover issues with nodes communicating with each other, but not with the +monitoring cluster (for example, intermittent network issues or memory pressure). +|======================= + +{monitoring} uses a single threaded scheduler to run the collection of {es} +monitoring data by all of the appropriate collectors on each node. This +scheduler is managed locally by each node and its interval is controlled by +specifying the `xpack.monitoring.collection.interval`, which defaults to 10 +seconds (`10s`), at either the node or cluster level. + +Fundamentally, each collector works on the same principle. Per collection +interval, each collector is checked to see whether it should run and then the +appropriate collectors run. The failure of an individual collector does not +impact any other collector. + +Once collection has completed, all of the monitoring data is passed to the +exporters to route the monitoring data to the monitoring clusters. + +If gaps exist in the monitoring charts in {kib}, it is typically because either +a collector failed or the monitoring cluster did not receive the data (for +example, it was being restarted). In the event that a collector fails, a logged +error should exist on the node that attempted to perform the collection. + +NOTE: Collection is currently done serially, rather than in parallel, to avoid + extra overhead on the elected master node. The downside to this approach + is that collectors might observe a different version of the cluster state + within the same collection period. In practice, this does not make a + significant difference and running the collectors in parallel would not + prevent such a possibility. + +For more information about the configuration options for the collectors, see +<>. + +[float] +[[es-monitoring-stack]] +=== Collecting data from across the Elastic Stack + +{monitoring} in {es} also receives monitoring data from other parts of the +Elastic Stack. In this way, it serves as an unscheduled monitoring data +collector for the stack. + +By default, data collection is disabled. {es} monitoring data is not +collected and all monitoring data from other sources such as {kib}, Beats, and +Logstash is ignored. You must set `xpack.monitoring.collection.enabled` to `true` +to enable the collection of monitoring data. See <>. + +Once data is received, it is forwarded to the exporters +to be routed to the monitoring cluster like all monitoring data. + +WARNING: Because this stack-level "collector" lives outside of the collection +interval of {monitoring} for {es}, it is not impacted by the +`xpack.monitoring.collection.interval` setting. Therefore, data is passed to the +exporters whenever it is received. This behavior can result in indices for {kib}, +Logstash, or Beats being created somewhat unexpectedly. + +While the monitoring data is collected and processed, some production cluster +metadata is added to incoming documents. This metadata enables {kib} to link the +monitoring data to the appropriate cluster. If this linkage is unimportant to +the infrastructure that you're monitoring, it might be simpler to configure +Logstash and Beats to report monitoring data directly to the monitoring cluster. +This scenario also prevents the production cluster from adding extra overhead +related to monitoring data, which can be very useful when there are a large +number of Logstash nodes or Beats. + +For more information about typical monitoring architectures, see +{xpack-ref}/how-monitoring-works.html[How Monitoring Works]. diff --git a/docs/reference/monitoring/exporters.asciidoc b/docs/reference/monitoring/exporters.asciidoc new file mode 100644 index 0000000000000..2a7729eee9425 --- /dev/null +++ b/docs/reference/monitoring/exporters.asciidoc @@ -0,0 +1,172 @@ +[role="xpack"] +[testenv="basic"] +[[es-monitoring-exporters]] +== Exporters + +The purpose of exporters is to take data collected from any Elastic Stack +source and route it to the monitoring cluster. It is possible to configure +more than one exporter, but the general and default setup is to use a single +exporter. + +There are two types of exporters in {es}: + +`local`:: +The default exporter used by {monitoring} for {es}. This exporter routes data +back into the _same_ cluster. See <>. + +`http`:: +The preferred exporter, which you can use to route data into any supported +{es} cluster accessible via HTTP. Production environments should always use a +separate monitoring cluster. See <>. + +Both exporters serve the same purpose: to set up the monitoring cluster and route +monitoring data. However, they perform these tasks in very different ways. Even +though things happen differently, both exporters are capable of sending all of +the same data. + +Exporters are configurable at both the node and cluster level. Cluster-wide +settings, which are updated with the +<>, take precedence over +settings in the `elasticsearch.yml` file on each node. When you update an +exporter, it is completely replaced by the updated version of the exporter. + +IMPORTANT: It is critical that all nodes share the same setup. Otherwise, +monitoring data might be routed in different ways or to different places. + +When the exporters route monitoring data into the monitoring cluster, they use +`_bulk` indexing for optimal performance. All monitoring data is forwarded in +bulk to all enabled exporters on the same node. From there, the exporters +serialize the monitoring data and send a bulk request to the monitoring cluster. +There is no queuing--in memory or persisted to disk--so any failure during the +export results in the loss of that batch of monitoring data. This design limits +the impact on {es} and the assumption is that the next pass will succeed. + +Routing monitoring data involves indexing it into the appropriate monitoring +indices. Once the data is indexed, it exists in a monitoring index that, by +default, is named with a daily index pattern. For {es} monitoring data, this is +an index that matches `.monitoring-es-6-*`. From there, the data lives inside +the monitoring cluster and must be curated or cleaned up as necessary. If you do +not curate the monitoring data, it eventually fills up the nodes and the cluster +might fail due to lack of disk space. + +TIP: You are strongly recommended to manage the curation of indices and +particularly the monitoring indices. To do so, you can take advantage of the +<> or +{curator-ref-current}/index.html[Elastic Curator]. + +//TO-DO: Add information about index lifecycle management https://github.com/elastic/x-pack-elasticsearch/issues/2814 + +When using cluster alerts, {watcher} creates daily `.watcher_history*` indices. +These are not managed by {monitoring} and they are not curated automatically. It +is therefore critical that you curate these indices to avoid an undesirable and +unexpected increase in the number of shards and indices and eventually the +amount of disk usage. If you are using a `local` exporter, you can set the +`xpack.watcher.history.cleaner_service.enabled` setting to `true` and curate the +`.watcher_history*` indices by using the +<>. See <>. + +There is also a disk watermark (known as the flood stage +watermark), which protects clusters from running out of disk space. When this +feature is triggered, it makes all indices (including monitoring indices) +read-only until the issue is fixed and a user manually makes the index writeable +again. While an active monitoring index is read-only, it will naturally fail to +write (index) new data and will continuously log errors that indicate the write +failure. For more information, see +{ref}/disk-allocator.html[Disk-based Shard Allocation]. + +[float] +[[es-monitoring-default-exporter]] +=== Default exporters + +If a node or cluster does not explicitly define an {monitoring} exporter, the +following default exporter is used: + +[source,yaml] +--------------------------------------------------- +xpack.monitoring.exporters.default_local: <1> + type: local +--------------------------------------------------- +<1> The exporter name uniquely defines the exporter, but it is otherwise unused. + When you specify your own exporters, you do not need to explicitly overwrite + or reference `default_local`. + +If another exporter is already defined, the default exporter is _not_ created. +When you define a new exporter, if the default exporter exists, it is +automatically removed. + +[float] +[[es-monitoring-templates]] +=== Exporter templates and ingest pipelines + +Before exporters can route monitoring data, they must set up certain {es} +resources. These resources include templates and ingest pipelines. The +following table lists the templates that are required before an exporter can +route monitoring data: + +[options="header"] +|======================= +| Template | Purpose +| `.monitoring-alerts` | All cluster alerts for monitoring data. +| `.monitoring-beats` | All Beats monitoring data. +| `.monitoring-es` | All {es} monitoring data. +| `.monitoring-kibana` | All {kib} monitoring data. +| `.monitoring-logstash` | All Logstash monitoring data. +|======================= + +The templates are ordinary {es} templates that control the default settings and +mappings for the monitoring indices. + +By default, monitoring indices are created daily (for example, +`.monitoring-es-6-2017.08.26`). You can change the default date suffix for +monitoring indices with the `index.name.time_format` setting. You can use this +setting to control how frequently monitoring indices are created by a specific +`http` exporter. You cannot use this setting with `local` exporters. For more +information, see <>. + +WARNING: Some users create their own templates that match _all_ index patterns, +which therefore impact the monitoring indices that get created. It is critical +that you do not disable `_source` storage for the monitoring indices. If you do, +{monitoring} for {kib} does not work and you cannot visualize monitoring data +for your cluster. + +The following table lists the ingest pipelines that are required before an +exporter can route monitoring data: + +[options="header"] +|======================= +| Pipeline | Purpose +| `xpack_monitoring_2` | Upgrades X-Pack monitoring data coming from X-Pack +5.0 - 5.4 to be compatible with the format used in {monitoring} 5.5. +| `xpack_monitoring_6` | A placeholder pipeline that is empty. +|======================= + +Exporters handle the setup of these resources before ever sending data. If +resource setup fails (for example, due to security permissions), no data is sent +and warnings are logged. + +NOTE: Empty pipelines are evaluated on the coordinating node during indexing and +they are ignored without any extra effort. This inherently makes them a safe, +no-op operation. + +For monitoring clusters that have disabled `node.ingest` on all nodes, it is +possible to disable the use of the ingest pipeline feature. However, doing so +blocks its purpose, which is to upgrade older monitoring data as our mappings +improve over time. Beginning in 6.0, the ingest pipeline feature is a +requirement on the monitoring cluster; you must have `node.ingest` enabled on at +least one node. + +WARNING: Once any node running 5.5 or later has set up the templates and ingest +pipeline on a monitoring cluster, you must use {kib} 5.5 or later to view all +subsequent data on the monitoring cluster. The easiest way to determine +whether this update has occurred is by checking for the presence of indices +matching `.monitoring-es-6-*` (or more concretely the existence of the +new pipeline). Versions prior to 5.5 used `.monitoring-es-2-*`. + +Each resource that is created by an {monitoring} exporter has a `version` field, +which is used to determine whether the resource should be replaced. The `version` +field value represents the latest version of {monitoring} that changed the +resource. If a resource is edited by someone or something external to +{monitoring}, those changes are lost the next time an automatic update occurs. + +include::local-export.asciidoc[] +include::http-export.asciidoc[] diff --git a/docs/reference/monitoring/http-export.asciidoc b/docs/reference/monitoring/http-export.asciidoc new file mode 100644 index 0000000000000..4dfe1a0c537ea --- /dev/null +++ b/docs/reference/monitoring/http-export.asciidoc @@ -0,0 +1,117 @@ +[role="xpack"] +[testenv="basic"] +[[http-exporter]] +=== HTTP Exporters + +The `http` exporter is the preferred exporter in {monitoring} because it enables +the use of a separate monitoring cluster. As a secondary benefit, it avoids +using a production cluster node as a coordinating node for indexing monitoring +data because all requests are HTTP requests to the monitoring cluster. + +The `http` exporter uses the low-level {es} REST Client, which enables it to +send its data to any {es} cluster it can access through the network. Its requests +make use of the <> parameter to +reduce bandwidth whenever possible, which helps to ensure that communications +between the production and monitoring clusters are as lightweight as possible. + +The `http` exporter supports a number of settings that control how it +communicates over HTTP to remote clusters. In most cases, it is not +necessary to explicitly configure these settings. For detailed +descriptions, see <>. + +[source,yaml] +---------------------------------- +xpack.monitoring.exporters: + my_local: <1> + type: local + my_remote: <2> + type: http + host: [ "10.1.2.3:9200", ... ] <3> + auth: <4> + username: my_username + password: changeme + connection: + timeout: 6s + read_timeout: 60s + ssl: ... <5> + proxy: + base_path: /some/base/path <6> + headers: <7> + My-Proxy-Header: abc123 + My-Other-Thing: [ def456, ... ] + index.name.time_format: YYYY-MM <8> + +---------------------------------- +<1> A `local` exporter defined explicitly whose arbitrary name is `my_local`. +<2> An `http` exporter defined whose arbitrary name is `my_remote`. This name +uniquely defines the exporter but is otherwise unused. +<3> `host` is a required setting for `http` exporters. It must specify the HTTP +port rather than the transport port. The default port value is `9200`. +<4> User authentication for those using {security} or some other + form of user authentication protecting the cluster. +<5> See <> for all TLS/SSL settings. If not supplied, +the default node-level TLS/SSL settings are used. +<6> Optional base path to prefix any outgoing request with in order to + work with proxies. +<7> Arbitrary key/value pairs to define as headers to send with every request. + The array-based key/value format sends one header per value. +<8> A mechanism for changing the date suffix used by default. + +NOTE: The `http` exporter accepts an array of `hosts` and it will round robin +through the list. It is a good idea to take advantage of that feature when the +monitoring cluster contains more than one node. + +Unlike the `local` exporter, _every_ node that uses the `http` exporter attempts +to check and create the resources that it needs. The `http` exporter avoids +re-checking the resources unless something triggers it to perform the checks +again. These triggers include: + +* The production cluster's node restarts. +* A connection failure to the monitoring cluster. +* The license on the production cluster changes. +* The `http` exporter is dynamically updated (and it is therefore replaced). + +The easiest way to trigger a check is to disable, then re-enable the exporter. + +WARNING: This resource management behavior can create a hole for users that +delete monitoring resources. Since the `http` exporter does not re-check its +resources unless one of the triggers occurs, this can result in malformed index +mappings. + +Unlike the `local` exporter, the `http` exporter is inherently routing requests +outside of the cluster. This situation means that the exporter must provide a +username and password when the monitoring cluster requires one (or other +appropriate security configurations, such as TLS/SSL settings). + +IMPORTANT: When discussing security relative to the `http` exporter, it is +critical to remember that all users are managed on the monitoring cluster. This +is particularly important to remember when you move from development +environments to production environments, where you often have dedicated +monitoring clusters. + +For more information about the configuration options for the `http` exporter, +see <>. + +[float] +[[http-exporter-dns]] +==== Using DNS Hosts in HTTP Exporters + +{monitoring} runs inside of the the JVM security manager. When the JVM has the +security manager enabled, the JVM changes the duration so that it caches DNS +lookups indefinitely (for example, the mapping of a DNS hostname to an IP +address). For this reason, if you are in an environment where the DNS response +might change from time-to-time (for example, talking to any load balanced cloud +provider), you are strongly discouraged from using DNS hostnames. + +Alternatively, you can set the JVM security property `networkaddress.cache.ttl`, +which accepts values in seconds. This property must be set for the node's JVM that +uses {monitoring} for {es} when using DNS that can change IP addresses. If you +do not apply this setting, the connection consistently fails after the IP +address changes. + +IMPORTANT: JVM security properties are different than system properties. They +cannot be set at startup via `-D` system property settings and instead they must +be set in code before the security manager has been setup _or_, more +appropriately, in the `$JAVA_HOME/lib/security/java.security` file. + +Restarting the node (and therefore the JVM) results in its cache being flushed. diff --git a/docs/reference/monitoring/index.asciidoc b/docs/reference/monitoring/index.asciidoc new file mode 100644 index 0000000000000..d6a55f44585dd --- /dev/null +++ b/docs/reference/monitoring/index.asciidoc @@ -0,0 +1,46 @@ +[role="xpack"] +[testenv="basic"] +[[es-monitoring]] += Monitoring {es} + +[partintro] +-- +{monitoring} enables you to easily monitor the health of your {es} cluster. The +monitoring metrics are collected from each node and stored in {es} indices. + +Each {es} node is considered unique based on its persistent UUID, which is +written on first start to its <> directory, which +defaults to `./data`. + +All settings associated with {monitoring} in {es} must be set in either the +`elasticsearch.yml` file for each node or, where possible, in the dynamic +cluster settings. For more information, see <>. + +[[es-monitoring-overview]] +{es} is also at the core of {monitoring} across the Elastic Stack. In all cases, +{monitoring} documents are just ordinary JSON documents built by monitoring each +Elastic Stack component at some collection interval, then indexing those +documents into the monitoring cluster. Each component in the stack is +responsible for monitoring itself and then forwarding those documents to {es} +for both routing and indexing (storage). + +The routing and indexing processes in {es} are handled by what are called +<> and +<>. In the past, collectors and exporters +were considered to be part of a monitoring "agent", but that term is generally +not used anymore. + +You can view monitoring data from {kib} where it’s easy to spot issues at a +glance or delve into the system behavior over time to diagnose operational +issues. In addition to the built-in status warnings, you can also set up custom +alerts based on the data in the monitoring indices. + +For an introduction to monitoring your Elastic stack, including Beats, Logstash, +and {kib}, see {xpack-ref}/xpack-monitoring.html[Monitoring the Elastic Stack]. + +-- + +include::collectors.asciidoc[] +include::exporters.asciidoc[] +include::pause-export.asciidoc[] + diff --git a/docs/reference/monitoring/local-export.asciidoc b/docs/reference/monitoring/local-export.asciidoc new file mode 100644 index 0000000000000..2bc757f07ecc8 --- /dev/null +++ b/docs/reference/monitoring/local-export.asciidoc @@ -0,0 +1,83 @@ +[role="xpack"] +[testenv="basic"] +[[local-exporter]] +=== Local Exporters + +The `local` exporter is the default exporter in {monitoring}. It routes data +back into the same (local) cluster. In other words, it uses the production +cluster as the monitoring cluster. For example: + +[source,yaml] +--------------------------------------------------- +xpack.monitoring.exporters.my_local_exporter: <1> + type: local +--------------------------------------------------- +<1> The exporter name uniquely defines the exporter, but it is otherwise unused. + +This exporter exists to provide a convenient option when hardware is simply not +available. It is also a way for developers to get an idea of what their actions +do for pre-production clusters when they do not have the time or resources to +provide a separate monitoring cluster. However, this exporter has disadvantages +that impact the local cluster: + +* All indexing impacts the local cluster and the nodes that hold the monitoring +indices' shards. +* Most collectors run on the elected master node. Therefore most indexing occurs +with the elected master node as the coordinating node, which is a bad practice. +* Any usage of {monitoring} for {kib} uses the local cluster's resources for +searches and aggregations, which means that they might not be available for +non-monitoring tasks. +* If the local cluster goes down, the monitoring cluster has inherently gone +down with it (and vice versa), which generally defeats the purpose of monitoring. + +For the `local` exporter, all setup occurs only on the elected master node. This +means that if you do not see any monitoring templates or ingest pipelines, the +elected master node is having issues or it is not configured in the same way. +Unlike the `http` exporter, the `local` exporter has the advantage of accessing +the monitoring cluster's up-to-date cluster state. It can therefore always check +that the templates and ingest pipelines exist without a performance penalty. If +the elected master node encounters errors while trying to create the monitoring +resources, it logs errors, ignores that collection, and tries again after the +next collection. + +The elected master node is the only node to set up resources for the `local` +exporter. Therefore all other nodes wait for the resources to be set up before +indexing any monitoring data from their own collectors. Each of these nodes logs +a message indicating that they are waiting for the resources to be set up. + +One benefit of the `local` exporter is that it lives within the cluster and +therefore no extra configuration is required when the cluster is secured with +{security}. All operations, including indexing operations, that occur from a +`local` exporter make use of the internal transport mechanisms within {es}. This +behavior enables the exporter to be used without providing any user credentials +when {security} is enabled. + +For more information about the configuration options for the `local` exporter, +see <>. + +[[local-exporter-cleaner]] +==== Cleaner Service + +One feature of the `local` exporter, which is not present in the `http` exporter, +is a cleaner service. The cleaner service runs once per day at 01:00 AM UTC on +the elected master node. + +The role of the cleaner service is to clean, or curate, the monitoring indices +that are older than a configurable amount of time (the default is `7d`). This +cleaner exists as part of the `local` exporter as a safety mechanism. The `http` +exporter does not make use of it because it could enable a single misconfigured +node to prematurely curate data from other production clusters that share the +same monitoring cluster. + +In a dedicated monitoring cluster, the cleaning service can be used without +having to also monitor the monitoring cluster. For example: + +[source,yaml] +--------------------------------------------------- +xpack.monitoring.collection.enabled: false <1> +xpack.monitoring.history.duration: 3d <2> +--------------------------------------------------- +<1> Disable the collection of data on the monitoring cluster. +<2> Lower the default history duration from `7d` to `3d`. The minimum value is +`1d`. This setting can be modified only when using a Gold or higher level +license. For the Basic license level, it uses the default of 7 days. diff --git a/docs/reference/monitoring/pause-export.asciidoc b/docs/reference/monitoring/pause-export.asciidoc new file mode 100644 index 0000000000000..128e72a463c2d --- /dev/null +++ b/docs/reference/monitoring/pause-export.asciidoc @@ -0,0 +1,36 @@ +[role="xpack"] +[testenv="basic"] +[[pause-export]] +== Pausing Data Collection + +To stop generating {monitoring} data in {es}, disable data collection: + +[source,yaml] +--------------------------------------------------- +xpack.monitoring.collection.enabled: false +--------------------------------------------------- + +When this setting is `false`, {es} monitoring data is not collected and all +monitoring data from other sources such as {kib}, Beats, and Logstash is ignored. + +You can update this setting by using the +{ref}/cluster-update-settings.html[Cluster Update Settings API]. + +If you want to separately disable a specific exporter, you can specify the +`enabled` setting (which defaults to `true`) per exporter. For example: + +[source,yaml] +--------------------------------------------------- +xpack.monitoring.exporters.my_http_exporter: + type: http + host: ["10.1.2.3:9200", "10.1.2.4:9200"] + enabled: false <1> +--------------------------------------------------- +<1> Disable the named exporter. If the same name as an existing exporter is not + used, then this will create a completely new exporter that is completely + ignored. This value can be set dynamically by using cluster settings. + +NOTE: Defining a disabled exporter prevents the default exporter from being + created. + +To re-start data collection, re-enable these settings. \ No newline at end of file