Introduce stability description to the REST API specification (#38413)

* introduce state to the REST API specification

* change state over to stability

* CCR is no GA updated to stable

* SQL is now GA so marked as stable

* Introduce `internal` as state for API's, marks stable in terms of lifetime but unstable in terms of guarantees on its output format since it exposes internal representations

* make setting a wrong stability value, or not setting it at all an error that causes the YAML test suite to fail

* update spec files to be explicit about their stability state

* Document the fact that stability needs to be defined

Otherwise the YAML test runner will fail (with a nice exception message)

* address check style violations

* update rest spec unit tests to include stability

* found one more test spec file not declaring stability, made sure stability appears after documentation everywhere

* cluster.state is stable, mark response in some way to denote its a key value format that can be changed during minors

* mark data frame API's as beta

* remove internal and private as states for an API

* removed the wrong enum values in the Stability Enum in the previous commit

Author: Mpdreamz

modules/lang-painless/src/test/resources/rest-api-spec/api/scripts_painless_context.json

@@ -1,5 +1,6 @@
 {
   "scripts_painless_context": {
+    "stability": "experimental",
     "methods": ["GET"],
     "url": {
       "paths": ["/_scripts/painless/_context"],

plugins/examples/rest-handler/src/test/resources/rest-api-spec/api/cat.example.json

@@ -1,6 +1,7 @@
 {
   "cat.example": {
     "documentation": "",
+    "stability" : "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/example"],

rest-api-spec/README.markdown

@@ -1,6 +1,7 @@
 {
   "bulk": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html",
+    "stability": "stable",
     "methods": ["POST", "PUT"],
     "url": {
       "paths": ["/_bulk", "/{index}/_bulk", "/{index}/{type}/_bulk"],

rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json

@@ -1,6 +1,7 @@
 {
   "cat.aliases": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-alias.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/aliases", "/_cat/aliases/{name}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json

@@ -1,6 +1,7 @@
 {
   "cat.allocation": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-allocation.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/allocation", "/_cat/allocation/{node_id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json

@@ -1,6 +1,7 @@
 {
   "cat.count": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-count.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/count", "/_cat/count/{index}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json

@@ -1,6 +1,7 @@
 {
   "cat.fielddata": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-fielddata.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/fielddata", "/_cat/fielddata/{fields}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json

@@ -1,6 +1,7 @@
 {
   "cat.health": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-health.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/health"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json

@@ -1,6 +1,7 @@
 {
   "cat.help": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json

@@ -1,6 +1,7 @@
 {
   "cat.indices": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-indices.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/indices", "/_cat/indices/{index}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json

@@ -1,6 +1,7 @@
 {
   "cat.master": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-master.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/master"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json

@@ -1,6 +1,7 @@
 {
   "cat.nodeattrs": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodeattrs.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/nodeattrs"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json

@@ -1,6 +1,7 @@
 {
   "cat.nodes": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodes.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/nodes"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json

@@ -1,6 +1,7 @@
 {
   "cat.pending_tasks": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-pending-tasks.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/pending_tasks"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json

@@ -1,6 +1,7 @@
 {
   "cat.plugins": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-plugins.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/plugins"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json

@@ -1,6 +1,7 @@
 {
   "cat.recovery": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-recovery.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/recovery", "/_cat/recovery/{index}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json

@@ -1,6 +1,7 @@
 {
   "cat.repositories": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-repositories.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/repositories"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json

@@ -1,6 +1,7 @@
 {
   "cat.segments": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segments.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/segments", "/_cat/segments/{index}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json

@@ -1,6 +1,7 @@
 {
   "cat.shards": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-shards.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/shards", "/_cat/shards/{index}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json

@@ -1,6 +1,7 @@
 {
   "cat.snapshots": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-snapshots.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": [

rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json

@@ -1,6 +1,7 @@
 {
   "cat.tasks": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/tasks"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json

@@ -1,6 +1,7 @@
 {
   "cat.templates": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-templates.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/templates", "/_cat/templates/{name}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json

@@ -1,6 +1,7 @@
 {
   "cat.thread_pool": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-thread-pool.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cat/thread_pool","/_cat/thread_pool/{thread_pool_patterns}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json

@@ -1,6 +1,7 @@
 {
   "clear_scroll": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
       "paths": [ "/_search/scroll"],

rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json

@@ -1,6 +1,7 @@
 {
   "cluster.allocation_explain": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-allocation-explain.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
       "paths": ["/_cluster/allocation/explain"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json

@@ -1,6 +1,7 @@
 {
   "cluster.get_settings": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cluster/settings"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json

@@ -1,6 +1,7 @@
 {
   "cluster.health": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-health.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cluster/health", "/_cluster/health/{index}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json

@@ -1,6 +1,7 @@
 {
   "cluster.pending_tasks": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-pending.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cluster/pending_tasks"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json

@@ -1,6 +1,7 @@
 {
   "cluster.put_settings": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+    "stability": "stable",
     "methods": ["PUT"],
     "url": {
       "paths": ["/_cluster/settings"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json

@@ -1,6 +1,7 @@
 {
   "cluster.remote_info": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-remote-info.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_remote/info"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json

@@ -1,6 +1,7 @@
 {
   "cluster.reroute": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-reroute.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
       "paths": ["/_cluster/reroute"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json

@@ -1,6 +1,7 @@
 {
   "cluster.state": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-state.html",
+    "stability" : "stable",
     "methods": ["GET"],
     "url": {
       "paths": [
@@ -56,6 +57,9 @@
         }
       }
     },
-    "body": null
+    "body": null,
+    "response": {
+      "treat_json_as_key_value" : true
+    }
   }
 }

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json

@@ -1,6 +1,7 @@
 {
   "cluster.stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-stats.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
       "paths": ["/_cluster/stats", "/_cluster/stats/nodes/{node_id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json

@@ -1,6 +1,7 @@
 {
   "count": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-count.html",
+    "stability": "stable",
     "methods": ["POST", "GET"],
     "url": {
       "paths": ["/_count", "/{index}/_count"],

rest-api-spec/src/main/resources/rest-api-spec/api/count.json

@@ -1,6 +1,7 @@
 {
   "create": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-index_.html",
+    "stability": "stable",
     "methods": ["PUT","POST"],
     "url": {
       "paths": ["/{index}/_create/{id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/create.json

@@ -1,6 +1,7 @@
 {
   "delete": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
       "paths": ["/{index}/_doc/{id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/delete.json

@@ -1,6 +1,7 @@
 {
   "delete_by_query": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
       "paths": ["/{index}/_delete_by_query"],

rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json

@@ -1,6 +1,7 @@
 {
   "delete_by_query_rethrottle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
       "paths": ["/_delete_by_query/{task_id}/_rethrottle"],

rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json

@@ -1,6 +1,7 @@
 {
   "delete_script": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
       "paths": [ "/_scripts/{id}" ],

rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json

@@ -1,6 +1,7 @@
 {
   "exists": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
       "paths": ["/{index}/_doc/{id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/exists.json

@@ -1,6 +1,7 @@
 {
   "exists_source": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
       "paths": ["/{index}/_source/{id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json

@@ -1,6 +1,7 @@
 {
   "explain": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-explain.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
       "paths": ["/{index}/_explain/{id}"],

rest-api-spec/src/main/resources/rest-api-spec/api/explain.json

@@ -1,6 +1,7 @@
 {
   "field_caps": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-field-caps.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
       "paths": [