support composable templates (#1943)

* support composable templates

* do not error on aiohttp warning

* rename new index template to composable index template

* remove old comment

Author: miguelgrinberg

Diff for: .gitignore

@@ -17,3 +17,5 @@ venv
 
 # sample code for GitHub issues
 issues
+.direnv
+.envrc

Diff for: elasticsearch_dsl/__init__.py

@@ -79,7 +79,14 @@
     construct_field,
 )
 from .function import SF
-from .index import AsyncIndex, AsyncIndexTemplate, Index, IndexTemplate
+from .index import (
+    AsyncComposableIndexTemplate,
+    AsyncIndex,
+    AsyncIndexTemplate,
+    ComposableIndexTemplate,
+    Index,
+    IndexTemplate,
+)
 from .mapping import AsyncMapping, Mapping
 from .query import Q, Query
 from .response import AggResponse, Response, UpdateByQueryResponse
@@ -102,6 +109,7 @@
     "A",
     "Agg",
     "AggResponse",
+    "AsyncComposableIndexTemplate",
     "AsyncDocument",
     "AsyncEmptySearch",
     "AsyncFacetedSearch",
@@ -117,6 +125,7 @@
     "Boolean",
     "Byte",
     "Completion",
+    "ComposableIndexTemplate",
     "ConstantKeyword",
     "CustomField",
     "Date",

Diff for: elasticsearch_dsl/_async/index.py

@@ -73,6 +73,47 @@ async def save(
         )
 
 
+class AsyncComposableIndexTemplate:
+    def __init__(
+        self,
+        name: str,
+        template: str,
+        index: Optional["AsyncIndex"] = None,
+        priority: Optional[int] = None,
+        **kwargs: Any,
+    ):
+        if index is None:
+            self._index = AsyncIndex(template, **kwargs)
+        else:
+            if kwargs:
+                raise ValueError(
+                    "You cannot specify options for Index when"
+                    " passing an Index instance."
+                )
+            self._index = index.clone()
+            self._index._name = template
+        self._template_name = name
+        self.priority = priority
+
+    def __getattr__(self, attr_name: str) -> Any:
+        return getattr(self._index, attr_name)
+
+    def to_dict(self) -> Dict[str, Any]:
+        d: Dict[str, Any] = {"template": self._index.to_dict()}
+        d["index_patterns"] = [self._index._name]
+        if self.priority is not None:
+            d["priority"] = self.priority
+        return d
+
+    async def save(
+        self, using: Optional[AsyncUsingType] = None
+    ) -> "ObjectApiResponse[Any]":
+        es = get_connection(using or self._index._using)
+        return await es.indices.put_index_template(
+            name=self._template_name, **self.to_dict()
+        )
+
+
 class AsyncIndex(IndexBase):
     _using: AsyncUsingType
 
@@ -102,13 +143,20 @@ def as_template(
         pattern: Optional[str] = None,
         order: Optional[int] = None,
     ) -> AsyncIndexTemplate:
-        # TODO: should we allow pattern to be a top-level arg?
-        # or maybe have an IndexPattern that allows for it and have
-        # Document._index be that?
         return AsyncIndexTemplate(
             template_name, pattern or self._name, index=self, order=order
         )
 
+    def as_composable_template(
+        self,
+        template_name: str,
+        pattern: Optional[str] = None,
+        priority: Optional[int] = None,
+    ) -> AsyncComposableIndexTemplate:
+        return AsyncComposableIndexTemplate(
+            template_name, pattern or self._name, index=self, priority=priority
+        )
+
     async def load_mappings(self, using: Optional[AsyncUsingType] = None) -> None:
         await self.get_or_create_mapping().update_from_es(
             self._name, using=using or self._using

Diff for: elasticsearch_dsl/_sync/index.py

@@ -69,6 +69,43 @@ def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]":
         return es.indices.put_template(name=self._template_name, body=self.to_dict())
 
 
+class ComposableIndexTemplate:
+    def __init__(
+        self,
+        name: str,
+        template: str,
+        index: Optional["Index"] = None,
+        priority: Optional[int] = None,
+        **kwargs: Any,
+    ):
+        if index is None:
+            self._index = Index(template, **kwargs)
+        else:
+            if kwargs:
+                raise ValueError(
+                    "You cannot specify options for Index when"
+                    " passing an Index instance."
+                )
+            self._index = index.clone()
+            self._index._name = template
+        self._template_name = name
+        self.priority = priority
+
+    def __getattr__(self, attr_name: str) -> Any:
+        return getattr(self._index, attr_name)
+
+    def to_dict(self) -> Dict[str, Any]:
+        d: Dict[str, Any] = {"template": self._index.to_dict()}
+        d["index_patterns"] = [self._index._name]
+        if self.priority is not None:
+            d["priority"] = self.priority
+        return d
+
+    def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]":
+        es = get_connection(using or self._index._using)
+        return es.indices.put_index_template(name=self._template_name, **self.to_dict())
+
+
 class Index(IndexBase):
     _using: UsingType
 
@@ -96,13 +133,20 @@ def as_template(
         pattern: Optional[str] = None,
         order: Optional[int] = None,
     ) -> IndexTemplate:
-        # TODO: should we allow pattern to be a top-level arg?
-        # or maybe have an IndexPattern that allows for it and have
-        # Document._index be that?
         return IndexTemplate(
             template_name, pattern or self._name, index=self, order=order
         )
 
+    def as_composable_template(
+        self,
+        template_name: str,
+        pattern: Optional[str] = None,
+        priority: Optional[int] = None,
+    ) -> ComposableIndexTemplate:
+        return ComposableIndexTemplate(
+            template_name, pattern or self._name, index=self, priority=priority
+        )
+
     def load_mappings(self, using: Optional[UsingType] = None) -> None:
         self.get_or_create_mapping().update_from_es(
             self._name, using=using or self._using

Diff for: elasticsearch_dsl/index.py

@@ -15,5 +15,9 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-from ._async.index import AsyncIndex, AsyncIndexTemplate  # noqa: F401
-from ._sync.index import Index, IndexTemplate  # noqa: F401
+from ._async.index import (  # noqa: F401
+    AsyncComposableIndexTemplate,
+    AsyncIndex,
+    AsyncIndexTemplate,
+)
+from ._sync.index import ComposableIndexTemplate, Index, IndexTemplate  # noqa: F401

Diff for: examples/alias_migration.py

@@ -44,6 +44,7 @@
 
 ALIAS = "test-blog"
 PATTERN = ALIAS + "-*"
+PRIORITY = 100
 
 
 class BlogPost(Document):
@@ -81,7 +82,9 @@ def setup() -> None:
     deploy.
     """
     # create an index template
-    index_template = BlogPost._index.as_template(ALIAS, PATTERN)
+    index_template = BlogPost._index.as_composable_template(
+        ALIAS, PATTERN, priority=PRIORITY
+    )
     # upload the template into elasticsearch
     # potentially overriding the one already there
     index_template.save()

Diff for: examples/async/alias_migration.py

@@ -45,6 +45,7 @@
 
 ALIAS = "test-blog"
 PATTERN = ALIAS + "-*"
+PRIORITY = 100
 
 
 class BlogPost(AsyncDocument):
@@ -82,7 +83,9 @@ async def setup() -> None:
     deploy.
     """
     # create an index template
-    index_template = BlogPost._index.as_template(ALIAS, PATTERN)
+    index_template = BlogPost._index.as_composable_template(
+        ALIAS, PATTERN, priority=PRIORITY
+    )
     # upload the template into elasticsearch
     # potentially overriding the one already there
     await index_template.save()

Diff for: examples/async/parent_child.py

@@ -226,7 +226,7 @@ async def save(self, **kwargs: Any) -> None:  # type: ignore[override]
 
 async def setup() -> None:
     """Create an IndexTemplate and save it into elasticsearch."""
-    index_template = Post._index.as_template("base")
+    index_template = Post._index.as_composable_template("base", priority=100)
     await index_template.save()
 
 

Diff for: examples/parent_child.py

@@ -225,7 +225,7 @@ def save(self, **kwargs: Any) -> None:  # type: ignore[override]
 
 def setup() -> None:
     """Create an IndexTemplate and save it into elasticsearch."""
-    index_template = Post._index.as_template("base")
+    index_template = Post._index.as_composable_template("base", priority=100)
     index_template.save()
 
 

Diff for: setup.cfg

@@ -11,5 +11,6 @@ filterwarnings =
     error
     ignore:Legacy index templates are deprecated in favor of composable templates.:elasticsearch.exceptions.ElasticsearchWarning
     ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version..*:DeprecationWarning
+    default:enable_cleanup_closed ignored.*:DeprecationWarning
 markers =
     sync: mark a test as performing I/O without asyncio.

Diff for: tests/conftest.py

@@ -122,6 +122,7 @@ def teardown_method(self, _: Any) -> None:
         )
         self.client.indices.delete(index="*", expand_wildcards=expand_wildcards)
         self.client.indices.delete_template(name="*")
+        self.client.indices.delete_index_template(name="*")
 
     def es_version(self) -> Tuple[int, ...]:
         if not hasattr(self, "_es_version"):
@@ -172,6 +173,9 @@ def write_client(client: Elasticsearch) -> Generator[Elasticsearch, None, None]:
     for index_name in client.indices.get(index="test-*", expand_wildcards="all"):
         client.indices.delete(index=index_name)
     client.options(ignore_status=404).indices.delete_template(name="test-template")
+    client.options(ignore_status=404).indices.delete_index_template(
+        name="test-template"
+    )
 
 
 @pytest_asyncio.fixture

Diff for: tests/test_integration/_async/test_index.py

@@ -19,6 +19,7 @@
 from elasticsearch import AsyncElasticsearch
 
 from elasticsearch_dsl import (
+    AsyncComposableIndexTemplate,
     AsyncDocument,
     AsyncIndex,
     AsyncIndexTemplate,
@@ -35,7 +36,31 @@ class Post(AsyncDocument):
 
 @pytest.mark.asyncio
 async def test_index_template_works(async_write_client: AsyncElasticsearch) -> None:
-    it = AsyncIndexTemplate("test-template", "test-*")
+    it = AsyncIndexTemplate("test-template", "test-legacy-*")
+    it.document(Post)
+    it.settings(number_of_replicas=0, number_of_shards=1)
+    await it.save()
+
+    i = AsyncIndex("test-legacy-blog")
+    await i.create()
+
+    assert {
+        "test-legacy-blog": {
+            "mappings": {
+                "properties": {
+                    "title": {"type": "text", "analyzer": "my_analyzer"},
+                    "published_from": {"type": "date"},
+                }
+            }
+        }
+    } == await async_write_client.indices.get_mapping(index="test-legacy-blog")
+
+
+@pytest.mark.asyncio
+async def test_composable_index_template_works(
+    async_write_client: AsyncElasticsearch,
+) -> None:
+    it = AsyncComposableIndexTemplate("test-template", "test-*")
     it.document(Post)
     it.settings(number_of_replicas=0, number_of_shards=1)
     await it.save()

Diff for: tests/test_integration/_sync/test_index.py

@@ -18,7 +18,15 @@
 import pytest
 from elasticsearch import Elasticsearch
 
-from elasticsearch_dsl import Date, Document, Index, IndexTemplate, Text, analysis
+from elasticsearch_dsl import (
+    ComposableIndexTemplate,
+    Date,
+    Document,
+    Index,
+    IndexTemplate,
+    Text,
+    analysis,
+)
 
 
 class Post(Document):
@@ -28,7 +36,31 @@ class Post(Document):
 
 @pytest.mark.sync
 def test_index_template_works(write_client: Elasticsearch) -> None:
-    it = IndexTemplate("test-template", "test-*")
+    it = IndexTemplate("test-template", "test-legacy-*")
+    it.document(Post)
+    it.settings(number_of_replicas=0, number_of_shards=1)
+    it.save()
+
+    i = Index("test-legacy-blog")
+    i.create()
+
+    assert {
+        "test-legacy-blog": {
+            "mappings": {
+                "properties": {
+                    "title": {"type": "text", "analyzer": "my_analyzer"},
+                    "published_from": {"type": "date"},
+                }
+            }
+        }
+    } == write_client.indices.get_mapping(index="test-legacy-blog")
+
+
+@pytest.mark.sync
+def test_composable_index_template_works(
+    write_client: Elasticsearch,
+) -> None:
+    it = ComposableIndexTemplate("test-template", "test-*")
     it.document(Post)
     it.settings(number_of_replicas=0, number_of_shards=1)
     it.save()

Diff for: tests/test_integration/test_examples/_async/test_alias_migration.py

@@ -28,7 +28,7 @@ async def test_alias_migration(async_write_client: AsyncElasticsearch) -> None:
     await alias_migration.setup()
 
     # verify that template, index, and alias has been set up
-    assert await async_write_client.indices.exists_template(name=ALIAS)
+    assert await async_write_client.indices.exists_index_template(name=ALIAS)
     assert await async_write_client.indices.exists(index=PATTERN)
     assert await async_write_client.indices.exists_alias(name=ALIAS)
 

Diff for: tests/test_integration/test_examples/_async/test_parent_child.py

@@ -45,7 +45,7 @@
 @pytest_asyncio.fixture
 async def question(async_write_client: AsyncElasticsearch) -> Question:
     await setup()
-    assert await async_write_client.indices.exists_template(name="base")
+    assert await async_write_client.indices.exists_index_template(name="base")
 
     # create a question object
     q = Question(

Diff for: tests/test_integration/test_examples/_sync/test_alias_migration.py

@@ -28,7 +28,7 @@ def test_alias_migration(write_client: Elasticsearch) -> None:
     alias_migration.setup()
 
     # verify that template, index, and alias has been set up
-    assert write_client.indices.exists_template(name=ALIAS)
+    assert write_client.indices.exists_index_template(name=ALIAS)
     assert write_client.indices.exists(index=PATTERN)
     assert write_client.indices.exists_alias(name=ALIAS)
 

Diff for: tests/test_integration/test_examples/_sync/test_parent_child.py

@@ -44,7 +44,7 @@
 @pytest.fixture
 def question(write_client: Elasticsearch) -> Question:
     setup()
-    assert write_client.indices.exists_template(name="base")
+    assert write_client.indices.exists_index_template(name="base")
 
     # create a question object
     q = Question(