Step28: full text search using django-haystack and elasticsearch

Author: yangxg

Diff for: Pipfile

@@ -1,17 +1,19 @@
 [[source]]
 name = "pypi"
-url = "https://pypi.douban.com/simple"
+url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
 fabric = "*"
 
 [packages]
-django = "*"
+django = "~=2.2"
 markdown = "*"
 gunicorn = "*"
 faker = "*"
 django-pure-pagination = "*"
+elasticsearch = ">=2,<3"
+django-haystack = "*"
 
 [requires]
 python_version = "3.6"

Diff for: Pipfile.lock

@@ -148,6 +148,7 @@ tutorial 分支为项目的主分支，每一篇教程的代码都和历史提
 25. [统计各个分类和标签下的文章数](https://www.zmrenwu.com/courses/hellodjango-blog-tutorial/materials/83/)
 26. [开启 Django 博客的 RSS 功能](https://www.zmrenwu.com/courses/hellodjango-blog-tutorial/materials/84/)
 27. [Django 博客实现简单的全文搜索](https://www.zmrenwu.com/courses/hellodjango-blog-tutorial/materials/85/)
+28. [Django Haystack 全文检索与关键词高亮](https://www.zmrenwu.com/courses/hellodjango-blog-tutorial/materials/86/)
 
 ## 公众号
 <p align="center">

Diff for: README.md

@@ -0,0 +1,17 @@
+from haystack.backends.elasticsearch2_backend import Elasticsearch2SearchBackend, Elasticsearch2SearchEngine
+
+DEFAULT_FIELD_MAPPING = {'type': 'string', "analyzer": "ik_max_word", "search_analyzer": "ik_smart"}
+
+
+class Elasticsearch2IkSearchBackend(Elasticsearch2SearchBackend):
+
+    def __init__(self, *args, **kwargs):
+        self.DEFAULT_SETTINGS['settings']['analysis']['analyzer']['ik_analyzer'] = {
+            "type": "custom",
+            "tokenizer": "ik_max_word",
+        }
+        super(Elasticsearch2IkSearchBackend, self).__init__(*args, **kwargs)
+
+
+class Elasticsearch2IkSearchEngine(Elasticsearch2SearchEngine):
+    backend = Elasticsearch2IkSearchBackend

Diff for: blog/elasticsearch2_ik_backend.py

@@ -0,0 +1,12 @@
+from haystack import indexes
+from .models import Post
+
+
+class PostIndex(indexes.SearchIndex, indexes.Indexable):
+    text = indexes.CharField(document=True, use_template=True)
+
+    def get_model(self):
+        return Post
+
+    def index_queryset(self, using=None):
+        return self.get_model().objects.all()

Diff for: blog/search_indexes.py

@@ -9,5 +9,5 @@
     path('archives/<int:year>/<int:month>/', views.ArchiveView.as_view(), name='archive'),
     path('categories/<int:pk>/', views.CategoryView.as_view(), name='category'),
     path('tags/<int:pk>/', views.TagView.as_view(), name='tag'),
-    path('search/', views.search, name='search'),
+    # path('search/', views.search, name='search'),
 ]

Diff for: blog/urls.py

@@ -0,0 +1,16 @@
+from django.utils.html import strip_tags
+from haystack.utils import Highlighter as HaystackHighlighter
+
+
+class Highlighter(HaystackHighlighter):
+    """
+    自定义关键词高亮器，不截断过短的文本（例如文章标题）
+    """
+
+    def highlight(self, text_block):
+        self.text_block = strip_tags(text_block)
+        highlight_locations = self.find_highlightable_words()
+        start_offset, end_offset = self.find_window(highlight_locations)
+        if len(text_block) < self.max_length:
+            start_offset = 0
+        return self.render_html(highlight_locations, start_offset, end_offset)

Diff for: blog/utils.py

@@ -31,7 +31,7 @@
     'django.contrib.staticfiles',
 
     'pure_pagination',  # 分页
-
+    'haystack',  # 搜索
     'blog.apps.BlogConfig',  # 注册 blog 应用
     'comments.apps.CommentsConfig',  # 注册 comments 应用
 ]
@@ -119,3 +119,17 @@
     'MARGIN_PAGES_DISPLAYED': 2,
     'SHOW_FIRST_PAGE_WHEN_INVALID': True,
 }
+
+# 搜索设置
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'blog.elasticsearch2_ik_backend.Elasticsearch2IkSearchEngine',
+        'URL': '',
+        'INDEX_NAME': 'hellodjango_blog_tutorial',
+    },
+}
+HAYSTACK_SEARCH_RESULTS_PER_PAGE = 10
+HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
+HAYSTACK_CUSTOM_HIGHLIGHTER = 'blog.utils.Highlighter'
+# HAYSTACK_DEFAULT_OPERATOR = 'AND'
+# HAYSTACK_FUZZY_MIN_SIM = 0.1

Diff for: blogproject/settings/common.py

@@ -5,3 +5,6 @@
 DEBUG = True
 
 ALLOWED_HOSTS = ['*']
+
+# 搜索设置
+HAYSTACK_CONNECTIONS['default']['URL'] = 'http://elasticsearch_local:9200/'

Diff for: blogproject/settings/local.py

@@ -5,3 +5,4 @@
 DEBUG = False
 
 ALLOWED_HOSTS = ['hellodjango-blog-tutorial-demo.zmrenwu.com']
+HAYSTACK_CONNECTIONS['default']['URL'] = 'http://hellodjango_blog_tutorial_elasticsearch:9200/'

Diff for: blogproject/settings/production.py

@@ -20,6 +20,7 @@
 
 urlpatterns = [
     path('admin/', admin.site.urls),
+    path('search/', include('haystack.urls')),
     path('', include('blog.urls')),
     path('', include('comments.urls')),
 

Diff for: blogproject/urls.py

@@ -0,0 +1,12 @@
+FROM elasticsearch:2.4.6-alpine
+
+COPY ./compose/production/elasticsearch/elasticsearch-analysis-ik-1.10.6.zip /usr/share/elasticsearch/plugins/
+RUN cd /usr/share/elasticsearch/plugins/ && mkdir ik && unzip elasticsearch-analysis-ik-1.10.6.zip -d ik/
+RUN rm /usr/share/elasticsearch/plugins/elasticsearch-analysis-ik-1.10.6.zip
+
+USER root
+COPY ./compose/production/elasticsearch/elasticsearch.yml /usr/share/elasticsearch/config/
+RUN chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/elasticsearch.yml
+
+USER elasticsearch
+

Diff for: compose/production/elasticsearch/Dockerfile

@@ -0,0 +1,2 @@
+bootstrap.memory_lock: true
+network.host: 0.0.0.0

Diff for: compose/production/elasticsearch/elasticsearch-analysis-ik-1.10.6.zip

@@ -2,6 +2,7 @@ version: '3'
 
 volumes:
   database_local:
+  esdata_local:
 
 services:
   hellodjango_blog_tutorial_local:
@@ -16,4 +17,27 @@ services:
       - .:/app
     ports:
       - "8000:8000"
-    command: /start.sh
+    command: /start.sh
+    depends_on:
+      - elasticsearch_local
+
+  elasticsearch_local:
+    build:
+      context: .
+      dockerfile: ./compose/production/elasticsearch/Dockerfile
+    image: hellodjango_blog_tutorial_elasticsearch_local
+    container_name: hellodjango_blog_tutorial_elasticsearch_local
+    volumes:
+      - esdata_local:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nproc: 65536
+      nofile:
+        soft: 65536
+        hard: 65536

Diff for: compose/production/elasticsearch/elasticsearch-analysis-ik-5.6.16.zip

@@ -3,6 +3,7 @@ version: '3'
 volumes:
   static:
   database:
+  esdata:
 
 services:
   hellodjango_blog_tutorial:
@@ -31,4 +32,25 @@ services:
       - static:/apps/hellodjango_blog_tutorial/static
     ports:
       - "80:80"
-      - "443:443"
+      - "443:443"
+
+  elasticsearch:
+    build:
+      context: .
+      dockerfile: ./compose/production/elasticsearch/Dockerfile
+    image: hellodjango_blog_tutorial_elasticsearch
+    container_name: hellodjango_blog_tutorial_elasticsearch
+    volumes:
+      - esdata:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nproc: 65536
+      nofile:
+        soft: 65536
+        hard: 65536

Diff for: compose/production/elasticsearch/elasticsearch.yml

@@ -46,6 +46,11 @@
       padding-left: 10px;
       white-space: pre;
     }
+
+    /* 搜索关键词高亮 */
+    span.highlighted {
+      color: red;
+    }
   </style>
 
   <!-- js -->
@@ -84,7 +89,7 @@ <h1><a href="{% url 'blog:index' %}"><b>Black</b> &amp; White</a></h1>
         <div id="header-search-box">
           <a id="search-menu" href="#"><span id="search-icon" class="ion-ios-search-strong"></span></a>
           <div id="search-form" class="search-form">
-            <form role="search" method="get" id="searchform" action="{% url 'blog:search' %}">
+            <form role="search" method="get" id="searchform" action="{% url 'haystack_search' %}">
               <input type="search" name="q" placeholder="搜索" required>
               <button type="submit"><span class="ion-ios-search-strong"></span></button>
             </form>

Diff for: local.yml

@@ -0,0 +1,2 @@
+{{ object.title }}
+{{ object.body }}