From a11d84da2a37c5448b98b0067dc6d7f7c6b85db4 Mon Sep 17 00:00:00 2001
From: =?utf8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@0d.be>
Date: Mon, 25 Jul 2022 16:41:33 +0200
Subject: [PATCH] revamp full text search to use postgresql

---
 debian/control                       |   1 -
 panikdb/settings.py                  |   9 +--
 panikdb/static/css/style.scss        |  18 +++++
 panikdb/templates/base.html          |   2 +-
 panikdb/templates/search/search.html | 108 +++++++++++----------------
 panikdb/urls.py                      |  12 +--
 panikdb/views.py                     |  32 ++++++++
 requirements.txt                     |   1 -
 8 files changed, 95 insertions(+), 88 deletions(-)

diff --git a/debian/control b/debian/control
index db6582c..6b750c0 100644
--- a/debian/control
+++ b/debian/control
@@ -13,7 +13,6 @@ Standards-Version: 3.9.1
 Package: python3-panikdb
 Architecture: all
 Depends: python3-django-ckeditor,
-         python3-django-haystack,
          python3-django-panik-agendas,
          python3-django-panik-combo,
          python3-django-panik-emissions,
diff --git a/panikdb/settings.py b/panikdb/settings.py
index 0ef539d..f8111e7 100644
--- a/panikdb/settings.py
+++ b/panikdb/settings.py
@@ -144,7 +144,6 @@ INSTALLED_APPS = (
     'django.contrib.admin',
     'registration',
     'ckeditor',
-    'haystack',
     'taggit',
     'emissions',
     'newsletter',
@@ -227,13 +226,6 @@ CKEDITOR_CONFIGS = {
 CKEDITOR_CONFIGS['small'] = copy.copy(CKEDITOR_CONFIGS['default'])
 CKEDITOR_CONFIGS['small']['height'] = 150
 
-HAYSTACK_CONNECTIONS = {
-    'default': {
-        'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
-        'URL': 'http://127.0.0.1:8985/solr/panik',
-    },
-}
-
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
@@ -312,6 +304,7 @@ COMBO_CELL_ASSET_SLOTS = {}
 COMBO_CELL_TEMPLATES = {}
 WIKI_SEARCH_BASE_URL = None
 CELL_CONDITIONS_ENABLED = False
+FTS_DICTIONARY_CONFIG = 'french'
 
 local_settings_file = os.environ.get(
     'PANIKDB_SETTINGS_FILE', os.path.join(os.path.dirname(__file__), 'local_settings.py')
diff --git a/panikdb/static/css/style.scss b/panikdb/static/css/style.scss
index e8e8804..b3ca1e1 100644
--- a/panikdb/static/css/style.scss
+++ b/panikdb/static/css/style.scss
@@ -989,3 +989,21 @@ div.section > div > *:first-child {
 a.button {
 	display: inline;
 }
+
+.search--form {
+	display: flex;
+	column-gap: 1em;
+	input {
+		flex: 1;
+		margin: 0;
+	}
+}
+
+.search--hits {
+	display: flex;
+	column-gap: 1em;
+	.section {
+		flex: 1;
+		margin: 0;
+	}
+}
diff --git a/panikdb/templates/base.html b/panikdb/templates/base.html
index c28e24e..1771621 100644
--- a/panikdb/templates/base.html
+++ b/panikdb/templates/base.html
@@ -30,7 +30,7 @@
     {% block subheader %}
     <div id="search">
       {% if request.user.id %}
-      <form class="gadjo-popup-ignore" action="{% url 'haystack_search' %}"
+      <form class="gadjo-popup-ignore" action="{% url 'panikdb-search' %}"
             data-wiki-search-base-url="{{ wiki_search_base_url }}">
         <input type="search" name="q" placeholder="Rechercher"/>
         <button class="icon-search"></button>
diff --git a/panikdb/templates/search/search.html b/panikdb/templates/search/search.html
index 2e086e9..84975b0 100644
--- a/panikdb/templates/search/search.html
+++ b/panikdb/templates/search/search.html
@@ -1,77 +1,53 @@
 {% extends 'base.html' %}
+{% load i18n %}
 
 {% block content %}
-    <h2>Recherche</h2>
+    <h2>{% trans 'Search '%}</h2>
 
-    <form method="get" action=".">
-        <table>
-            {{ form.as_table }}
-            <tr>
-                <td>&nbsp;</td>
-                <td>
-                  <button>Rechercher</button>
-                </td>
-            </tr>
-        </table>
-
-        {% if query %}
-    <!-- Begin faceting. -->
-
-<p>
-    {% if facets.fields.categories %}
-    Catégories :
-      {# Provide only the top 5 categories #}
-        {% for category in facets.fields.categories|slice:":5" %}
-          {% if category.0 in selected_categories %}
-            <strong>{{ category.0 }}</strong> ({{ category.1 }})
-          {% else %}
-            <span><a href="{{ request.get_full_path }}&amp;selected_facets=categories_exact:{{ category.0|urlencode }}">{{ category.0 }}</a> ({{ category.1 }})</span>
-          {% endif %}
-        {% endfor %}
-    {% else %}
-        No categories facets.
+    <div class="bo-block">
+    <form class="search--form" method="get" action=".">
+      <input type="search" name="q" value="{{ query }}">
+      <button>{% trans "Search" %}</button>
+    </form>
+    </div>
+
+    {% if hits %}
+    <div class="search--hits">
+
+    {% if hits.episodes %}
+    <div class="section">
+    <h3>{% trans "Episodes" %}</h3>
+    <ul>
+    {% for hit in hits.episodes %}
+      <li><a href="{{ hit.get_absolute_url }}">{{ hit.emission.title }} - {{ hit.title }}</a></li>
+    {% endfor %}
+    </ul>
+    </div>
     {% endif %}
-</p>
 
-<p>
-    {% if facets.fields.tags %}
-    Tags :
-      {# Provide only the top 5 tags #}
-        {% for tag in facets.fields.tags|slice:":5" %}
-          {% if tag.0 in selected_tags %}
-            <strong>{{ tag.0 }}</strong> ({{ tag.1 }})
-          {% else %}
-            <span><a href="{{ request.get_full_path }}&amp;selected_facets=tags_exact:{{ tag.0|urlencode }}">{{ tag.0 }}</a> ({{ tag.1 }})</span>
-          {% endif %}
-        {% endfor %}
-    {% else %}
-        No tags facets.
+    {% if hits.emissions %}
+    <div class="section">
+    <h3>{% trans "Emissions" %}</h3>
+    <ul>
+    {% for hit in hits.emissions %}
+      <li><a href="{{ hit.get_absolute_url }}">{{ hit.title }}</a></li>
+    {% endfor %}
+    </ul>
+    </div>
     {% endif %}
-</p>
-
-
-    <!-- End faceting -->
-
-            <h3>Résultats</h3>
 
+    {% if hits.newitems %}
+    <div class="section">
+    <h3>{% trans "Newsitems" %}</h3>
+    <ul>
+    {% for hit in hits.newsitems %}
+      <li><a href="{{ hit.get_absolute_url }}">{{ hit.title }}</a></li>
+    {% endfor %}
+    </ul>
+    </div>
+    {% endif %}
 
-            {% for result in page.object_list %}
-                <p>
-                    <a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a>
-                </p>
-            {% empty %}
-                <p>Aucun résultat trouvé.</p>
-            {% endfor %}
+    </div>
+    {% endif %}
 
-            {% if page.has_previous or page.has_next %}
-                <div>
-                    {% if page.has_previous %}<a href="?q={{ query }}&amp;page={{ page.previous_page_number }}">{% endif %}&laquo; Précédente{% if page.has_previous %}</a>{% endif %}
-                    |
-                    {% if page.has_next %}<a href="?q={{ query }}&amp;page={{ page.next_page_number }}">{% endif %}Suvante &raquo;{% if page.has_next %}</a>{% endif %}
-                </div>
-            {% endif %}
-        {% else %}
-            {# Show some example queries to run, maybe query syntax, something else? #}
-        {% endif %}
-    </form>
 {% endblock %}
diff --git a/panikdb/urls.py b/panikdb/urls.py
index 71e11c5..7079ef8 100644
--- a/panikdb/urls.py
+++ b/panikdb/urls.py
@@ -13,10 +13,6 @@ from django.urls import path, reverse_lazy
 from django.views.generic import RedirectView
 from emissions.models import Emission, Episode, NewsItem
 from emissions.urls import urlpatterns as emissions_urlpatterns
-from emissions.views import FacetedSearchView
-from haystack.forms import FacetedSearchForm
-from haystack.query import SearchQuerySet
-from haystack.views import search_view_factory
 from matos.urls import urlpatterns as matos_urlpatterns
 from newsletter.urls import management_patterns as newsletter_urlpatterns
 from nonstop.urls import public_urlpatterns as nonstop_public_urlpatterns
@@ -34,17 +30,11 @@ from .regie.urls import urlpatterns as regie_urlpatterns
 from .service_messages.urls import urlpatterns as service_messages_urlpatterns
 from .urls_utils import cms_permission_required, decorated_includes
 
-sqs = SearchQuerySet().models(Emission, Episode, NewsItem).facet('categories').facet('tags')
-
 urlpatterns = [
     url(r'^$', views.home, name='home'),
     url(r'^ckeditor/upload/', login_required(ckeditor_upload)),
     url(r'^ckeditor/browse/', login_required(ckeditor_browse)),
-    url(
-        r'^search/',
-        search_view_factory(view_class=FacetedSearchView, form_class=FacetedSearchForm, searchqueryset=sqs),
-        name='haystack_search',
-    ),
+    url(r'^search/', login_required(views.search), name='panikdb-search'),
     url(r'^emissions/$', views.emissions_list, name='emission-list'),
     url(r'^emissions/', decorated_includes(login_required, include(emissions_urlpatterns))),
     url(r'^matos/', decorated_includes(login_required, include(matos_urlpatterns))),
diff --git a/panikdb/views.py b/panikdb/views.py
index eb2ed8f..bb13593 100644
--- a/panikdb/views.py
+++ b/panikdb/views.py
@@ -4,8 +4,10 @@ import logging
 import emissions.views
 import newsletter.views
 from combo.data.models import Page
+from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
+from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
 from django.core.exceptions import PermissionDenied
 from django.shortcuts import redirect
 from django.template import loader
@@ -57,6 +59,36 @@ class Home(TemplateView):
 home = login_required(Home.as_view())
 
 
+class Search(TemplateView):
+    template_name = 'search/search.html'
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context['query'] = self.request.GET.get('q', '')
+        vector = (
+            SearchVector('title', config=settings.FTS_DICTIONARY_CONFIG, weight='A')
+            + SearchVector('subtitle', config=settings.FTS_DICTIONARY_CONFIG, weight='A')
+            + SearchVector('text', config=settings.FTS_DICTIONARY_CONFIG, weight='B')
+        )
+        query = SearchQuery(self.request.GET.get('q', ''), config=settings.FTS_DICTIONARY_CONFIG)
+        context['hits'] = {
+            'episodes': Episode.objects.annotate(rank=SearchRank(vector, query))
+            .filter(rank__gte=0.1)
+            .select_related('emission')
+            .order_by('-rank')[:10],
+            'emissions': Emission.objects.annotate(rank=SearchRank(vector, query))
+            .filter(rank__gte=0.1)
+            .order_by('-rank')[:10],
+            'newsitems': NewsItem.objects.annotate(rank=SearchRank(vector, query))
+            .filter(rank__gte=0.1)
+            .order_by('-rank')[:10],
+        }
+        return context
+
+
+search = Search.as_view()
+
+
 class EmissionListView(emissions.views.EmissionListView):
     pass
 
diff --git a/requirements.txt b/requirements.txt
index 1538005..2d02b75 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,6 @@
 Django==1.7
 django-ckeditor
 django-taggit
-git+https://github.com/toastdriven/django-haystack.git
 django-registration-redux
 django-jquery
 django-datetime-widget
-- 
2.39.2