Merge pull request #2 from meilisearch/distinct

Add url_hash distinct

Author: curquiza

Pipfile

@@ -14,6 +14,7 @@ pytest = "==3.10.0"
 requests-iap = "==0.2.0"
 python-helpscout-v2 = "==1.0.1"
 meilisearch = "*"
+pyhash = "*"
 
 [dev-packages]
 pylint = "==2.3.1"

Pipfile.lock

@@ -4,6 +4,7 @@
 import time
 import meilisearch
 from builtins import range
+import pyhash
 
 def clean_one_field(value):
     if value is None:
@@ -31,13 +32,16 @@ def parse_record(record):
     for k, v in record['hierarchy_radio'].items():
         key = 'hierarchy_radio_' + k
         new_hierarchy_radio = {**{key: v}, **new_hierarchy_radio}
+    hasher = pyhash.fnv1_32()
+    new_record_url_hash = {}
+    new_record_url_hash['url_hash'] = hasher(record['url'])
     del record['weight']
     del record['hierarchy']
     del record['hierarchy_radio']
     del record['hierarchy_camel']
     del record['hierarchy_radio_camel']
     del record['content_camel']
-    return {**record, **new_weight, **new_hierarchy, **new_hierarchy_radio}
+    return {**record, **new_weight, **new_hierarchy, **new_hierarchy_radio, **new_record_url_hash}
 
 class MeiliSearchHelper:
     """MeiliSearchHelper"""
@@ -54,7 +58,7 @@ class MeiliSearchHelper:
             "level",
             "position"
         ],
-        "distinctField": "url",
+        "distinctField": "url_hash",
         "rankingRules": {
             "page_rank": "dsc",
             "level": "dsc",
@@ -96,6 +100,7 @@ def update_schema_based_on(self, record):
         base_schema = {
             'anchor':    ['displayed'],
             'url':       ['displayed'],
+            'url_hash':  ['displayed', 'ranked'],
             'content':   ['indexed', 'displayed'],
             'objectID':  ['identifier', 'indexed', 'displayed'],
             'page_rank': ['indexed', 'ranked'],