Merge pull request #1 from meilisearch/config

Fix config and rankingOrder

Author: curquiza

.editorconfig

@@ -0,0 +1,18 @@
+# This file is for unifying the coding style for different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[**.py]
+indent_style = space
+indent_size = 4
+
+[**.html]
+indent_style = space
+indent_size = 4

README.md

@@ -8,6 +8,9 @@ _Will be generalized soon for all documentations_
 
 This project supports Python 3.6+.
 
+Set both environment variables `APPLICATION_ID` and `API_KEY`.
+
+Then, run:
 ```bash
 $ pipenv install
 $ pipenv shell

config/config.json

@@ -6,8 +6,8 @@
   "stop_urls": [],
   "selectors": {
     "lvl0": {
-      "selector": ".sidebar-heading",
-      "global": false,
+      "selector": ".sidebar-heading.open",
+      "global": true,
       "default_value": "Documentation"
     },
     "lvl1": ".theme-default-content h1",
@@ -31,9 +31,6 @@
       "lang"
     ]
   },
-  "conversation_id": [
-    "1067906698"
-  ],
   "scrap_start_urls": true,
-  "nb_hits": 998
+  "nb_hits": 920
 }

config/config_original.json

@@ -0,0 +1,40 @@
+{
+  "index_name": "9tz3lqoi",
+  "start_urls": [
+    "https://docs.meilisearch.com"
+  ],
+  "stop_urls": [],
+  "selectors": {
+    "lvl0": {
+      "selector": "p.sidebar-heading.open",
+      "global": true,
+      "default_value": "Documentation"
+    },
+    "lvl1": ".theme-default-content h1",
+    "lvl2": ".theme-default-content h2",
+    "lvl3": ".theme-default-content h3",
+    "lvl4": ".theme-default-content h4",
+    "lvl5": ".theme-default-content h5",
+    "text": ".theme-default-content p, .theme-default-content li",
+    "lang": {
+      "selector": "/html/@lang",
+      "type": "xpath",
+      "global": true,
+      "default_value": "en-US"
+    }
+  },
+  "selectors_exclude": [
+    ".table-of-contents"
+  ],
+  "strip_chars": " .,;:#",
+  "custom_settings": {
+    "attributesForFaceting": [
+      "lang"
+    ]
+  },
+  "conversation_id": [
+    "1067906698"
+  ],
+  "scrap_start_urls": false,
+  "nb_hits": 898
+}

scraper/src/meilisearch_helper.py

@@ -45,10 +45,10 @@ class MeiliSearchHelper:
     # Go to the end of this file to understand criteria order
     SETTINGS = {
         "rankingOrder": [
-            "_number_of_words",
-            "_sum_of_typos",
-            "_sum_of_words_attribute",
-            "_word_proximity",
+            "_words",
+            "_typo",
+            "_attribute",
+            "_proximity",
             "_exact",
             "page_rank",
             "level",