Release of 8.16.0 (#354)

* add to release

* update formatting

Author: joemcelroy

.github/workflows/tests.yml

@@ -17,9 +17,9 @@ jobs:
       fail-fast: false
       matrix:
         es_stack:
-          - 8.14.2
-          - 8.15.0
-          - 8.16.0-SNAPSHOT
+          - 8.15.3
+          - 8.16.0
+          - 8.17.0-SNAPSHOT
     runs-on: ubuntu-latest
     services:
       elasticsearch:

supporting-blog-content/hybrid-search-for-an-e-commerce-product-catalogue/product-store-search/api/api.py

@@ -11,27 +11,30 @@
 
 
 def get_client_es():
-    with open('../config.yml', 'r') as file:
+    with open("../config.yml", "r") as file:
         config = yaml.safe_load(file)
-    return Elasticsearch(
-        cloud_id=config['cloud_id'],
-        api_key=config['api_key']
-    )
+    return Elasticsearch(cloud_id=config["cloud_id"], api_key=config["api_key"])
 
 
 def get_text_vector(sentences):
-    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+    model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
     embeddings = model.encode(sentences)
     return embeddings
 
 
 def build_query(term=None, categories=None, product_types=None, brands=None):
-    must_query = [{"match_all": {}}] if not term else [{
-        "multi_match": {
-            "query": term,
-            "fields": ["name", "category", "description"]
-        }
-    }]
+    must_query = (
+        [{"match_all": {}}]
+        if not term
+        else [
+            {
+                "multi_match": {
+                    "query": term,
+                    "fields": ["name", "category", "description"],
+                }
+            }
+        ]
+    )
 
     filters = []
     if categories:
@@ -42,17 +45,23 @@ def build_query(term=None, categories=None, product_types=None, brands=None):
         filters.append({"terms": {"brand.keyword": brands}})
 
     return {
-        "_source": ["id", "brand", "name", "price", "currency", "image_link", "category", "tag_list"],
-        "query": {
-            "bool": {
-                "must": must_query,
-                "filter": filters
-            }
-        }
+        "_source": [
+            "id",
+            "brand",
+            "name",
+            "price",
+            "currency",
+            "image_link",
+            "category",
+            "tag_list",
+        ],
+        "query": {"bool": {"must": must_query, "filter": filters}},
     }
 
 
-def build_hybrid_query(term=None, categories=None, product_types=None, brands=None, hybrid=False):
+def build_hybrid_query(
+    term=None, categories=None, product_types=None, brands=None, hybrid=False
+):
     # Standard query
     organic_query = build_query(term, categories, product_types, brands)
 
@@ -65,81 +74,79 @@ def build_hybrid_query(term=None, categories=None, product_types=None, brands=No
             "retriever": {
                 "rrf": {
                     "retrievers": [
-                        {
-                            "standard": {
-                                "query": organic_query['query']
-                            }
-                        },
+                        {"standard": {"query": organic_query["query"]}},
                         {
                             "knn": {
                                 "field": "description_embeddings",
                                 "query_vector": vector,
                                 "k": 5,
                                 "num_candidates": 20,
-                                "filter": {
-                                    "bool": {
-                                        "filter": []
-                                    }
-                                }
+                                "filter": {"bool": {"filter": []}},
                             }
-                        }
+                        },
                     ],
                     "rank_window_size": 20,
-                    "rank_constant": 5
+                    "rank_constant": 5,
                 }
             },
-            "_source": organic_query['_source']
+            "_source": organic_query["_source"],
         }
 
         if categories:
-            query['retriever']['rrf']['retrievers'][1]['knn']['filter']['bool']['filter'].append({
-                "terms": {"category": categories}
-            })
+            query["retriever"]["rrf"]["retrievers"][1]["knn"]["filter"]["bool"][
+                "filter"
+            ].append({"terms": {"category": categories}})
         if product_types:
-            query['retriever']['rrf']['retrievers'][1]['knn']['filter']['bool']['filter'].append({
-                "terms": {"product_type": product_types}
-            })
+            query["retriever"]["rrf"]["retrievers"][1]["knn"]["filter"]["bool"][
+                "filter"
+            ].append({"terms": {"product_type": product_types}})
         if brands:
-            query['retriever']['rrf']['retrievers'][1]['knn']['filter']['bool']['filter'].append({
-                "terms": {"brand.keyword": brands}
-            })
+            query["retriever"]["rrf"]["retrievers"][1]["knn"]["filter"]["bool"][
+                "filter"
+            ].append({"terms": {"brand.keyword": brands}})
     else:
         query = organic_query
 
     return query
 
 
-def search_products(term, categories=None, product_types=None, brands=None, promote_products=[], hybrid=False):
+def search_products(
+    term,
+    categories=None,
+    product_types=None,
+    brands=None,
+    promote_products=[],
+    hybrid=False,
+):
     query = build_hybrid_query(term, categories, product_types, brands, hybrid)
 
     if promote_products and not hybrid:
         query = {
-            "query": {
-                "pinned": {
-                    "ids": promote_products,
-                    "organic": query['query']
-                }
-            },
-            "_source": query['_source']
+            "query": {"pinned": {"ids": promote_products, "organic": query["query"]}},
+            "_source": query["_source"],
         }
 
     print(query)
     response = get_client_es().search(index="products-catalog", body=query, size=20)
 
     results = []
-    for hit in response['hits']['hits']:
+    for hit in response["hits"]["hits"]:
         print(f"Product Name: {hit['_source']['name']}, Score: {hit['_score']}")
 
-        results.append({
-            "id": hit['_source']['id'],
-            "brand": hit['_source']['brand'],
-            "name": hit['_source']['name'],
-            "price": hit['_source']['price'],
-            "currency": hit['_source']['currency'] if hit['_source']['currency'] else "USD",
-            "image_link": hit['_source']['image_link'],
-            "category": hit['_source']['category'],
-            "tags": hit['_source'].get('tag_list', [])
-        })
+        results.append(
+            {
+                "id": hit["_source"]["id"],
+                "brand": hit["_source"]["brand"],
+                "name": hit["_source"]["name"],
+                "price": hit["_source"]["price"],
+                "currency": (
+                    hit["_source"]["currency"] if hit["_source"]["currency"] else "USD"
+                ),
+                "image_link": hit["_source"]["image_link"],
+                "category": hit["_source"]["category"],
+                "tags": hit["_source"].get("tag_list", []),
+            }
+        )
 
     return results
 
@@ -149,51 +156,55 @@ def get_facets_data(term, categories=None, product_types=None, brands=None):
     query["aggs"] = {
         "product_types": {"terms": {"field": "product_type"}},
         "categories": {"terms": {"field": "category"}},
-        "brands": {"terms": {"field": "brand.keyword"}}
+        "brands": {"terms": {"field": "brand.keyword"}},
     }
     response = get_client_es().search(index="products-catalog", body=query, size=0)
 
     return {
         "product_types": [
-            {"product_type": bucket['key'], "count": bucket['doc_count']}
-            for bucket in response['aggregations']['product_types']['buckets']
+            {"product_type": bucket["key"], "count": bucket["doc_count"]}
+            for bucket in response["aggregations"]["product_types"]["buckets"]
         ],
         "categories": [
-            {"category": bucket['key'], "count": bucket['doc_count']}
-            for bucket in response['aggregations']['categories']['buckets']
+            {"category": bucket["key"], "count": bucket["doc_count"]}
+            for bucket in response["aggregations"]["categories"]["buckets"]
         ],
         "brands": [
-            {"brand": bucket['key'], "count": bucket['doc_count']}
-            for bucket in response['aggregations']['brands']['buckets']
-        ]
+            {"brand": bucket["key"], "count": bucket["doc_count"]}
+            for bucket in response["aggregations"]["brands"]["buckets"]
+        ],
     }
 
 
-@app.route('/api/products/search', methods=['GET'])
+@app.route("/api/products/search", methods=["GET"])
 def search():
-    query = request.args.get('query')
-    categories = request.args.getlist('selectedCategories[]')
-    product_types = request.args.getlist('selectedProductTypes[]')
-    brands = request.args.getlist('selectedBrands[]')
-    hybrid = request.args.get('hybrid', 'False').lower() == 'true'
-    results = search_products(query, categories=categories, product_types=product_types,
-                              brands=brands,
-                              promote_products=promote_products_free_gluten,
-                              hybrid=hybrid)
+    query = request.args.get("query")
+    categories = request.args.getlist("selectedCategories[]")
+    product_types = request.args.getlist("selectedProductTypes[]")
+    brands = request.args.getlist("selectedBrands[]")
+    hybrid = request.args.get("hybrid", "False").lower() == "true"
+    results = search_products(
+        query,
+        categories=categories,
+        product_types=product_types,
+        brands=brands,
+        promote_products=promote_products_free_gluten,
+        hybrid=hybrid,
+    )
     return jsonify(results)
 
 
-@app.route('/api/products/facets', methods=['GET'])
+@app.route("/api/products/facets", methods=["GET"])
 def facets():
-    query = request.args.get('query')
-    categories = request.args.getlist('selectedCategories[]')
-    product_types = request.args.getlist('selectedProductTypes[]')
-    brands = request.args.getlist('selectedBrands[]')
-    results = get_facets_data(query, categories=categories,
-                              product_types=product_types,
-                              brands=brands)
+    query = request.args.get("query")
+    categories = request.args.getlist("selectedCategories[]")
+    product_types = request.args.getlist("selectedProductTypes[]")
+    brands = request.args.getlist("selectedBrands[]")
+    results = get_facets_data(
+        query, categories=categories, product_types=product_types, brands=brands
+    )
     return jsonify(results)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     app.run(debug=True)

supporting-blog-content/hybrid-search-for-an-e-commerce-product-catalogue/product-store-search/files/dataset/generate_data.py

@@ -1,15 +1,26 @@
 import csv
 import json
 
-desired_fields = ["id", "brand", "name", "price", "price_sign", "currency",
-                  "image_link", "description", "rating", "category",
-                  "product_type", "tag_list"]
+desired_fields = [
+    "id",
+    "brand",
+    "name",
+    "price",
+    "price_sign",
+    "currency",
+    "image_link",
+    "description",
+    "rating",
+    "category",
+    "product_type",
+    "tag_list",
+]
 
 input_file = "dataset_products.csv"  # Replace with your actual filename
 output_file = "products.json"
 
 # Open CSV file
-with open(input_file, 'r') as csvfile:
+with open(input_file, "r") as csvfile:
     # Read CSV data using DictReader
     csv_reader = csv.DictReader(csvfile)
 
@@ -37,8 +48,8 @@
         json_data.append(product_data)
 
 # Open JSON file for writing
-with open(output_file, 'w') as jsonfile:
+with open(output_file, "w") as jsonfile:
     # Write JSON data to file with indentation
     json.dump(json_data, jsonfile, indent=4)
 
-print(f"Converted CSV data to JSON and saved to {output_file}")
+print(f"Converted CSV data to JSON and saved to {output_file}")