Merge remote-tracking branch 'origin/main' into 8.x

Author: pquentin

.github/workflows/backport.yml

@@ -1,16 +1,26 @@
 name: Backport
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
       - labeled
 
 jobs:
   backport:
-    runs-on: ubuntu-latest
     name: Backport
+    runs-on: ubuntu-latest
+    # Only react to merged PRs for security reasons.
+    # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
+    if: >
+      github.event.pull_request.merged
+      && (
+        github.event.action == 'closed'
+        || (
+          github.event.action == 'labeled'
+          && contains(github.event.label.name, 'backport')
+        )
+      )
     steps:
-      - name: Backport
-        uses: tibdex/backport@v2
+      - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2.0.4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python3 -m pip install setuptools wheel twine
@@ -32,7 +32,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python3 -m pip install nox
@@ -47,7 +47,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python3 -m pip install nox
@@ -66,8 +66,9 @@ jobs:
           "3.9",
           "3.10",
           "3.11",
+          "3.12",
         ]
-        es-version: [8.0.0, 8.9.0]
+        es-version: [8.0.0, 8.11.0]
 
     steps:
       - name: Checkout Repository
@@ -82,7 +83,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set up Python for Nox
-        if: matrix.python-version != '3.11'
+        if: matrix.python-version != '3.12'
         uses: actions/setup-python@v4
         with:
           python-version: 3
@@ -93,4 +94,4 @@ jobs:
         run: |
           nox -rs test-${{ matrix.python-version }}
         env:
-          WAIT_FOR_ES: "1"
+          WAIT_FOR_ES: "1"

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3"
+    python: "3.12"
 
 python:
   install:

Changelog.rst

@@ -3,6 +3,16 @@
 Changelog
 =========
 
+8.11.0 (2023-11-13)
+-------------------
+
+* Added support for Python 3.12 (`#1680`_)
+* Added support for Search.collase() (`#1649`_, contributed by `@qcoumes`_)
+
+.. _@qcoumes: https://github.com/qcoumes
+.. _#1680: https://github.com/elastic/elasticsearch-dsl-py/pull/1680
+.. _#1649: https://github.com/elastic/elasticsearch-dsl-py/pull/1649
+
 8.9.0 (2023-09-07)
 ------------------
 

docs/search_dsl.rst

@@ -18,6 +18,12 @@ The ``Search`` object represents the entire search request:
 
   * pagination
 
+  * highlighting
+
+  * suggestions
+
+  * collapsing
+
   * additional parameters
 
   * associated client
@@ -433,7 +439,26 @@ keyword arguments will be added to the suggest's json as-is which means that it
 should be one of ``term``, ``phrase`` or ``completion`` to indicate which type
 of suggester should be used.
 
+Collapsing
+~~~~~~~~~~
+
+To collapse search results use the ``collapse`` method on your ``Search`` object:
+
+.. code:: python
+
+    s = Search().query("match", message="GET /search")
+    # collapse results by user_id
+    s = s.collapse("user_id")
+
+The top hits will only include one result per ``user_id``. You can also expand
+each collapsed top hit with the ``inner_hits`` parameter,
+``max_concurrent_group_searches`` being the number of concurrent requests
+allowed to retrieve the inner hits per group:
+
+.. code:: python
 
+    inner_hits = {"name": "recent_search", "size": 5, "sort": [{"@timestamp": "desc"}]}
+    s = s.collapse("user_id", inner_hits=inner_hits, max_concurrent_group_searches=4)
 
 More Like This Query
 ~~~~~~~~~~~~~~~~~~~~

elasticsearch_dsl/__init__.py

@@ -84,7 +84,7 @@
 from .utils import AttrDict, AttrList, DslBase
 from .wrappers import Range
 
-VERSION = (8, 9, 0)
+VERSION = (8, 11, 0)
 __version__ = VERSION
 __versionstr__ = ".".join(map(str, VERSION))
 __all__ = [

elasticsearch_dsl/search.py

@@ -120,6 +120,7 @@ def __init__(self, using="default", index=None, doc_type=None, extra=None):
 
         self._doc_type = []
         self._doc_type_map = {}
+        self._collapse = {}
         if isinstance(doc_type, (tuple, list)):
             self._doc_type.extend(doc_type)
         elif isinstance(doc_type, collections.abc.Mapping):
@@ -293,6 +294,7 @@ def _clone(self):
         s = self.__class__(
             using=self._using, index=self._index, doc_type=self._doc_type
         )
+        s._collapse = self._collapse.copy()
         s._doc_type_map = self._doc_type_map.copy()
         s._extra = self._extra.copy()
         s._params = self._params.copy()
@@ -318,6 +320,7 @@ def __init__(self, **kwargs):
 
         self.aggs = AggsProxy(self)
         self._sort = []
+        self._collapse = {}
         self._source = None
         self._highlight = {}
         self._highlight_opts = {}
@@ -568,6 +571,27 @@ def sort(self, *keys):
             s._sort.append(k)
         return s
 
+    def collapse(self, field=None, inner_hits=None, max_concurrent_group_searches=None):
+        """
+        Add collapsing information to the search request.
+        If called without providing ``field``, it will remove all collapse
+        requirements, otherwise it will replace them with the provided
+        arguments.
+        The API returns a copy of the Search object and can thus be chained.
+        """
+        s = self._clone()
+        s._collapse = {}
+
+        if field is None:
+            return s
+
+        s._collapse["field"] = field
+        if inner_hits:
+            s._collapse["inner_hits"] = inner_hits
+        if max_concurrent_group_searches:
+            s._collapse["max_concurrent_group_searches"] = max_concurrent_group_searches
+        return s
+
     def highlight_options(self, **kwargs):
         """
         Update the global highlighting options used for this request. For
@@ -663,6 +687,9 @@ def to_dict(self, count=False, **kwargs):
             if self._sort:
                 d["sort"] = self._sort
 
+            if self._collapse:
+                d["collapse"] = self._collapse
+
             d.update(recursive_to_dict(self._extra))
 
             if self._source not in (None, {}):

noxfile.py

@@ -35,6 +35,7 @@
         "3.9",
         "3.10",
         "3.11",
+        "3.12",
     ]
 )
 def test(session):
@@ -52,7 +53,7 @@ def test(session):
     session.run("pytest", *argv)
 
 
-@nox.session()
+@nox.session(python="3.12")
 def format(session):
     session.install("black~=23.0", "isort")
     session.run("black", "--target-version=py37", *SOURCE_FILES)
@@ -62,7 +63,7 @@ def format(session):
     lint(session)
 
 
-@nox.session
+@nox.session(python="3.12")
 def lint(session):
     session.install("flake8", "black~=23.0", "isort")
     session.run("black", "--check", "--target-version=py37", *SOURCE_FILES)

setup.cfg

@@ -13,3 +13,4 @@ filterwarnings =
     # https://github.com/elastic/elasticsearch-py/issues/2181#issuecomment-1490932964
     ignore:The 'body' parameter is deprecated .*:DeprecationWarning
     ignore:Legacy index templates are deprecated in favor of composable templates.:elasticsearch.exceptions.ElasticsearchWarning
+    ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version..*:DeprecationWarning

setup.py

@@ -19,7 +19,7 @@
 
 from setuptools import find_packages, setup
 
-VERSION = (8, 9, 0)
+VERSION = (8, 11, 0)
 __version__ = VERSION
 __versionstr__ = ".".join(map(str, VERSION))
 
@@ -68,6 +68,7 @@
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
     ],

tests/test_search.py

@@ -256,6 +256,38 @@ def test_sort_by_score():
         s.sort("-_score")
 
 
+def test_collapse():
+    s = search.Search()
+
+    inner_hits = {"name": "most_recent", "size": 5, "sort": [{"@timestamp": "desc"}]}
+    s = s.collapse("user.id", inner_hits=inner_hits, max_concurrent_group_searches=4)
+
+    assert {
+        "field": "user.id",
+        "inner_hits": {
+            "name": "most_recent",
+            "size": 5,
+            "sort": [{"@timestamp": "desc"}],
+        },
+        "max_concurrent_group_searches": 4,
+    } == s._collapse
+    assert {
+        "collapse": {
+            "field": "user.id",
+            "inner_hits": {
+                "name": "most_recent",
+                "size": 5,
+                "sort": [{"@timestamp": "desc"}],
+            },
+            "max_concurrent_group_searches": 4,
+        }
+    } == s.to_dict()
+
+    s = s.collapse()
+    assert {} == s._collapse
+    assert search.Search().to_dict() == s.to_dict()
+
+
 def test_slice():
     s = search.Search()
     assert {"from": 3, "size": 7} == s[3:10].to_dict()
@@ -305,6 +337,7 @@ def test_complex_example():
         s.query("match", title="python")
         .query(~Q("match", title="ruby"))
         .filter(Q("term", category="meetup") | Q("term", category="conference"))
+        .collapse("user_id")
         .post_filter("terms", tags=["prague", "czech"])
         .script_fields(more_attendees="doc['attendees'].value + 42")
     )
@@ -342,6 +375,7 @@ def test_complex_example():
                 "aggs": {"avg_attendees": {"avg": {"field": "attendees"}}},
             }
         },
+        "collapse": {"field": "user_id"},
         "highlight": {
             "order": "score",
             "fields": {"title": {"fragment_size": 50}, "body": {"fragment_size": 50}},