Update 9.0 - backports from main

.buildkite/pipeline.yml

@@ -12,7 +12,7 @@ steps:
         ruby_source:
           - "ruby"
         transport:
-          - "8.3"
+          - "8.4"
       adjustments:
         - with: # JRuby tests
             ruby: "9.4"
@@ -55,8 +55,8 @@ steps:
     key: run-serverless-tests
     depends_on:
       - step: create-serverless
-    soft_fail:
-      - exit_status: 1
+    # soft_fail:
+    #   - exit_status: 1
     agents:
       provider: "gcp"
     env:

.github/make.sh

@@ -128,6 +128,7 @@ docker run \
        -u "$(id -u)" \
        --env "RUBY_VERSION=${RUBY_VERSION}" \
        --env "WORKFLOW=${WORKFLOW}" \
+       --env "CLIENTS_GITHUB_TOKEN=${CLIENTS_GITHUB_TOKEN}" \
        --name test-runner \
        --volume "${repo}:/usr/src/app" \
        --rm \

.github/workflows/9.0.yml

@@ -7,7 +7,7 @@ on:
     branches:
       - 9.0
 jobs:
-  test-9.0:
+  tests:
     env:
       TEST_ES_SERVER: http://localhost:9250
       PORT: 9250

.github/workflows/otel.yml

@@ -2,10 +2,10 @@ name: opentelemetry
 on:
   push:
     branches:
-      - main
+      - 9.0
   pull_request:
     branches:
-      - main
+      - 9.0
 jobs:
   test-otel:
     name: 'Test Open Telemetry'

.gitignore

@@ -16,4 +16,5 @@ profile/**/data/*.json
 parsed_alternative_report.json
 .byebug_history
 build/
-*.gem
+*.gem
+elastic-client-generator-ruby

CHANGELOG-9.x.md

@@ -2,7 +2,7 @@
 
 ## Gem
 
-The size of both `elasticsearch` and `elasticsearch-api` gems will be smaller, since some unnecessary files that were being included in the gem have been removed.
+The size of both `elasticsearch` and `elasticsearch-api` gems will be smaller, since some unnecessary files that were being included in the gem have been removed. There's also been a lot of old code cleanup for `9.x`.
 
 The required Ruby version is set to `2.6` to keep compatiblity wit JRuby 9.3. However, we only test the code against currently supported Ruby versions.
 
@@ -12,6 +12,11 @@ The CI build now runs tests to ensure compatibility with Elasticsearch Serverles
 
 ## Elasticsearch API
 
+* The source code is now based on `elasticsearch-specification`, so the API documentation is much more detailed and extensive.
+* Scroll APIs: Since sending the `scroll_id` as a parameter was deprecated, now it needs to be sent in the body for `clear_scroll`, `scroll`.
+* `indices.get_field_mapping` - `:fields` is a required parameter.
+* The functions in `utils.rb` that had names starting with double underscore have been renamed to remove the underscores.
+
 ### Development
 
 #### Testing

Gemfile

@@ -38,5 +38,5 @@ end
 group :development, :test do
   gem 'debug' unless defined?(JRUBY_VERSION)
   gem 'rspec'
-  gem 'rubocop', '>= 1.51' unless defined?(JRUBY_VERSION) && Gem::Version.new(JRUBY_VERSION) <= Gem::Version.new('9.4')
+  gem 'rubocop', '>= 1.51' unless defined?(JRUBY_VERSION)
 end

elasticsearch-api/Rakefile

@@ -35,6 +35,7 @@ namespace :test do
   task :unit
   RSpec::Core::RakeTask.new(:unit) do |t|
     t.pattern = 'spec/unit/**/*_spec.rb'
+    t.exclude_pattern = 'spec/unit/perform_request_spec.rb' unless ENV['TEST_WITH_OTEL']
   end
 
   desc 'Run unit and integration tests'

elasticsearch-api/lib/elasticsearch/api.rb

@@ -15,80 +15,102 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "cgi"
-require "multi_json"
-
-require "elasticsearch/api/version"
-require "elasticsearch/api/namespace/common"
-require "elasticsearch/api/utils"
+require 'cgi'
+require 'multi_json'
+require 'elasticsearch/api/version'
+require 'elasticsearch/api/utils'
 require 'elasticsearch/api/response'
 
-Dir[ File.expand_path('../api/actions/**/*.rb', __FILE__) ].each   { |f| require f }
-Dir[ File.expand_path('../api/namespace/**/*.rb', __FILE__) ].each { |f| require f }
+Dir[File.expand_path('api/actions/**/*.rb', __dir__)].each { |f| require f }
 
 module Elasticsearch
   # This is the main module for including all API endpoint functions
   # It includes the namespace modules from ./api/actions
   module API
+    include Elasticsearch::API::Actions
     DEFAULT_SERIALIZER = MultiJson
 
     HTTP_GET          = 'GET'.freeze
     HTTP_HEAD         = 'HEAD'.freeze
     HTTP_POST         = 'POST'.freeze
     HTTP_PUT          = 'PUT'.freeze
     HTTP_DELETE       = 'DELETE'.freeze
-    UNDERSCORE_SEARCH = '_search'.freeze
-    UNDERSCORE_ALL    = '_all'.freeze
-    DEFAULT_DOC       = '_doc'.freeze
 
-    # Auto-include all namespaces in the receiver
+    module CommonClient
+      attr_reader :client
+
+      def initialize(client)
+        @client = client
+      end
+
+      def perform_request(method, path, params = {}, body = nil, headers = nil, request_opts = {})
+        client.perform_request(method, path, params, body, headers, request_opts)
+      end
+    end
+
+    # Add new namespaces to this constant
     #
-    def self.included(base)
-      base.send :include,
-                Elasticsearch::API::Common,
-                Elasticsearch::API::Actions,
-                Elasticsearch::API::Cluster,
-                Elasticsearch::API::Nodes,
-                Elasticsearch::API::Indices,
-                Elasticsearch::API::Ingest,
-                Elasticsearch::API::Snapshot,
-                Elasticsearch::API::Tasks,
-                Elasticsearch::API::Cat,
-                Elasticsearch::API::Remote,
-                Elasticsearch::API::DanglingIndices,
-                Elasticsearch::API::Features,
-                Elasticsearch::API::AsyncSearch,
-                Elasticsearch::API::Autoscaling,
-                Elasticsearch::API::CrossClusterReplication,
-                Elasticsearch::API::DataFrameTransformDeprecated,
-                Elasticsearch::API::Enrich,
-                Elasticsearch::API::Eql,
-                Elasticsearch::API::Fleet,
-                Elasticsearch::API::Graph,
-                Elasticsearch::API::IndexLifecycleManagement,
-                Elasticsearch::API::License,
-                Elasticsearch::API::Logstash,
-                Elasticsearch::API::Migration,
-                Elasticsearch::API::MachineLearning,
-                Elasticsearch::API::Rollup,
-                Elasticsearch::API::SearchableSnapshots,
-                Elasticsearch::API::Security,
-                Elasticsearch::API::SnapshotLifecycleManagement,
-                Elasticsearch::API::SQL,
-                Elasticsearch::API::SSL,
-                Elasticsearch::API::TextStructure,
-                Elasticsearch::API::Transform,
-                Elasticsearch::API::Watcher,
-                Elasticsearch::API::XPack,
-                Elasticsearch::API::SearchApplication,
-                Elasticsearch::API::Synonyms,
-                Elasticsearch::API::Esql,
-                Elasticsearch::API::Inference,
-                Elasticsearch::API::Simulate,
-                Elasticsearch::API::Connector,
-                Elasticsearch::API::QueryRules
+    API_NAMESPACES = [:async_search,
+                      :cat,
+                      :cross_cluster_replication,
+                      :cluster,
+                      :connector,
+                      :dangling_indices,
+                      :enrich,
+                      :eql,
+                      :esql,
+                      :features,
+                      :fleet,
+                      :graph,
+                      :index_lifecycle_management,
+                      :indices,
+                      :inference,
+                      :ingest,
+                      :license,
+                      :logstash,
+                      :migration,
+                      :machine_learning,
+                      :nodes,
+                      :query_rules,
+                      :search_application,
+                      :searchable_snapshots,
+                      :security,
+                      :simulate,
+                      :snapshot_lifecycle_management,
+                      :snapshot,
+                      :sql,
+                      :ssl,
+                      :synonyms,
+                      :tasks,
+                      :text_structure,
+                      :transform,
+                      :watcher,
+                      :xpack].freeze
+
+    UPPERCASE_APIS = ['sql', 'ssl'].freeze
+    API_NAMESPACES.each do |namespace|
+      name = namespace.to_s
+      module_name = if UPPERCASE_APIS.include?(name)
+                      name.upcase
+                    elsif name == 'xpack'
+                      'XPack'
+                    else
+                      name.split('_').map(&:capitalize).join
+                    end
+      class_name = "#{module_name}Client"
+
+      klass = Class.new(Object) do
+        include CommonClient, Object.const_get("Elasticsearch::API::#{module_name}::Actions")
+      end
+      Object.const_set(class_name, klass)
+      define_method(name) do
+        instance_variable_set("@#{name}", klass.new(self))
+      end
     end
 
+    alias ml machine_learning
+    alias ilm index_lifecycle_management
+
     # The serializer class
     #
     def self.serializer

elasticsearch-api/lib/elasticsearch/api/actions/async_search/delete.rb

@@ -15,19 +15,22 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
-# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+# Auto generated from commit 69cbe7cbe9f49a2886bb419ec847cffb58f8b4fb
+# @see https://github.com/elastic/elasticsearch-specification
 #
 module Elasticsearch
   module API
     module AsyncSearch
       module Actions
-        # Deletes an async search by ID. If the search is still running, the search request will be cancelled. Otherwise, the saved search results are deleted.
+        # Delete an async search.
+        # If the asynchronous search is still running, it is cancelled.
+        # Otherwise, the saved search results are deleted.
+        # If the Elasticsearch security features are enabled, the deletion of a specific async search is restricted to: the authenticated user that submitted the original search request; users that have the +cancel_task+ cluster privilege.
         #
-        # @option arguments [String] :id The async search ID
+        # @option arguments [String] :id A unique identifier for the async search. (*Required*)
         # @option arguments [Hash] :headers Custom HTTP headers
         #
-        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
+        # @see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-async-search-submit
         #
         def delete(arguments = {})
           request_opts = { endpoint: arguments[:endpoint] || 'async_search.delete' }
@@ -47,7 +50,7 @@ def delete(arguments = {})
           _id = arguments.delete(:id)
 
           method = Elasticsearch::API::HTTP_DELETE
-          path   = "_async_search/#{Utils.__listify(_id)}"
+          path   = "_async_search/#{Utils.listify(_id)}"
           params = {}
 
           Elasticsearch::API::Response.new(

elasticsearch-api/lib/elasticsearch/api/actions/async_search/get.rb

@@ -15,22 +15,30 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
-# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+# Auto generated from commit 69cbe7cbe9f49a2886bb419ec847cffb58f8b4fb
+# @see https://github.com/elastic/elasticsearch-specification
 #
 module Elasticsearch
   module API
     module AsyncSearch
       module Actions
-        # Retrieves the results of a previously submitted async search request given its ID.
+        # Get async search results.
+        # Retrieve the results of a previously submitted asynchronous search request.
+        # If the Elasticsearch security features are enabled, access to the results of a specific async search is restricted to the user or API key that submitted it.
         #
-        # @option arguments [String] :id The async search ID
-        # @option arguments [Time] :wait_for_completion_timeout Specify the time that the request should block waiting for the final response
-        # @option arguments [Time] :keep_alive Specify the time interval in which the results (partial or final) for this search will be available
+        # @option arguments [String] :id A unique identifier for the async search. (*Required*)
+        # @option arguments [Time] :keep_alive The length of time that the async search should be available in the cluster.
+        #  When not specified, the +keep_alive+ set with the corresponding submit async request will be used.
+        #  Otherwise, it is possible to override the value and extend the validity of the request.
+        #  When this period expires, the search, if still running, is cancelled.
+        #  If the search is completed, its saved results are deleted.
         # @option arguments [Boolean] :typed_keys Specify whether aggregation and suggester names should be prefixed by their respective types in the response
+        # @option arguments [Time] :wait_for_completion_timeout Specifies to wait for the search to be completed up until the provided timeout.
+        #  Final results will be returned if available before the timeout expires, otherwise the currently available results will be returned once the timeout expires.
+        #  By default no timeout is set meaning that the currently available results will be returned without any additional wait.
         # @option arguments [Hash] :headers Custom HTTP headers
         #
-        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
+        # @see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-async-search-submit
         #
         def get(arguments = {})
           request_opts = { endpoint: arguments[:endpoint] || 'async_search.get' }
@@ -50,7 +58,7 @@ def get(arguments = {})
           _id = arguments.delete(:id)
 
           method = Elasticsearch::API::HTTP_GET
-          path   = "_async_search/#{Utils.__listify(_id)}"
+          path   = "_async_search/#{Utils.listify(_id)}"
           params = Utils.process_params(arguments)
 
           Elasticsearch::API::Response.new(

elasticsearch-api/lib/elasticsearch/api/actions/async_search/status.rb

@@ -15,20 +15,25 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
-# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+# Auto generated from commit 69cbe7cbe9f49a2886bb419ec847cffb58f8b4fb
+# @see https://github.com/elastic/elasticsearch-specification
 #
 module Elasticsearch
   module API
     module AsyncSearch
       module Actions
-        # Retrieves the status of a previously submitted async search request given its ID.
+        # Get the async search status.
+        # Get the status of a previously submitted async search request given its identifier, without retrieving search results.
+        # If the Elasticsearch security features are enabled, the access to the status of a specific async search is restricted to:
+        # * The user or API key that submitted the original async search request.
+        # * Users that have the +monitor+ cluster privilege or greater privileges.
         #
-        # @option arguments [String] :id The async search ID
-        # @option arguments [Time] :keep_alive Specify the time interval in which the results (partial or final) for this search will be available
+        # @option arguments [String] :id A unique identifier for the async search. (*Required*)
+        # @option arguments [Time] :keep_alive The length of time that the async search needs to be available.
+        #  Ongoing async searches and any saved search results are deleted after this period. Server default: 5d.
         # @option arguments [Hash] :headers Custom HTTP headers
         #
-        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
+        # @see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-async-search-submit
         #
         def status(arguments = {})
           request_opts = { endpoint: arguments[:endpoint] || 'async_search.status' }
@@ -48,7 +53,7 @@ def status(arguments = {})
           _id = arguments.delete(:id)
 
           method = Elasticsearch::API::HTTP_GET
-          path   = "_async_search/status/#{Utils.__listify(_id)}"
+          path   = "_async_search/status/#{Utils.listify(_id)}"
           params = Utils.process_params(arguments)
 
           Elasticsearch::API::Response.new(