[7.17] Fix CI

Author: pquentin

.ci/Dockerfile renamed to .buildkite/Dockerfile

@@ -7,19 +7,14 @@ ARG BUILDER_UID=1000
 ARG BUILDER_GID=1000
 ENV BUILDER_USER elastic
 ENV BUILDER_GROUP elastic
+ENV PATH="${PATH}:/var/lib/elastic/.local/bin"
 
 # Create user
 RUN groupadd --system -g ${BUILDER_GID} ${BUILDER_GROUP} \
     && useradd --system --shell /bin/bash -u ${BUILDER_UID} -g ${BUILDER_GROUP} -d /var/lib/elastic -m elastic 1>/dev/null 2>/dev/null \
     && mkdir -p /code/elasticsearch-py && mkdir /code/elasticsearch-py/build \
-    && chown -R ${BUILDER_USER}:${BUILDER_GROUP} /code/elasticsearch-py
-COPY --chown=$BUILDER_USER:$BUILDER_GROUP . .
+    && chown -R ${BUILDER_USER}:${BUILDER_GROUP} /code/
 WORKDIR /code/elasticsearch-py
 USER ${BUILDER_USER}:${BUILDER_GROUP}
-COPY dev-requirements.txt .
-RUN python -m pip install \
-    -U --no-cache-dir \
-    --disable-pip-version-check \
-    nox -rdev-requirements.txt
+RUN python -m pip install --disable-pip-version-check nox
 COPY --chown=$BUILDER_USER:$BUILDER_GROUP . .
-RUN python -m pip install -U -e .

.buildkite/certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD
+VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/
+40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y
+RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq
+Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n
+gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT
+8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C
+AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY
+MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX
+dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb
+GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0
+hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj
+cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx
+EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI=
+-----END CERTIFICATE-----

.buildkite/certs/ca.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD
+VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/
+40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y
+RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq
+Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n
+gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT
+8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C
+AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY
+MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX
+dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb
+GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0
+hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj
+cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx
+EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI=
+-----END CERTIFICATE-----

.buildkite/certs/testnode.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDODCCAiCgAwIBAgIVAKLWEcNzTd4B0NqnrJL0xAKaS8DWMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTIzMDgyMTE3MjcwMloXDTI2MDgyMDE3MjcwMlowEzERMA8G
+A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8
+eLXL3ZX5v8JlHcfg+96Bpq24EeiqV+7RPPKbcH80ODjkETqYUpam+TcOl2gt23p/
+rpiPSSpOX8pFdmY78wTmxo2GCQZ/db2h0gZOOYpb8HQku+hJ4bAmtzizrqWW76Wz
+csen3DSUkT0bKkJTjUMmwVhRaMpfv8EIcUbrHAwc3VCj7grnFL0kdAuQa6iyBH4I
+lTUYOIOVyEJ8zZ7R4BJO3QU+TRuJ5+w/QiZMeDqxtrdDL37vYQHPW7L/XISCCOMp
+sA3avzFphoQXBQ8mjdB8Txkd4sH7mJTqnRp5ILhRzVpcPPgQYFeIB567B+kFeSau
+aJJmc0EVgOcK5aSMtOH3AgMBAAGjYjBgMB0GA1UdDgQWBBQsZbZDudZ63h52FlU5
+N2g3pznkETAfBgNVHSMEGDAWgBR1YUax123eUBRIJN0RZ+GOC2KPTzATBgNVHREE
+DDAKgghpbnN0YW5jZTAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAyv0Cw
+OrvZn7FHHS8TJI5vTi1F43R/eSNMNL/+q/nK93KaxWJH1T4zrJhrJ9KpzkFcalXP
+bu02oTh28b3o3QpS2wdwMv/Q3NLoMBEmQlG2UrELFvV43nS8LCiwCX3o11L1HZP3
+1Z/rclwxbA4OQ/ZkPcol++TDZQTM/8WkIdZmTL4UDb/ppDjX24nTOitkMRZlYAOY
+mid9GGExhKrUJ0I9/A3w1hWRA1Hwc+1TFDcPphl2x2uQ9HJFBueAvuFXmIjDki1x
+qrvnFZ+mneI9kR4m82MX900WF15KS35GzmMui0tsf0wbfy3Jh+WnpMlIIa2OQXw7
+prbkg9tScQSsvhC8
+-----END CERTIFICATE-----

.buildkite/certs/testnode.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvHi1y92V+b/CZR3H4PvegaatuBHoqlfu0Tzym3B/NDg45BE6
+mFKWpvk3DpdoLdt6f66Yj0kqTl/KRXZmO/ME5saNhgkGf3W9odIGTjmKW/B0JLvo
+SeGwJrc4s66llu+ls3LHp9w0lJE9GypCU41DJsFYUWjKX7/BCHFG6xwMHN1Qo+4K
+5xS9JHQLkGuosgR+CJU1GDiDlchCfM2e0eASTt0FPk0biefsP0ImTHg6sba3Qy9+
+72EBz1uy/1yEggjjKbAN2r8xaYaEFwUPJo3QfE8ZHeLB+5iU6p0aeSC4Uc1aXDz4
+EGBXiAeeuwfpBXkmrmiSZnNBFYDnCuWkjLTh9wIDAQABAoIBAAU0iEDTI9s78pB8
+XBLYofKOuemFhRl/SDc7KbAlUT4N93RFDYs7bLG73Eto3xW1JBL2rXv3l1WGy71T
+YctyEMaW4T28bhODGvOnK0lpyWp0n6CMGARCWW0YTlaYEjay866bEuyN5l3cDQX9
+Csvn8NzXJitJa51tXFVxW3YO1j7Nyc/M59oyBZ1ARYYmQqFYLEu6lvJOW0cKDFkZ
+AcMVlOIxZQL/Mf+RO72aQGVuYNjqxlLIXLuE9zFR2gDFM2+l3FMUWDGHGBDFyjKU
+iMk4+sSlOTFXqO9VQzua6FLFMsQT6m5PFD4uPY92KR6CPfH/NrWqwqr+jpjaU+gs
+3U9GN+ECgYEA58qX7tKPk7CWdk3kyk5NsNcs/qib+heXWEubfhoU8LmSnbBQhOAz
+wi//r/xm0OHGj84y66+G3T347iudrLjhr07oGM1QfjYT3kb90efLjwAfCECtyVYL
+EQrWO5UeoTnmrhlB1mGL3sWaVAsVqNLz8i2H5c7sj0hxHsvM62159r8CgYEA0Cff
+opJqmUpMpHm3sgjMWctylVrHBuQe5cl5Ad80pbd6mvtt4TvGXbUGNdzURfyve9DS
+x1CVlj4Sz8VuelFQgYL+7/qUqZoms1aSgJpxWv8ou+wUHmlF3kVO8VKt3BNHV+8J
+euSB6NG91BGguBoHgnOoVcjbDGdhJGRTojCNWskCgYEA1jE3nwDCnrbTA3XNk0ky
+r9TXhmgm4r+EIpqTkL7nVOAXZVJ1xaQtECgsveKe3C2WwHLKSVMFbFMFQonZha+/
+FbHz9l9cH5U3XPL7QEpTp8xz4LtsHJ4/UbtS5vJQwKnxyjYaydGQYAb4KuunUz/F
+H6kFaM6DeZB2v/+SWIfs6Z8CgYARUdAEyeP+vzTjVpFXSe4e5pOxI619wEtl2T6t
+TjImO78C2DrLS9r0fxR2NNqgvCapybVQCj94EdAk44uOt+dX71thAluORRpFP8XO
+14rpBGQSRtFhumaq2N95quR2dFAyW9xREmRQx+rgk1rpFplbXF48TQsU3CE0Evj2
+fM22KQKBgDhob7M9sWvXecxoyy3J17jUTcFqmqKcqGnx3ZJ7Q9CgAfjYqNNQws27
+wTuaJB0PEuCOu4t+lUHEBMIjGkBfo1bHd4EZaW04Xgbfn2j8MK2e+9GlRtedxxFg
+c1JdRb5+eTgPwLcDsmMWIW357PDW7RDEI07G1ZB4SqxGTKkU7JOW
+-----END RSA PRIVATE KEY-----

.ci/functions/cleanup.sh renamed to .buildkite/functions/cleanup.sh

@@ -0,0 +1,26 @@
+steps:
+  - label: ":elasticsearch: ES {{matrix.stack_version }} :python: Python {{ matrix.python }} ({{ matrix.connection_class }})"
+    agents:
+      provider: "gcp"
+    env:
+      PYTHON_VERSION: "{{ matrix.python }}"
+      TEST_SUITE: "platinum"
+      STACK_VERSION: "{{ matrix.stack_version }}"
+      PYTHON_CONNECTION_CLASS: "{{ matrix.connection_class }}"
+    matrix:
+      setup:
+        python:
+          - "2.7"
+          - "3.6"
+          - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        connection_class:
+          - "Urllib3HttpConnection"
+          - "RequestsHttpConnection"
+        stack_version:
+          - "7.17.0-SNAPSHOT"
+          - "8.0.0-SNAPSHOT"
+    command: ./.buildkite/run-tests

.ci/functions/imports.sh renamed to .buildkite/functions/imports.sh

@@ -0,0 +1,9 @@
+{
+  "jobs": [
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-py-integration-tests",
+      "allow_org_users": true
+    }
+  ]
+}

.ci/functions/wait-for-container.sh renamed to .buildkite/functions/wait-for-container.sh

@@ -31,6 +31,12 @@ cleanup_node $es_node_name
 master_node_name=${es_node_name}
 cluster_name=${moniker}${suffix}
 
+BUILDKITE=${BUILDKITE-false}
+# Set vm.max_map_count kernel setting to 262144 if we're in CI
+if [[ "$BUILDKITE" == "true" ]]; then
+  sudo sysctl -w vm.max_map_count=262144
+fi
+
 declare -a volumes
 environment=($(cat <<-END
   --env ELASTIC_PASSWORD=$elastic_password

.buildkite/pipeline.yml

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [[ -z "$NOX_SESSION" ]]; then
+  NOX_SESSION=test-${PYTHON_VERSION%-dev}
+fi
+nox -s $NOX_SESSION

.buildkite/pull-requests.json

@@ -24,28 +24,25 @@ echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS ${PYTHON_CONNECTION_CLAS
 echo -e "\033[1m>>>>> Build [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
 docker build \
-       --file .ci/Dockerfile \
+       --file .buildkite/Dockerfile \
        --tag elastic/elasticsearch-py \
-       --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
+       --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
+       --build-arg "BUILDER_UID=$(id -u)" \
+       --build-arg "BUILDER_GID=$(id -g)" \
        .
 
 echo -e "\033[1m>>>>> Run [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-if [[ "$STACK_VERSION" == "8.0.0-SNAPSHOT" ]]; then
-  export ELASTIC_CLIENT_APIVERSIONING="true"
-fi
-
 mkdir -p junit
 docker run \
-  -u "$(id -u)" \
+  -u "$(id -u):$(id -g)" \
   --network=${network_name} \
   --env "STACK_VERSION=${STACK_VERSION}" \
   --env "ELASTICSEARCH_URL=${elasticsearch_url}" \
   --env "TEST_SUITE=${TEST_SUITE}" \
   --env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \
   --env "TEST_TYPE=server" \
-  --env "ELASTIC_CLIENT_APIVERSIONING=${ELASTIC_CLIENT_APIVERSIONING:-false}" \
   --name elasticsearch-py \
   --rm \
   elastic/elasticsearch-py \
-  python setup.py test
+  nox -s test-${PYTHON_VERSION}

.ci/run-elasticsearch.sh renamed to .buildkite/run-elasticsearch.sh

@@ -8,23 +8,23 @@
 export STACK_VERSION="${STACK_VERSION:=8.0.0-SNAPSHOT}"
 export TEST_SUITE="${TEST_SUITE:=platinum}"
 export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
-export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}"
+export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}"
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
-echo -e "\033[1m>>>>> Start [$STACK_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-DETACH=true bash .ci/run-elasticsearch.sh
+echo "--- :elasticsearch: Starting Elasticsearch"
+DETACH=true bash $script_path/run-elasticsearch.sh
 
 if [[ -n "$RUNSCRIPTS" ]]; then
   for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
     echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
     CONTAINER_NAME=${RUNSCRIPT} \
       DETACH=true \
-      bash .ci/run-${RUNSCRIPT}.sh
+      bash $script_path/run-${RUNSCRIPT}.sh
   done
 fi
 
-echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-bash .ci/run-repository.sh
+echo "+++ :python: Client tests"
+bash $script_path/run-repository.sh