Merge branch 'pandas-dev:main' into pdep_voting_wip

Author: noatamir

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
       image: ubuntu-2004:2022.04.1
     resource_class: arm.large
     environment:
-      ENV_FILE: ci/deps/circle-39-arm64.yaml
+      ENV_FILE: ci/deps/circle-310-arm64.yaml
       PYTEST_WORKERS: auto
       PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
       PYTEST_TARGET: "pandas"
@@ -26,7 +26,6 @@ jobs:
       image: ubuntu-2004:2022.04.1
     resource_class: arm.large
     environment:
-      ENV_FILE: ci/deps/circle-38-arm64.yaml
       TRIGGER_SOURCE: << pipeline.trigger_source >>
     steps:
       - checkout

.circleci/setup_env.sh

@@ -54,10 +54,7 @@ if pip list | grep -q ^pandas; then
     pip uninstall -y pandas || true
 fi
 
-echo "Build extensions"
-python setup.py build_ext -q -j4
-
 echo "Install pandas"
-python -m pip install --no-build-isolation --no-use-pep517 -e .
+python -m pip install --no-build-isolation -ve .
 
 echo "done"

.github/CODEOWNERS

@@ -0,0 +1,22 @@
+# github
+.github/                          @mroeschke
+
+# ci
+ci/                               @mroeschke
+
+# web
+web/                              @datapythonista
+
+# docs
+doc/cheatsheet                    @Dr-Irv
+
+# pandas
+pandas/_libs/                     @WillAyd
+pandas/_libs/tslibs/*             @MarcoGorelli
+pandas/_typing.py                 @Dr-Irv
+pandas/core/groupby/*             @rhshadrach
+pandas/core/tools/datetimes.py    @MarcoGorelli
+pandas/io/excel/*                 @rhshadrach
+pandas/io/formats/style.py        @attack68
+pandas/io/formats/style_render.py @attack68
+pandas/io/formats/templates       @attack68

.github/actions/build_pandas/action.yml

@@ -1,5 +1,9 @@
 name: Build pandas
 description: Rebuilds the C extensions and installs pandas
+inputs:
+  editable:
+    description: Whether to build pandas in editable mode (default true)
+    default: true
 runs:
   using: composite
   steps:
@@ -12,9 +16,9 @@ runs:
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j $N_JOBS
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        if [[ ${{ inputs.editable }} == "true" ]]; then
+          pip install -e . --no-build-isolation -v
+        else
+          pip install . --no-build-isolation -v
+        fi
       shell: bash -el {0}
-      env:
-        # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-        N_JOBS: ${{ runner.os == 'macOS' && 3 || 2 }}

.github/actions/setup-conda/action.yml

@@ -3,23 +3,14 @@ inputs:
   environment-file:
     description: Conda environment file to use.
     default: environment.yml
-  environment-name:
-    description: Name to use for the Conda environment
-    default: test
-  extra-specs:
-    description: Extra packages to install
-    required: false
 runs:
   using: composite
   steps:
     - name: Install ${{ inputs.environment-file }}
-      uses: mamba-org/provision-with-micromamba@v15
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: ${{ inputs.environment-file }}
-        environment-name: ${{ inputs.environment-name }}
-        extra-specs: ${{ inputs.extra-specs }}
-        channels: conda-forge
-        channel-priority: 'strict'
+        environment-name: test
         condarc-file: ci/condarc.yml
-        cache-env: true
+        cache-environment: true
         cache-downloads: true

.github/workflows/cache-cleanup.yml

@@ -0,0 +1,30 @@
+name: Clean closed branch caches
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clean Cache
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+          BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
+
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/code-checks.yml

@@ -63,8 +63,24 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
+      with:
+        editable: false
 
     # The following checks are independent of each other and should still be run if one fails
+
+    # TODO: The doctests have to be run first right now, since the Cython doctests only work
+    # with pandas installed in non-editable mode
+    # This can be removed once pytest-cython doesn't require C extensions to be installed inplace
+    - name: Run doctests
+      run: cd ci && ./code_checks.sh doctests
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
+    - name: Install pandas in editable mode
+      id: build-editable
+      uses: ./.github/actions/build_pandas
+      with:
+        editable: true
+
     - name: Check for no warnings when building single-page docs
       run: ci/code_checks.sh single-docs
       if: ${{ steps.build.outcome == 'success' && always() }}
@@ -73,10 +89,6 @@ jobs:
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Run doctests
-      run: ci/code_checks.sh doctests
-      if: ${{ steps.build.outcome == 'success' && always() }}
-
     - name: Run docstring validation
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/codeql.yml

@@ -18,6 +18,7 @@ jobs:
       actions: read
       contents: read
       security-events: write
+    if: github.repository_owner == 'pandas-dev'
 
     strategy:
       fail-fast: false

.github/workflows/package-checks.yml

@@ -14,6 +14,10 @@ on:
 permissions:
   contents: read
 
+defaults:
+  run:
+    shell: bash -el {0}
+
 jobs:
   pip:
     if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
@@ -40,13 +44,38 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install required dependencies
-        run: |
-          python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
-          python -m pip install versioneer[toml]
-        shell: bash -el {0}
-
       - name: Pip install with extra
         run: |
-          python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation
+          python -m pip install .[${{ matrix.extra }}] -v
         shell: bash -el {0}
+  conda_forge_recipe:
+    if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    name: Test Conda Forge Recipe - Python ${{ matrix.python-version }}
+    concurrency:
+      # https://i.8713187.xyzmunity/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-conda-forge-recipe-${{ matrix.python-version }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: recipe-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            boa
+            conda-verify
+          cache-downloads: true
+          cache-environment: true
+
+      - name: Build conda package
+        run: conda mambabuild ci --no-anaconda-upload --verify --strict-verify --output --output-folder .

.github/workflows/stale-pr.yml

@@ -11,6 +11,7 @@ jobs:
   stale:
     permissions:
       pull-requests: write
+    if: github.repository_owner == 'pandas-dev'
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/stale@v8

.github/workflows/unit-tests.yml

@@ -31,14 +31,14 @@ jobs:
         pattern: [""]
         include:
           - name: "Downstream Compat"
-            env_file: actions-39-downstream_compat.yaml
+            env_file: actions-311-downstream_compat.yaml
             pattern: "not slow and not network and not single_cpu"
             pytest_target: "pandas/tests/test_downstream.py"
           - name: "Minimum Versions"
             env_file: actions-39-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
           - name: "Locale: it_IT"
-            env_file: actions-310.yaml
+            env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-it"
             # Use the utf8 version as the default, it has no bad side-effect.
@@ -48,7 +48,7 @@ jobs:
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "it_IT"
           - name: "Locale: zh_CN"
-            env_file: actions-310.yaml
+            env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-zh-hans"
             # Use the utf8 version as the default, it has no bad side-effect.
@@ -58,15 +58,15 @@ jobs:
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
           - name: "Copy-on-Write"
-            env_file: actions-310.yaml
+            env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "--max-worker-restart 0"
           - name: "Numpy Dev"
-            env_file: actions-310-numpydev.yaml
+            env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
             # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
@@ -229,11 +229,10 @@ jobs:
         run: |
           /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir --no-deps -U pip wheel setuptools
+          python -m pip install -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
-          python setup.py build_ext -q -j$(nproc)
-          python -m pip install --no-cache-dir --no-build-isolation --no-use-pep517 -e .
-          python -m pip list
+          python -m pip install --no-cache-dir --no-build-isolation -e .
+          python -m pip list --no-cache-dir
           export PANDAS_CI=1
           python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
     concurrency:
@@ -268,10 +267,9 @@ jobs:
         run: |
           /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir --no-deps -U pip wheel setuptools
+          python -m pip install -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
-          python setup.py build_ext -q -j$(nproc)
-          python -m pip install --no-cache-dir --no-build-isolation --no-use-pep517 -e .
+          python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
       - name: Run Tests
@@ -339,16 +337,15 @@ jobs:
         run: |
           python --version
           python -m pip install --upgrade pip setuptools wheel
-          python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
+          python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install git+https://github.com/nedbat/coveragepy.git
           python -m pip install versioneer[toml]
           python -m pip install python-dateutil pytz cython hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip list
 
       - name: Build Pandas
         run: |
-          python setup.py build_ext -q -j4
-          python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+          python -m pip install -e . --no-build-isolation --no-index
 
       - name: Build Version
         run: |