Ci lint

.bazelrc

@@ -5,20 +5,9 @@ build --copt=-DTFX_BSL_USE_ARROW_C_ABI
 build --cxxopt=-std=c++17
 build --host_cxxopt=-std=c++17
 
-# Needed to avoid zetasql proto error.
-build --protocopt=--experimental_allow_proto3_optional
-
 # icu@: In create_linking_context: in call to create_linking_context(),
 # parameter 'user_link_flags' is deprecated and will be removed soon.
 # It may be temporarily re-enabled by setting --incompatible_require_linker_input_cc_api=false
 build --incompatible_require_linker_input_cc_api=false
 build:macos --apple_platform_type=macos
 build:macos_arm64 --cpu=darwin_arm64
-
-# Most of these warnings are triggered from ZetaSQL, disable for now. Ideally,
-# we would disable these only for projects we don't control, but there does not
-# seem to be an easy way to do that yet.
-build --copt   -Wno-sign-compare
-build --copt   -Wno-deprecated-declarations
-build --copt   -Wno-return-type
-build --copt   -Wno-unused-but-set-parameter

.bazelversion

@@ -1 +1 @@
-6.5.0
+6.5.0

.github/reusable-build/action.yml

@@ -0,0 +1,39 @@
+name: Resusable steps to build tfx-bsl
+
+inputs:
+  python-version:
+    description: 'Python version'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+
+  - name: Set up Python ${{ inputs.python-version }}
+    uses: actions/setup-python@v5
+    with:
+      python-version: ${{ inputs.python-version }}
+
+  - name: Upgrade pip
+    shell: bash
+    run: |
+      python -m pip install --upgrade pip pytest
+
+  - name: Build the package for Python ${{ inputs.python-version }}
+    shell: bash
+    run: |
+      version="${{ inputs.python-version }}"
+      docker compose run -e PYTHON_VERSION=$(echo "$version" | sed 's/\.//') manylinux2010
+
+  - name: Upload wheel artifact for Python ${{ inputs.python-version }}
+    uses: actions/[email protected]
+    with:
+      name: tfx-bsl-wheel-py${{ inputs.python-version }}
+      path: dist/*.whl
+      if-no-files-found: error
+
+  - name: Install built wheel
+    shell: bash
+    run: |
+      pip install twine
+      twine check dist/*
+      pip install dist/*.whl

.github/workflows/build.yml

@@ -21,6 +21,8 @@ on:
   pull_request:
     branches:
       - master
+  release:
+    types: [published]
 
 jobs:
   build:
@@ -33,28 +35,38 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - name: Build tfx-bsl
+      id: build-tfx-bsl
+      uses: ./.github/reusable-build
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Upgrade pip
-      run: |
-        python -m pip install --upgrade pip
 
-    - name: Build the manylinux2010 image
-      run: docker compose build manylinux2010
-
-    - name: Build the package for Python ${{ matrix.python-version }}
-      run: |
-        version="${{ matrix.python-version }}"
-        docker compose run -e PYTHON_VERSION=$(echo "$version" | sed 's/\.//') manylinux2010
-
-    - name: Upload wheel artifact for Python ${{ matrix.python-version }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: tfx-bsl-wheel-py${{ matrix.python-version }}
-        path: dist/*.whl
-
-    - name: Install built wheel
-      run: pip install dist/*.whl
+  upload_to_pypi:
+    name: Upload to PyPI
+    runs-on: ubuntu-latest
+    if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags'))
+    needs: [build]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/tfx-bsl/
+    permissions:
+      id-token: write
+      attestations: write
+    steps:
+      - name: Retrieve wheels
+        uses: actions/[email protected]
+        with:
+          merge-multiple: true
+          path: wheels
+      - name: Generate artifact attestations for wheels
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: "wheels/*"
+      - name: List the build artifacts
+        run: |
+          ls -lAs wheels/
+      - name: Upload to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1.9
+        with:
+          packages_dir: wheels/

.github/workflows/ci-lint.yml

@@ -0,0 +1,21 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+     branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+      with:
+        # Ensure the full history is fetched
+        # This is required to run pre-commit on a specific set of commits
+        # TODO: Remove this when all the pre-commit issues are fixed
+        fetch-depth: 0
+    - uses: actions/[email protected]
+      with:
+        python-version: 3.13
+    - uses: pre-commit/[email protected]

tfx_bsl/arrow/sql_util.py → .github/workflows/test.yml

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,17 +11,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Arrow Array utilities."""
 
-import sys
+name: Test tfx-bsl
 
-# pytype: disable=import-error
-# pylint: disable=unused-import
-# pylint: disable=g-import-not-at-top
-# See b/148667210 for why the ImportError is ignored.
-try:
-  from tfx_bsl.cc.tfx_bsl_extension.arrow.sql_util import RecordBatchSQLSliceQuery
-except ImportError as err:
-  sys.stderr.write(
-      "Error importing tfx_bsl_google_extension.arrow.sql_util. "
-      "Some tfx_bsl functionalities are not available: {}".format(err))
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Build tfx-bsl
+      id: build-tfx-bsl
+      uses: ./.github/reusable-build
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Test
+      run: |
+        pip install pytest
+        rm bazel-*
+        pytest -vvv

.gitignore

@@ -126,4 +126,4 @@ dmypy.json
 .pyre/
 
 # pb2.py files
-*_pb2.py
+*_pb2.py

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+# pre-commit is a tool to perform a predefined set of tasks manually and/or
+# automatically before git commits are made.
+#
+# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level
+#
+# Common tasks
+#
+# - Register git hooks: pre-commit install --install-hooks
+# - Run on all files:   pre-commit run --all-files
+#
+# These pre-commit hooks are run as CI.
+#
+# NOTE: if it can be avoided, add configs/args in pyproject.toml or below instead of creating a new `.config.file`.
+# https://pre-commit.ci/#configuration
+ci:
+  autoupdate_schedule: monthly
+  autofix_commit_msg: |
+    [pre-commit.ci] Apply automatic pre-commit fixes
+
+repos:
+  # general
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+        exclude: '\.svg$|\.patch$'
+      - id: trailing-whitespace
+        exclude: '\.svg$|\.patch$'
+      - id: check-json
+      - id: check-yaml
+        args: [--allow-multiple-documents, --unsafe]
+      - id: check-toml
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.6
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format

LICENSE

@@ -226,4 +226,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

WORKSPACE

@@ -71,18 +71,6 @@ http_archive(
     ],
 )
 
-# Needed by abseil-py by zetasql.
-http_archive(
-    name = "six_archive",
-    build_file = "//third_party:six.BUILD",
-    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
-    strip_prefix = "six-1.10.0",
-    urls = [
-        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
-        "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
-    ],
-)
-
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
@@ -217,45 +205,6 @@ load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")  #, "go_repository")
 
 gazelle_dependencies()
 
-################################################################################
-# ZetaSQL                                                                      #
-################################################################################
-
-ZETASQL_COMMIT = "a516c6b26d183efc4f56293256bba92e243b7a61"  # 11/01/2024
-
-http_archive(
-    name = "com_google_zetasql",
-    patch_args = ["-p1"],
-    patches = ["//third_party:zetasql.patch"],
-    sha256 = "1afc2210d4aad371eff0a6bfdd8417ba99e02183a35dff167af2fa6097643f26",
-    strip_prefix = "zetasql-%s" % ZETASQL_COMMIT,
-    urls = ["https://github.com/google/zetasql/archive/%s.tar.gz" % ZETASQL_COMMIT],
-)
-
-load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1")
-
-zetasql_deps_step_1()
-
-load("@com_google_zetasql//bazel:zetasql_deps_step_2.bzl", "zetasql_deps_step_2")
-
-zetasql_deps_step_2(
-    analyzer_deps = True,
-    evaluator_deps = True,
-    java_deps = False,
-    testing_deps = False,
-    tools_deps = False,
-)
-
-# No need to run zetasql_deps_step_3 and zetasql_deps_step_4 since all necessary dependencies are
-# already installed.
-
-# load("@com_google_zetasql//bazel:zetasql_deps_step_3.bzl", "zetasql_deps_step_3")
-
-# zetasql_deps_step_3()
-
-# load("@com_google_zetasql//bazel:zetasql_deps_step_4.bzl", "zetasql_deps_step_4")
-
-# zetasql_deps_step_4()
 
 _PLATFORMS_VERSION = "0.0.6"