Merge pull request #63 from questdb/nw_large_string_support

feat: support arrow large utf8 string format

Author: amunra

ci/pip_install_deps.py

@@ -3,19 +3,28 @@
 import shlex
 import textwrap
 import platform
+import argparse
 
 
+arg_parser = argparse.ArgumentParser(
+    prog='pip_install_deps.py',
+    description='installs dependencies'
+)
+
+arg_parser.add_argument('--pandasVersion')
+
 class UnsupportedDependency(Exception):
     pass
 
 
-def pip_install(package):
+def pip_install(package, version=None):
     args = [
         sys.executable,
         '-m', 'pip', 'install',
         '--upgrade',
         '--only-binary', ':all:',
-        package]
+        package if version is None else f'{package}=={version}'
+    ]
     args_s = ' '.join(shlex.quote(arg) for arg in args)
     sys.stderr.write(args_s + '\n')
     res = subprocess.run(
@@ -35,9 +44,9 @@ def pip_install(package):
         sys.exit(res.returncode)
 
 
-def try_pip_install(package):
+def try_pip_install(package, version=None):
     try:
-        pip_install(package)
+        pip_install(package, version)
     except UnsupportedDependency as e:
         msg = textwrap.indent(str(e), ' ' * 8)
         sys.stderr.write(f'    Ignored unsatisfiable dependency:\n{msg}\n')
@@ -52,12 +61,18 @@ def ensure_timezone():
         pip_install('pytz')
 
 
-def main():
+def main(args):
     ensure_timezone()
     pip_install('pip')
     pip_install('setuptools')
     try_pip_install('fastparquet>=2023.10.1')
-    try_pip_install('pandas')
+
+    print(f"DEBUG: {args}, pandas: {args.pandasVersion}")
+
+    if args.pandasVersion is not None and args.pandasVersion != '':
+        try_pip_install('pandas', args.pandasVersion)
+    else:
+        try_pip_install('pandas')
     try_pip_install('numpy')
     try_pip_install('pyarrow')
 
@@ -80,4 +95,5 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    args = arg_parser.parse_args()
+    main(args)

ci/run_tests_pipeline.yaml

@@ -11,6 +11,10 @@ stages:
             linux:
               imageName: "ubuntu-latest"
               poolName: "Azure Pipelines"
+            linux-old-pandas:
+              imageName: "ubuntu-latest"
+              poolName: "Azure Pipelines"
+              pandasVersion: "2.1.4"
             linux-qdb-master:
               imageName: "ubuntu-latest"
               poolName: "Azure Pipelines"
@@ -31,11 +35,16 @@ stages:
             lfs: false
             submodules: true
           - task: UsePythonVersion@0
-          - script: python3 --version
           - script: |
+              python3 --version
               python3 -m pip install cython
-              python3 ci/pip_install_deps.py
-            displayName: Installing Python dependencies
+            displayName: "Install cython"
+          - script: python3 ci/pip_install_deps.py
+            displayName: "Install pandas latest"
+            condition: eq(variables.pandasVersion, '')
+          - script:  python3 ci/pip_install_deps.py --pandasVersion==$(pandasVersion)
+            displayName: "Install pandas older"
+            condition: ne(variables.pandasVersion, '')
           - script: python3 proj.py build
             displayName: "Build"
           - script: |
@@ -47,7 +56,7 @@ stages:
             inputs:
               mavenPOMFile: 'questdb/pom.xml'
               jdkVersionOption: '1.11'
-              options: "install -DskipTests -Pbuild-web-console"
+              options: "-DskipTests -Pbuild-web-console"
             condition: eq(variables.vsQuestDbMaster, true)
           - script: python3 proj.py test 1
             displayName: "Test vs released"