Merge branch 'main' into pysr-integration-test

Author: MilesCranmer

.JuliaFormatter.toml

@@ -0,0 +1,8 @@
+ignore = [
+    "examples",
+    ".git",
+    ".CondaPkg",
+    ".ipynb_checkpoints",
+    ".pytest_cache",
+    ".venv",
+]

.github/workflows/docscleanup.yml

@@ -15,16 +15,15 @@ jobs:
 
       - name: Delete preview and history
         run: |
-            git config user.name "Documenter.jl"
-            git config user.email "[email protected]"
-            git rm -rf "previews/PR$PRNUM"
-            git commit -m "delete preview"
-            git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
+            if [ -d "$DIR" ]; then
+              git config user.name "Documenter.jl"
+              git config user.email "[email protected]"
+              git rm -rf "$DIR"
+              git commit -m "delete preview"
+              git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
+              git push --force origin gh-pages-new:gh-pages
+            fi
         env:
-            PRNUM: ${{ github.event.number }}
+            DIR: "previews/PR${{ github.event.number }}"
 
-      - name: Push changes
-        run: |
-            git push --force origin gh-pages-new:gh-pages
-            
 # Workflow copied from https://github.com/CliMA/TimeMachine.jl

.github/workflows/tagbot.yml

@@ -4,6 +4,22 @@ on:
     types:
       - created
   workflow_dispatch:
+    inputs:
+      lookback:
+        default: "3"
+permissions:
+  actions: read
+  checks: read
+  contents: write
+  deployments: read
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
 jobs:
   TagBot:
     if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
@@ -12,4 +28,6 @@ jobs:
       - uses: JuliaRegistries/TagBot@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          # Edit the following line to reflect the actual name of the GitHub Secret containing your private key
           ssh: ${{ secrets.DOCUMENTER_KEY }}
+          # ssh: ${{ secrets.NAME_OF_MY_SSH_PRIVATE_KEY_SECRET }}

.github/workflows/tests-nightly.yml

@@ -38,6 +38,7 @@ jobs:
       - uses: julia-actions/julia-runtest@v1
         env:
           JULIA_DEBUG: PythonCall
+          JULIA_NUM_THREADS: '2'
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:

.github/workflows/tests.yml

@@ -43,6 +43,7 @@ jobs:
         uses: julia-actions/julia-runtest@v1
         env:
           JULIA_DEBUG: PythonCall
+          JULIA_NUM_THREADS: '2'
       - name: Process coverage
         uses: julia-actions/julia-processcoverage@v1
       - name: Upload coverage to Codecov
@@ -56,17 +57,21 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        pyversion: ["3.x", "3.7"]
+        pyversion: ["3.x", "3.8"]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.pyversion }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyversion }}
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v1
+        with:
+          version: '1'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest pytest-cov virtualenv
+          pip install flake8 pytest pytest-cov nbval numpy virtualenv
           cp pysrc/juliacall/juliapkg-dev.json pysrc/juliacall/juliapkg.json
           pip install -e .
       - name: Lint with flake8
@@ -77,7 +82,9 @@ jobs:
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Run tests
         run: |
-          pytest -s --cov=pysrc
+          pytest -s --nbval --cov=pysrc ./pytest/
+        env:
+          PYTHON_JULIACALL_THREADS: '2'
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
         env:

CondaPkg.toml

@@ -1,6 +1,9 @@
 [deps.libstdcxx-ng]
 version = "<=julia"
 
+[deps.openssl]
+version = "<=julia"
+
 [deps.python]
 build = "**cpython**"
-version = ">=3.7,<4"
+version = ">=3.8,<4"

Project.toml

@@ -1,7 +1,7 @@
 name = "PythonCall"
 uuid = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 authors = ["Christopher Doris <github.com/cjdoris>"]
-version = "0.9.15"
+version = "0.9.23"
 
 [deps]
 CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
@@ -17,10 +17,19 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
 
 [compat]
-CondaPkg = "0.2.21"
+Aqua = "0 - 999"
+CondaPkg = "0.2.23"
+Dates = "1"
+Libdl = "1"
 MacroTools = "0.5"
+Markdown = "1"
+Pkg = "1"
+REPL = "1"
 Requires = "1"
+Serialization = "1"
 Tables = "1"
+Test = "1"
+TestItemRunner = "0 - 999"
 UnsafePointers = "1"
 julia = "1.6.1"
 

README.md

@@ -1,4 +1,4 @@
-<h1><img src="https://raw.githubusercontent.com/JuliaPy/PythonCall.jl/main/docs/src/assets/logo.png" alt="PythonCall.jl logo" style="width: 100px;"><br>PythonCall &amp;&nbsp;JuliaCall</h1>
+<h1><img src="docs/src/assets/logo.png" alt="PythonCall.jl logo" style="width: 100px;"><br>PythonCall &amp;&nbsp;JuliaCall</h1>
 
 [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
 [![Stable Documentation](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliapy.github.io/PythonCall.jl/stable)
@@ -14,7 +14,7 @@ Bringing [**Python®**](https://www.python.org/) and [**Julia**](https://juliala
 - Fast non-copying conversion of numeric arrays in either direction: modify Python arrays (e.g. `bytes`, `array.array`, `numpy.ndarray`) from Julia or Julia arrays from Python.
 - Helpful wrappers: interpret Python sequences, dictionaries, arrays, dataframes and IO streams as their Julia counterparts, and vice versa.
 - Beautiful stack-traces.
-- Supports modern systems: tested on Windows, MacOS and Linux, 64-bit, Julia 1.6.1 upwards and Python 3.7 upwards.
+- Supports modern systems: tested on Windows, MacOS and Linux, 64-bit, Julia 1.6.1 upwards and Python 3.8 upwards.
 
 ⭐ If you like this, a GitHub star would be lovely thank you. ⭐
 
@@ -40,9 +40,8 @@ In this example we use the Python module JuliaCall from an IPython notebook to t
 
 ## What about PyCall?
 
-The existing package [PyCall](https://github.com/JuliaPy/PyCall.jl) is another similar interface to Python. Here we note some key differences, but a more detailed comparison is in the documentation.
+The existing package [PyCall](https://github.com/JuliaPy/PyCall.jl) is another similar interface to Python. Here we note some key differences:.
 - PythonCall supports a wider range of conversions between Julia and Python, and the conversion mechanism is extensible.
 - PythonCall by default never copies mutable objects when converting, but instead directly wraps the mutable object. This means that modifying the converted object modifies the original, and conversion is faster.
 - PythonCall does not usually automatically convert results to Julia values, but leaves them as Python objects. This makes it easier to do Pythonic things with these objects (e.g. accessing methods) and is type-stable.
-- PythonCall installs dependencies into a separate Conda environment for each Julia project. This means each Julia project can have an isolated set of Python dependencies.
-- PythonCall supports Julia 1.6.1+ and Python 3.7+ whereas PyCall supports Julia 0.7+ and Python 2.7+.
+- PythonCall installs dependencies into a separate Conda environment for each Julia project using [CondaPkg](https://github.com/JuliaPy/CondaPkg.jl). This means each Julia project can have an isolated set of Python dependencies.

bump.jl

@@ -9,7 +9,7 @@ function bump(file, oldpat, newpat)
     oldtext = read(file, String)
     newtext = replace(oldtext, oldpat => newpat)
     @assert newtext != oldtext
-    write(file, newtext)    
+    write(file, newtext)
 end
 
 function bumpver(file, pattern, oldver, newver)
@@ -27,4 +27,5 @@ bumpver("pysrc/juliacall/__init__.py", "__version__ = '{}'\n", oldver, newver)
 bumpver("pysrc/juliacall/juliapkg.json", "\"version\": \"={}\"", oldver, newver)
 bumpver("pysrc/juliacall/juliapkg-dev.json", "\"version\": \"={}\"", oldver, newver)
 bumpver("src/PythonCall.jl", "VERSION = v\"{}\"", oldver, newver)
+bumpver("src/Core/Core.jl", "VERSION = v\"{}\"", oldver, newver)
 bump("docs/src/releasenotes.md", "## Unreleased", "## $newver ($(today()))")

docs/customdocs.jl

@@ -53,8 +53,8 @@ function runner(::Type{CustomDocExpander}, node, page, doc)
         Dict{Symbol,Any}(    # NOTE: Not sure about what to put here.                            
             :module => Main, # This is supposed to be tracking python code.
             :path => "",
-            :linenumber => 0
-        )
+            :linenumber => 0,
+        ),
     )::Docs.DocStr
 
     # NOTE: This was modified because the original Documenter.create_docsnode was generating unreachable links
@@ -80,12 +80,10 @@ function _parse_docs(code::AbstractString)
     m = match(r"^(.+?)\s*-\s*(.+?)\s*(\n[\s\S]*)$", strip(code))
 
     if isnothing(m)
-        error(
-            """
-            Invalid docstring:
-            $(code)
-            """
-        )
+        error("""
+              Invalid docstring:
+              $(code)
+              """)
     end
 
     name = Symbol(m[1])
@@ -134,4 +132,4 @@ function _create_docsnode(docstring, result, object, page, doc)
     return MarkdownAST.Node(docsnode)
 end
 
-end # module CustomDocs
+end # module CustomDocs

docs/make.jl

@@ -5,29 +5,22 @@ include("customdocs.jl")
 makedocs(
     sitename = "PythonCall & JuliaCall",
     modules = [PythonCall],
+    format = Documenter.HTML(assets = ["assets/favicon.ico"]),
     warnonly = [:missing_docs], # avoid raising error when docs are missing
     pages = [
         "Home" => "index.md",
-        "The Julia module PythonCall" => [
-            "Guide" => "pythoncall.md",
-            "Reference" => "pythoncall-reference.md",
-        ],
-        "The Python module JuliaCall" => [
-            "Guide" => "juliacall.md",
-            "Reference" => "juliacall-reference.md",
-        ],
+        "The Julia module PythonCall" =>
+            ["Guide" => "pythoncall.md", "Reference" => "pythoncall-reference.md"],
+        "The Python module JuliaCall" =>
+            ["Guide" => "juliacall.md", "Reference" => "juliacall-reference.md"],
         "Conversion" => [
             "Julia to Python" => "conversion-to-python.md",
             "Python to Julia" => "conversion-to-julia.md",
         ],
         "compat.md",
         "faq.md",
-        "pycall.md",
         "releasenotes.md",
-    ]
+    ],
 )
 
-deploydocs(
-    repo = raw"github.com/JuliaPy/PythonCall.jl.git",
-    push_preview = true
-)
+deploydocs(repo = raw"github.com/JuliaPy/PythonCall.jl.git", push_preview = true)

docs/src/assets/favicon.ico

@@ -9,6 +9,12 @@ functions to bridge a gap. We aim to keep these as minimal as possible.
 
 Whenever a Python exception is displayed by Julia, `sys.last_traceback` and friends are set. This allows the post-mortem debugger `pdb.pm()` to work. Disable by setting `PythonCall.CONFIG.auto_sys_last_traceback = false`.
 
+## Julia standard library
+
+Python objects can be serialised with the [`Serialization`](https://docs.julialang.org/en/v1/stdlib/Serialization/) stdlib.
+This uses [`pickle`](https://docs.python.org/3/library/pickle.html) library under the hood.
+You can opt into using [`dill`](https://pypi.org/project/dill/) instead by setting the environment variable `JULIA_PYTHONCALL_PICKLE="dill"`.
+
 ## Tabular data / Pandas
 
 The abstract type [`PyTable`](@ref) is for wrapper types around Python tables, providing the
@@ -67,8 +73,9 @@ The `juliacall` IPython extension adds these features to your IPython session:
 
 The extension is experimental and unstable - the API can change at any time.
 
-Enable the extension with `%load_ext juliacall`.
-See [the IPython docs](https://ipython.readthedocs.io/en/stable/config/extensions/).
+You can explicitly enable the extension with `%load_ext juliacall`, but
+it will automatically be loaded if `juliacall` is imported and IPython is detected.
+You can disable this behavior with an [environment variable](@ref julia-config).
 
 The `%%julia` cell magic can synchronise variables between Julia and Python by listing them
 on the first line:
@@ -88,6 +95,9 @@ In [5]: z
 Out[5]: '2^8 = 256'
 ```
 
+Also see [the IPython docs](https://ipython.readthedocs.io/en/stable/config/extensions/)
+for more information on extensions.
+
 ## Asynchronous Julia code (including Makie)
 
 Asynchronous Julia code will not normally run while Python is executing, unless it is in a

docs/src/assets/logo.png

@@ -47,7 +47,7 @@ From Python, the arguments to a Julia function will be converted according to th
 | `ctypes.c_voidp`                                                                                             | `Ptr{Cvoid}`, `Ptr`                                         |
 | `ctypes.c_char_p`                                                                                            | `Cstring`, `Ptr{Cchar}`, `Ptr`                              |
 | `ctypes.c_wchar_p`                                                                                           | `Cwstring`, `Ptr{Cwchar}`, `Ptr`                            |
-| `numpy.intXX`/`numpy.uintXX`/`numpy.floatXX`                                                                 | `Integer`, `Rational`, `Real`, `Number`                     |
+| `numpy.bool_`/`numpy.intXX`/`numpy.uintXX`/`numpy.floatXX`                                                   | `Bool`, `Integer`, `Rational`, `Real`, `Number`             |
 | Objects satisfying the buffer or array interface                                                             | `Array`, `AbstractArray`                                    |
 | **Low priority (fallback to `Py`).**                                                                         |                                                             |
 | Anything                                                                                                     | `Py`                                                        |

docs/src/compat.md

@@ -1,33 +1,27 @@
 # FAQ & Troubleshooting
 
-## Is PythonCall/JuliaCall thread safe?
+## Can I use PythonCall and PyCall together?
 
-No.
+Yes, you can use both PyCall and PythonCall in the same Julia session. This is platform-dependent:
+- On most systems the Python interpreter used by PythonCall and PyCall must be the same (see below).
+- On Windows it appears to be possible for PythonCall and PyCall to use different interpreters.
 
-Some rules if you are writing multithreaded code:
-- Only call Python functions from the first thread.
-- You probably also need to call `PythonCall.GC.disable()` on the main thread before any
-  threaded block of code. Remember to call `PythonCall.GC.enable()` again afterwards.
-  (This is because Julia finalizers can be called from any thread.)
-- Julia intentionally causes segmentation faults as part of the GC safepoint mechanism.
-  If unhandled, these segfaults will result in termination of the process. To enable signal handling,
-  set `PYTHON_JULIACALL_HANDLE_SIGNALS=yes` before any calls to import juliacall. This is equivalent
-  to starting julia with `julia --handle-signals=yes`, the default behavior in Julia. 
-  See discussion [here](https://github.com/JuliaPy/PythonCall.jl/issues/219#issuecomment-1605087024) for more information.
-- You may still encounter problems.
+To force PythonCall to use the same Python interpreter as PyCall, set the environment variable [`JULIA_PYTHONCALL_EXE`](@ref pythoncall-config) to `"@PyCall"`. Note that this will opt out of automatic dependency management using CondaPkg.
 
-Related issues: [#201](https://github.com/JuliaPy/PythonCall.jl/issues/201), [#202](https://github.com/JuliaPy/PythonCall.jl/issues/202)
+Alternatively, to force PyCall to use the same interpreter as PythonCall, set the environment variable `PYTHON` to [`PythonCall.python_executable_path()`](@ref) and then `Pkg.build("PyCall")`. You will need to do this each time you change project, because PythonCall by default uses a different Python for each project.
 
-## Does it work on Apple silicon (ARM, M1, M2, ...)?
+## Is PythonCall/JuliaCall thread safe?
 
-Maybe. Your mileage may vary.
+Yes, as of v0.9.22, provided you handle the GIL correctly. See the guides for
+[PythonCall](@ref jl-multi-threading) and [JuliaCall](@ref py-multi-threading).
 
-In general, PythonCall and JuliaCall are only supported on platforms with
-[Tier 1](https://julialang.org/downloads/#supported_platforms) level of support by Julia.
-Currently, Apple silicon is Tier 2, so is not supported.
+Before, tricks such as disabling the garbage collector were required. See the
+[old docs](https://juliapy.github.io/PythonCall.jl/v0.9.21/faq/#Is-PythonCall/JuliaCall-thread-safe?).
 
-Due to time constraints, issues affecting only unsupported platforms will not be
-investigated. It is much more likely to be an issue with Julia itself than PythonCall.
+Related issues:
+[#201](https://github.com/JuliaPy/PythonCall.jl/issues/201),
+[#202](https://github.com/JuliaPy/PythonCall.jl/issues/202),
+[#529](https://github.com/JuliaPy/PythonCall.jl/pull/529)
 
 ## Issues when Numpy arrays are expected
 
@@ -92,4 +86,6 @@ Related issues: [#255](https://github.com/JuliaPy/PythonCall.jl/issues/255)
 
 ## Can I use JuliaCall to run Julia inside applications with embedded Python?
 
-Yes, it may be possible. See an example of how to have Julia running inside the Python that is running inside Blender here https://discourse.julialang.org/t/running-julia-inside-blender-through-vscode-using-pythoncall-juliacall/96838.
+Yes, it may be possible. A good example of that is having Julia running inside the Python that is running inside Blender, as presented in [this Discourse post](https://discourse.julialang.org/t/running-julia-inside-blender-through-vscode-using-pythoncall-juliacall/96838/6).
+From the point that one has JuliaCall running inside Python, if it has access to the terminal, one can even launch a Julia REPL there, and if needed connect with VSCode Julia extension to it.
+The full Python script to install, launch JuliaCall, and launch a Julia REPL in Blender is [here](https://gist.github.com/cdsousa/d820d27174238c0d48e5252355584172).

docs/src/conversion-to-julia.md

@@ -7,4 +7,4 @@ Bringing [**Python®**](https://www.python.org/) and [**Julia**](https://juliala
 - Fast non-copying conversion of numeric arrays in either direction: modify Python arrays (e.g. `bytes`, `array.array`, `numpy.ndarray`) from Julia or Julia arrays from Python.
 - Helpful wrappers: interpret Python sequences, dictionaries, arrays, dataframes and IO streams as their Julia counterparts, and vice versa.
 - Beautiful stack-traces.
-- Works anywhere: tested on Windows, MacOS and Linux, 32- and 64-bit, Julia Julia 1.6.1 upwards and Python 3.7 upwards.
+- Works anywhere: tested on Windows, MacOS and Linux, 32- and 64-bit, Julia Julia 1.6.1 upwards and Python 3.8 upwards.