Merge branch 'main' into gc

Author: Christopher Doris

Project.toml

@@ -1,7 +1,7 @@
 name = "PythonCall"
 uuid = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 authors = ["Christopher Doris <github.com/cjdoris>"]
-version = "0.9.3"
+version = "0.9.4"
 
 [deps]
 CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
@@ -16,7 +16,7 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
 
 [compat]
-CondaPkg = "0.2.11"
+CondaPkg = "0.2.12"
 MacroTools = "0.5"
 Requires = "1"
 Tables = "1"

docs/src/index.md

@@ -5,6 +5,6 @@ Bringing [**Python®**](https://www.python.org/) and [**Julia**](https://juliala
 - Simple syntax, so the Python code looks like Python and the Julia code looks like Julia.
 - Intuitive and flexible conversions between Julia and Python: anything can be converted, you are in control.
 - Fast non-copying conversion of numeric arrays in either direction: modify Python arrays (e.g. `bytes`, `array.array`, `numpy.ndarray`) from Julia or Julia arrays from Python.
-- Helpful wrappers: interpret Python sequences, dictionaries, arrays, dataframes and IO streams as their Julia couterparts, and vice versa.
+- Helpful wrappers: interpret Python sequences, dictionaries, arrays, dataframes and IO streams as their Julia counterparts, and vice versa.
 - Beautiful stack-traces.
 - Works anywhere: tested on Windows, MacOS and Linux, 32- and 64-bit, Julia Julia 1.6.1 upwards and Python 3.7 upwards.

docs/src/releasenotes.md

@@ -1,6 +1,15 @@
 # Release Notes
 
 ## Unreleased
+* Experimental new function `juliacall.interactive()` allows the Julia async event loop to
+  run in the background of the Python REPL.
+* Experimental new IPython extension `juliacall.ipython` providing the `%jl` and `%%jl`
+  magics for executing Julia code.
+* Experimental new module `juliacall.importer` allowing you to write Python modules in
+  Julia.
+* Bug fixes.
+
+## 0.9.4 (2022-07-26)
 * Bug fixes.
 
 ## 0.9.3 (2022-07-02)

pysrc/juliacall/__init__.py

@@ -1,7 +1,7 @@
 # This module gets modified by PythonCall when it is loaded, e.g. to include Core, Base
 # and Main modules.
 
-__version__ = '0.9.3'
+__version__ = '0.9.4'
 
 _newmodule = None
 
@@ -21,6 +21,13 @@ def convert(T, x):
         _convert = PythonCall.seval("pyjlcallback((T,x)->pyjl(pyconvert(pyjlvalue(T)::Type,x)))")
     return _convert(T, x)
 
+def interactive(enable=True):
+    "Allow the Julia event loop to run in the background of the Python REPL."
+    if enable:
+        PythonCall._set_python_input_hook()
+    else:
+        PythonCall._unset_python_input_hook()
+
 class JuliaError(Exception):
     "An error arising in Julia code."
     def __init__(self, exception, backtrace=None):
@@ -122,6 +129,7 @@ def args_from_config():
     CONFIG['opt_sysimage'] = sysimg = path_option('sysimage', check_exists=True)[0]
     CONFIG['opt_threads'] = int_option('threads', accept_auto=True)[0]
     CONFIG['opt_warn_overwrite'] = choice('warn_overwrite', ['yes', 'no'])[0]
+    CONFIG['opt_handle_signals'] = 'no'
 
     # Stop if we already initialised
     if CONFIG['inited']:
@@ -178,6 +186,7 @@ def jlstr(x):
             return 'raw"' + x.replace('"', '\\"').replace('\\', '\\\\') + '"'
         script = '''
         try
+            Base.require(Main, :CompilerSupportLibraries_jll)
             import Pkg
             ENV["JULIA_PYTHONCALL_LIBPTR"] = {}
             ENV["JULIA_PYTHONCALL_EXE"] = {}

pysrc/juliacall/importer.py

@@ -0,0 +1,70 @@
+import base64
+import io
+import os
+import sys
+
+from . import newmodule, Base
+from importlib.machinery import ModuleSpec, SourceFileLoader
+
+class Finder:
+    def __init__(self, jlext='.jl', pyext='.py'):
+        self.jlext = jlext
+        self.pyext = pyext
+
+    def find_spec(self, fullname, path, target=None):
+        if path is None:
+            path = sys.path
+            if '.' in fullname:
+                return
+            name = fullname
+        else:
+            name = fullname.split('.')[-1]
+        for root in path:
+            jlfile = os.path.join(root, name + self.jlext)
+            if os.path.isfile(jlfile):
+                jlfile = os.path.realpath(jlfile)
+                pyfile = os.path.join(root, name + self.pyext)
+                gen_file(jlfile, pyfile)
+                return ModuleSpec(fullname, SourceFileLoader(fullname, pyfile), origin=jlfile)
+
+def install(**kw):
+    finder = Finder(**kw)
+    sys.meta_path.insert(0, finder)
+    return finder
+
+def uninstall(finder):
+    sys.meta_path.remove(finder)
+
+def gen_code(jl):
+    buf = io.StringIO()
+    pr = lambda x: print(x, file=buf)
+    jl2 = jl.replace('\\', '\\\\').replace("'", "\\'")
+    pr('# This file was automatically generated by juliacall.importer')
+    pr('import juliacall.importer')
+    pr('juliacall.importer.exec_module(__name__,')
+    pr("'''"+jl2+"''')")
+    return buf.getvalue()
+
+def gen_file(jl, py):
+    with open(jl, encoding='utf-8') as fp:
+        jlcode = fp.read()
+    pycode = gen_code(jlcode)
+    with open(py, 'w', encoding='utf-8') as fp:
+        fp.write(pycode)
+
+def exec_module(name, code):
+    pymod = sys.modules[name]
+    jlmod = newmodule(name)
+    jlmod.seval('begin\n' + code + '\nend')
+    delattr(pymod, 'juliacall')
+    setattr(pymod, '__jl_code__', code)
+    setattr(pymod, '__jl_module__', jlmod)
+    ks = [str(k) for k in Base.names(jlmod)]
+    ks = [k for k in ks if k != name]
+    if not ks:
+        ks = [str(k) for k in Base.names(jlmod, all=True)]
+        ks = [k for k in ks if not (k == name or k == 'include' or k == 'eval' or k.startswith('_') or '#' in k)]
+    setattr(pymod, '__all__', ks)
+    setattr(pymod, '__doc__', str(Base.Docs.doc(jlmod)))
+    for k in ks:
+        setattr(pymod, k, getattr(jlmod, k))

pysrc/juliacall/ipython.py

@@ -0,0 +1,27 @@
+from IPython.core.magic import Magics, magics_class, line_cell_magic
+from . import Main, Base, PythonCall
+
+@magics_class
+class JuliaMagics(Magics):
+    
+    @line_cell_magic
+    def julia(self, line, cell=None):
+        code = line if cell is None else cell
+        ans = Main.seval('begin\n' + code + '\nend')
+        Base.flush(Base.stdout)
+        Base.flush(Base.stderr)
+        if not code.strip().endswith(';'):
+            return ans
+
+def load_ipython_extension(ip):
+    # register magics
+    ip.register_magics(JuliaMagics(ip))
+    # redirect stdout/stderr
+    PythonCall.seval("""begin
+        const _redirected_stdout = redirect_stdout()
+        const _redirected_stderr = redirect_stderr()
+        const _py_stdout = PyIO(pyimport("sys" => "stdout"), buflen=1)
+        const _py_stderr = PyIO(pyimport("sys" => "stderr"), buflen=1)
+        const _redirect_stdout_task = @async write($_py_stdout, $_redirected_stdout)
+        const _redirect_stderr_task = @async write($_py_stderr, $_redirected_stderr)
+    end""")

pysrc/juliacall/juliapkg-dev.json

@@ -3,7 +3,7 @@
     "packages": {
         "PythonCall": {
             "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d",
-            "version": "0.9.3",
+            "version": "=0.9.4",
             "path": "../..",
             "dev": true
         }

pysrc/juliacall/juliapkg.json

@@ -3,7 +3,7 @@
     "packages": {
         "PythonCall": {
             "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d",
-            "version": "0.9.3"
+            "version": "=0.9.4"
         }
     }
 }

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = juliacall
-version = 0.9.3
+version = 0.9.4
 description = Julia and Python in seamless harmony
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -15,9 +15,9 @@ zip_safe = False
 package_dir =
     =pysrc
 packages = juliacall
-python_requires = ~=3.5
+python_requires = ~=3.7
 install_requires =
-    juliapkg ~=0.1.0
+    juliapkg ~=0.1.8
 
 [options.package_data]
 juliacall =

src/Py.jl

@@ -445,7 +445,8 @@ Base.powermod(x::Number, y::Py, z::Number) = pypow(x, y, z)
 Base.powermod(x::Py, y::Number, z::Number) = pypow(x, y, z)
 
 # documentation
-function Base.Docs.getdoc(x::Py, @nospecialize(sig))
+function Base.Docs.getdoc(x::Py, @nospecialize(sig)=Union{})
+    pyisnull(x) && return nothing
     parts = []
     inspect = pyimport("inspect")
     # head line

src/PythonCall.jl

@@ -1,6 +1,6 @@
 module PythonCall
 
-const VERSION = v"0.9.3"
+const VERSION = v"0.9.4"
 const ROOT_DIR = dirname(@__DIR__)
 
 using Base: @propagate_inbounds

src/compat/gui.jl

@@ -165,3 +165,36 @@ function event_loop_on(g::Symbol; interval::Real = 0.04, fix::Bool = false)
     callback = new_event_loop_callback(string(g), Float64(interval))
     EVENT_LOOPS[g] = Timer(t -> callback(), 0; interval = interval)
 end
+
+function _python_input_hook()
+    try
+        @static if Sys.iswindows()
+            # on windows, we can call yield in a loop because _kbhit() lets us know
+            # when to stop
+            while true
+                yield()
+                if ccall(:_kbhit, Cint, ()) != 0
+                    break
+                end
+                sleep(0.01)
+            end
+        else
+            # on other platforms, if readline is enabled, the input hook is called
+            # repeatedly so the loop is not required
+            yield()
+        end
+    catch
+        return Cint(1)
+    end
+    return Cint(0)
+end
+
+function _set_python_input_hook()
+    C.PyOS_SetInputHook(@cfunction(_python_input_hook, Cint, ()))
+    return
+end
+
+function _unset_python_input_hook()
+    C.PyOS_SetInputHook(C_NULL)
+    return
+end

src/cpython/context.jl

@@ -52,6 +52,26 @@ function init_context()
         # Find Python executable
         exe_path = get(ENV, "JULIA_PYTHONCALL_EXE", "")
         if exe_path == "" || exe_path == "@CondaPkg"
+            if Sys.islinux()
+                # Ensure libstdc++ in the Conda environment is compatible with the one
+                # linked in Julia. This is platform/version dependent, so needs to occur at
+                # runtime.
+                #
+                # To figure out cxx_version for a given Julia version, run
+                #   strings /path/to/julia/lib/julia/libstdc++.so.6 | grep GLIBCXX
+                # then look at
+                #   https://gcc.gnu.org/onlinedocs/gcc-12.1.0/libstdc++/manual/manual/abi.html
+                # for the highest GCC version compatible with the highest GLIBCXX version.
+                if Base.VERSION <= v"1.6.2"
+                    # GLIBCXX_3.4.26
+                    cxx_version = ">=3.4,<9.2"
+                else
+                    # GLIBCXX_3.4.29
+                    # checked up to v1.8.0
+                    cxx_version = ">=3.4,<11.4"
+                end
+                CondaPkg.add("libstdcxx-ng", version=cxx_version, channel="conda-forge", temp=true, file=joinpath(@__DIR__, "..", "..", "CondaPkg.toml"), resolve=false)
+            end
             # By default, we use Python installed by CondaPkg.
             exe_path = Sys.iswindows() ? joinpath(CondaPkg.envdir(), "python.exe") : joinpath(CondaPkg.envdir(), "bin", "python")
             # It's not sufficient to only activate the env while Python is initialising,

src/cpython/extras.jl

@@ -41,8 +41,17 @@ PyBuffer_Release(_b) = begin
     return
 end
 
+function PyOS_SetInputHook(hook::Ptr{Cvoid})
+    Base.unsafe_store!(POINTERS.PyOS_InputHookPtr, hook)
+    return
+end
+
+function PyOS_GetInputHook()
+    return Base.unsafe_load(POINTERS.PyOS_InputHookPtr)
+end
+
 function PyOS_RunInputHook()
-    hook = Base.unsafe_load(Ptr{Ptr{Cvoid}}(dlsym(CTX.lib_ptr, :PyOS_InputHook)))
+    hook = PyOS_GetInputHook()
     if hook == C_NULL
         return false
     else

src/cpython/pointers.jl

@@ -273,6 +273,7 @@ const CAPI_OBJECTS = Set([
     $([:($name :: Ptr{Cvoid} = C_NULL) for name in CAPI_FUNCS]...)
     $([:($name :: PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...)
     $([:($name :: PyPtr = C_NULL) for name in CAPI_OBJECTS]...)
+    PyOS_InputHookPtr :: Ptr{Ptr{Cvoid}} = C_NULL
     PyJuliaBase_Type :: PyPtr = C_NULL
     PyExc_JuliaError :: PyPtr = C_NULL
 end
@@ -290,6 +291,7 @@ const POINTERS = CAPIPointers()
     ]...)
     $([:(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))))) for name in CAPI_EXCEPTIONS]...)
     $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...)
+    p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook)
 end
 
 for (name, (argtypes, rettype)) in CAPI_FUNC_SIGS

src/jlwrap/callback.jl

@@ -10,14 +10,19 @@ function pyjlcallback_call(self, args_::Py, kwargs_::Py)
         args = pyconvert(Vector{Py}, args_)
         kwargs = pyconvert(Dict{Symbol,Py}, kwargs_)
         Py(self(args...; kwargs...))
-    elseif pylen(args_) > 0
+    elseif (nargs = pylen(args_)) > 0
         args = pyconvert(Vector{Py}, args_)
-        if length(args) == 1
+        @assert length(args) == nargs
+        if nargs == 1
             Py(self(args[1]))
-        elseif length(args) == 2
+        elseif nargs == 2
             Py(self(args[1], args[2]))
-        elseif length(args) == 3
+        elseif nargs == 3
             Py(self(args[1], args[2], args[3]))
+        elseif nargs == 4
+            Py(self(args[1], args[2], args[3], args[4]))
+        elseif nargs == 5
+            Py(self(args[1], args[2], args[3], args[4], args[5]))
         else
             Py(self(args...))
         end
@@ -53,7 +58,7 @@ end
 pyjlcallback(f) = pyjl(pyjlcallbacktype, f)
 
 """
-    pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing)
+    pyfunc(f; [name], [qualname], [doc], [signature])
 
 Wrap the callable `f` as an ordinary Python function.
 
@@ -63,10 +68,18 @@ The name, qualname, docstring or signature can optionally be set with `name`, `q
 Unlike `Py(f)` (or `pyjl(f)`), the arguments passed to `f` are always of type `Py`, i.e.
 they are never converted.
 """
-function pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing)
+function pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing, wrap=pywrapcallback)
     f2 = ispy(f) ? f : pyjlcallback(f)
-    f3 = pywrapcallback(f2)
-    pydel!(f2)
+    if wrap isa Pair
+        wrapargs, wrapfunc = wrap
+    else
+        wrapargs, wrapfunc = (), wrap
+    end
+    if wrapfunc isa AbstractString
+        f3 = pybuiltins.eval(wrapfunc, pydict())(f2, wrapargs...)
+    else
+        f3 = wrapfunc(f2, wrapargs...)
+    end
     f3.__name__ = name === nothing ? "<lambda>" : name
     f3.__qualname__ = name === nothing ? "<lambda>" : qualname
     if doc !== nothing