Big rewrite

juliapy/julia/__init__.py

@@ -1,6 +1,6 @@
-_CONFIG = dict()
+CONFIG = dict()
 
-def _init_():
+def init():
     import os, os.path, sys, ctypes as c, types, shutil, subprocess
     libpath = os.environ.get('JULIAPY_LIB')
     if libpath is None:
@@ -12,52 +12,47 @@ def _init_():
         else:
             if not os.path.isfile(exepath):
                 raise Exception('JULIAPY_EXE=%s does not exist' % repr(exepath))
-        _CONFIG['exepath'] = exepath
+        CONFIG['exepath'] = exepath
         libpath = subprocess.run([exepath, '-e', 'import Libdl; print(abspath(Libdl.dlpath("libjulia")))'], stdout=(subprocess.PIPE)).stdout.decode('utf8')
     else:
         if not os.path.isfile(libpath):
             raise Exception('JULIAPY_LIB=%s does not exist' % repr(libpath))
-    _CONFIG['libpath'] = libpath
+    CONFIG['libpath'] = libpath
     try:
         d = os.getcwd()
         os.chdir(os.path.dirname(libpath))
         lib = c.CDLL(libpath)
     finally:
         os.chdir(d)
 
-    _CONFIG['lib'] = lib
+    CONFIG['lib'] = lib
     lib.jl_init__threading.argtypes = []
     lib.jl_init__threading.restype = None
     lib.jl_init__threading()
     lib.jl_eval_string.argtypes = [c.c_char_p]
     lib.jl_eval_string.restype = c.c_void_p
     res = lib.jl_eval_string(
         '''
-        ENV["PYTHONJL_LIBPTR"] = "{}"
-        import Python
-        Python.with_gil() do
-            Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main)
+        try
+            ENV["PYTHONJL_LIBPTR"] = "{}"
+            import Python
+            Python.with_gil() do
+                Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main)
+            end
+        catch err
+            @error "Error loading Python.jl" err=err
+            rethrow()
         end
         '''.format(c.pythonapi._handle).encode('utf8'))
     if res is None:
         raise Exception('Python.jl did not start properly. Ensure that the Python package is installed in Julia.')
 
-    class Wrapper(types.ModuleType):
-
-        def __getattr__(self, k):
-            return getattr(self.Main, k)
-
-        def __dir__(self):
-            return super().__dir__() + self.Main.__dir__()
-
-    sys.modules['julia'].__class__ = Wrapper
-
-_init_()
-del _init_
+init()
+del init
 
 Core = Main.Core
 Base = Main.Base
 Python = Main.Python
 
-def _import(*names):
-    Main.eval(Base.Meta.parse('import ' + ', '.join(names)))
+def newmodule(name):
+    return Base.Module(Base.Symbol(name))

src/PyArray.jl

@@ -3,101 +3,147 @@
 
 Interpret the Python array `o` as a Julia array.
 
-Type parameters which are not given or set to `missing` are inferred:
+The input may be anything supporting the buffer protocol or the numpy array interface.
+This includes, `bytes`, `bytearray`, `array.array`, `numpy.ndarray`, `pandas.Series`.
+
 - `T` is the (Julia) element type.
 - `N` is the number of dimensions.
 - `R` is the type of elements of the underlying buffer (which may be different from `T` to allow some basic conversion).
 - `M` is true if the array is mutable.
 - `L` is true if the array supports fast linear indexing.
 """
 mutable struct PyArray{T,N,R,M,L} <: AbstractArray{T,N}
-    o :: PyObject
+    ref :: PyRef
     ptr :: Ptr{R}
     size :: NTuple{N,Int}
     length :: Int
     bytestrides :: NTuple{N,Int}
     handle :: Any
 end
-
 const PyVector{T,R,M,L} = PyArray{T,1,R,M,L}
 const PyMatrix{T,R,M,L} = PyArray{T,2,R,M,L}
 export PyArray, PyVector, PyMatrix
 
-function PyArray{T,N,R,M,L}(o::PyObject, info=pyarray_info(o)) where {T,N,R,M,L}
-    # R - buffer element type
-    if R === missing
-        return PyArray{T, N, info.eltype, M, L}(o, info)
-    elseif R isa Type
-        Base.allocatedinline(R) || error("source must be allocated inline, got R=$R")
-        Base.aligned_sizeof(R) == info.elsize || error("source elements must have size $(info.elsize), got R=$R")
-    else
-        error("R must be missing or a type")
-    end
+ispyreftype(::Type{<:PyArray}) = true
+pyptr(x::PyArray) = pyptr(x.ref)
+Base.unsafe_convert(::Type{CPyPtr}, x::PyArray) = pyptr(x.ref)
+C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyArray} = CTryConvertRule_trywrapref(o, T)
 
+function PyArray{T,N,R,M,L}(o::PyRef, info) where {T,N,R,M,L}
     # T - array element type
-    if T === missing
-        return PyArray{pyarray_default_T(R), N, R, M, L}(o, info)
-    elseif T isa Type
-        # great
-    else
-        error("T must be missing or a type")
-    end
+    T isa Type || error("T must be a type, got T=$T")
 
-    # N
-    if N === missing
-        return PyArray{T, Int(info.ndims), R, M, L}(o, info)
-    elseif N isa Int
-        N == info.ndims || error("source dimension is $(info.ndims), got N=$N")
-        # great
-    elseif N isa Integer
-        return PyArray{T, Int(N), R, M, L}(o, info)
-    else
-        error("N must be missing or an integer")
-    end
+    # N - number of dimensions
+    N isa Integer || error("N must be an integer, got N=$N")
+    N isa Int || return PyArray{T, Int(N), R, M, L}(o, info)
+    N == info.ndims || error("source dimension is $(info.ndims), got N=$N")
 
-    # M
-    if M === missing
-        return PyArray{T, N, R, Bool(info.mutable), L}(o, info)
-    elseif M === true
-        info.mutable || error("source is immutable, got L=$L")
-    elseif M === false
-        # great
-    else
-        error("M must be missing, true or false")
-    end
+    # R - buffer element type
+    R isa Type || error("R must be a type, got R=$R")
+    Base.allocatedinline(R) || error("source elements must be allocated inline, got R=$R")
+    Base.aligned_sizeof(R) == info.elsize || error("source elements must have size $(info.elsize), got R=$R")
 
-    bytestrides = NTuple{N, Int}(info.bytestrides)
-    size = NTuple{N, Int}(info.size)
+    # M - mutable
+    M isa Bool || error("M must be true or false, got M=$M")
+    !M || info.mutable || error("source is immutable, got M=$M")
 
-    # L
-    if L === missing
-        return PyArray{T, N, R, M, N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides}(o, info)
-    elseif L === true
-        N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides || error("not linearly indexable")
-    elseif L === false
-        # great
-    else
-        error("L must be missing, true or false")
-    end
+    bytestrides = info.bytestrides
+    size = info.size
+
+    # L - linear indexable
+    L isa Bool || error("L must be true or false, got L=$L")
+    !L || N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides || error("not linearly indexable, got L=$L")
 
-    PyArray{T, N, R, M, L}(o, Ptr{R}(info.ptr), size, N==0 ? 1 : prod(size), bytestrides, info.handle)
+    PyArray{T, N, R, M, L}(PyRef(o), Ptr{R}(info.ptr), size, N==0 ? 1 : prod(size), bytestrides, info.handle)
 end
-PyArray{T,N,R,M}(o) where {T,N,R,M} = PyArray{T,N,R,M,missing}(o)
-PyArray{T,N,R}(o) where {T,N,R} = PyArray{T,N,R,missing}(o)
-PyArray{T,N}(o) where {T,N} = PyArray{T,N,missing}(o)
-PyArray{T}(o) where {T} = PyArray{T,missing}(o)
-PyArray(o) where {} = PyArray{missing}(o)
-
-pyobject(x::PyArray) = x.o
-
-function pyarray_info(o::PyObject)
-    # TODO: support the numpy array interface too
-    b = PyBuffer(o, C.PyBUF_RECORDS_RO)
-    (ndims=b.ndim, eltype=b.eltype, elsize=b.itemsize, mutable=!b.readonly, bytestrides=b.strides, size=b.shape, ptr=b.buf, handle=b)
+PyArray{T,N,R,M}(o::PyRef, info) where {T,N,R,M} = PyArray{T,N,R,M, N≤1 || size_to_fstrides(info.bytestrides[1], info.size...) == info.bytestrides}(o, info)
+PyArray{T,N,R}(o::PyRef, info) where {T,N,R} = PyArray{T,N,R, info.mutable}(o, info)
+PyArray{T,N}(o::PyRef, info) where {T,N} = PyArray{T,N, info.eltype}(o, info)
+PyArray{T}(o::PyRef, info) where {T} = PyArray{T, info.ndims}(o, info)
+PyArray{<:Any,N}(o::PyRef, info) where {N} = PyArray{pyarray_default_T(info.eltype), N}(o, info)
+PyArray(o::PyRef, info) = PyArray{pyarray_default_T(info.eltype)}(o, info)
+
+(::Type{A})(o; opts...) where {A<:PyArray} = begin
+    ref = PyRef(o)
+    info = pyarray_info(ref; opts...)
+    info = (
+        ndims = Int(info.ndims),
+        eltype = info.eltype :: Type,
+        elsize = Int(info.elsize),
+        mutable = info.mutable :: Bool,
+        bytestrides = NTuple{Int(info.ndims), Int}(info.bytestrides),
+        size = NTuple{Int(info.ndims), Int}(info.size),
+        ptr = Ptr{Cvoid}(info.ptr),
+        handle = info.handle,
+    )
+    A(ref, info)
+end
+
+function pyarray_info(ref; buffer=true, array=true, copy=true)
+    if array && pyhasattr(ref, "__array_interface__")
+        pyconvertdescr(x) = begin
+            @py ```
+            def convert(x):
+                def fix(x):
+                    a = x[0]
+                    a = (a, a) if isinstance(a, str) else (a[0], a[1])
+                    b = x[1]
+                    c = x[2] if len(x)>2 else 1
+                    return (a, b, c)
+                if x is None or isinstance(x, str):
+                    return x
+                else:
+                    return [fix(y) for y in x]
+            $(r::Union{Nothing,String,Vector{Tuple{Tuple{String,String}, PyObject, Int}}}) = convert($x)
+            ```
+            r isa Vector ? [(a, pyconvertdescr(b), c) for (a,b,c) in r] : r
+        end
+        ai = pygetattr(ref, "__array_interface__")
+        pyconvert(Int, ai["version"]) == 3 || error("wrong version")
+        size = pyconvert(Tuple{Vararg{Int}}, ai["shape"])
+        ndims = length(size)
+        typestr = pyconvert(String, ai["typestr"])
+        descr = pyconvertdescr(ai.get("descr"))
+        eltype = pytypestrdescr_to_type(typestr, descr)
+        elsize = Base.aligned_sizeof(eltype)
+        strides = pyconvert(Union{Nothing, Tuple{Vararg{Int}}}, ai.get("strides"))
+        strides === nothing && (strides = size_to_cstrides(elsize, size...))
+        pyis(ai.get("mask"), pynone()) || error("mask not supported")
+        offset = pyconvert(Union{Nothing, Int}, ai.get("offset"))
+        offset === nothing && (offset = 0)
+        data = pyconvert(Union{PyObject, Tuple{UInt, Bool}, Nothing}, ai.get("data"))
+        if data isa Tuple
+            ptr = Ptr{Cvoid}(data[1])
+            mutable = !data[2]
+            handle = (ref, ai)
+        else
+            buf = PyBuffer(data === nothing ? ref : data)
+            ptr = buf.buf
+            mutable = !buf.readonly
+            handle = (ref, ai, buf)
+        end
+        return (ndims=ndims, eltype=eltype, elsize=elsize, mutable=mutable, bytestrides=strides, size=size, ptr=ptr, handle=handle)
+    end
+    if array && pyhasattr(ref, "__array_struct__")
+        # TODO
+    end
+    if buffer && C.PyObject_CheckBuffer(ref)
+        try
+            b = PyBuffer(ref, C.PyBUF_RECORDS_RO)
+            return (ndims=b.ndim, eltype=b.eltype, elsize=b.itemsize, mutable=!b.readonly, bytestrides=b.strides, size=b.shape, ptr=b.buf, handle=b)
+        catch
+        end
+    end
+    if array && copy && pyhasattr(ref, "__array__")
+        try
+            return pyarray_info(pycall(PyRef, pygetattr(PyRef, ref, "__array__")); buffer=buffer, array=array, copy=false)
+        catch
+        end
+    end
+    error("given object does not support the buffer protocol or array interface")
 end
 
 Base.isimmutable(x::PyArray{T,N,R,M,L}) where {T,N,R,M,L} = !M
-Base.pointer(x::PyArray{T,N,T}) where {T,N} = x.ptr
 Base.size(x::PyArray) = x.size
 Base.length(x::PyArray) = x.length
 Base.IndexStyle(::Type{PyArray{T,N,R,M,L}}) where {T,N,R,M,L} = L ? Base.IndexLinear() : Base.IndexCartesian()
@@ -120,13 +166,23 @@ Base.@propagate_inbounds Base.setindex!(x::PyArray{T,N,R,true,L}, v, i::Vararg{I
     end
 
 pyarray_default_T(::Type{R}) where {R} = R
-pyarray_default_T(::Type{CPyObjRef}) = PyObject
+pyarray_default_T(::Type{C.PyObjectRef}) = PyObject
 
 pyarray_load(::Type{T}, p::Ptr{T}) where {T} = unsafe_load(p)
-pyarray_load(::Type{T}, p::Ptr{CPyObjRef}) where {T} = (o=unsafe_load(p).ptr; o==C_NULL ? throw(UndefRefError()) : pyconvert(T, pyborrowedobject(o)))
+pyarray_load(::Type{T}, p::Ptr{C.PyObjectRef}) where {T} = begin
+    o = unsafe_load(p).ptr
+    isnull(o) && throw(UndefRefError())
+    ism1(C.PyObject_Convert(o, T)) && pythrow()
+    takeresult(T)
+end
 
 pyarray_store!(p::Ptr{T}, v::T) where {T} = unsafe_store!(p, v)
-pyarray_store!(p::Ptr{CPyObjRef}, v::T) where {T} = (C.Py_DecRef(unsafe_load(p).ptr); unsafe_store!(p, CPyObjRef(pyptr(pyincref!(pyobject(v))))))
+pyarray_store!(p::Ptr{C.PyObjectRef}, v::T) where {T} = begin
+    o = C.PyObject_From(v)
+    isnull(o) && pythrow()
+    C.Py_DecRef(unsafe_load(p).ptr)
+    unsafe_store!(p, C.PyObjectRef(o))
+end
 
 pyarray_offset(x::PyArray{T,N,R,M,true}, i::Int) where {T,N,R,M} =
     N==0 ? 0 : (i-1) * x.bytestrides[1]

src/PyBuffer.jl

@@ -1,5 +1,5 @@
 """
-    PyBuffer(o)
+    PyBuffer(o, [flags=C.PyBUF_FULL_RO])
 
 A reference to the underlying buffer of `o`, if it satisfies the buffer protocol.
 
@@ -20,16 +20,15 @@ Has the following properties:
 """
 mutable struct PyBuffer
     info :: Array{C.Py_buffer, 0}
-    function PyBuffer(o::PyObject, flags::Integer=C.PyBUF_FULL_RO)
+    function PyBuffer(o, flags::Integer=C.PyBUF_FULL_RO)
         info = fill(C.Py_buffer())
         check(C.PyObject_GetBuffer(o, pointer(info), flags))
         b = new(info)
         finalizer(b) do b
             if CONFIG.isinitialized
-                err = with_gil() do
+                with_gil(false) do
                     C.PyBuffer_Release(pointer(b.info))
                 end
-                check(err)
             end
         end
         b

src/PyCode.jl

@@ -0,0 +1,21 @@
+mutable struct PyCode
+    ref :: PyRef
+    code :: String
+    filename :: String
+    mode :: Symbol
+    PyCode(code::String, filename::String, mode::Symbol) = begin
+        mode in (:exec, :eval) || error("invalid mode $(repr(mode))")
+        new(PyRef(), code, filename, mode)
+    end
+end
+export PyCode
+
+ispyreftype(::Type{PyCode}) = true
+pyptr(co::PyCode) = begin
+    ptr = co.ref.ptr
+    if isnull(ptr)
+        ptr = co.ref.ptr = C.Py_CompileString(co.code, co.filename, co.mode == :exec ? C.Py_file_input : co.mode == :eval ? C.Py_eval_input : error("invalid mode $(repr(co.mode))"))
+    end
+    ptr
+end
+Base.unsafe_convert(::Type{CPyPtr}, x::PyCode) = checknull(pyptr(x))