diff --git a/AGENTS.md b/AGENTS.md index b194a6eb..643cdb55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,10 @@ the process uses its newer `libstdc++`. - `julia --project=docs docs/make.jl` requires a valid Git `origin` so Documenter can infer source links; it fails during `makedocs` in checkouts without one. +- `Documenter.HTML` discovers its default edit-link branch with `git remote show + origin`, which contacts the remote. Set `edit_link` explicitly for a local build + without that network lookup; v1's docs use `edit_link="v1"`. Disabling source + remotes alone does not prevent the HTML constructor's branch lookup. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. @@ -25,10 +29,43 @@ The majority of tests live in the Julia package; Python tests cover functionalit `main`'s `ValueBase`/`AnyValue` hierarchy. Keep the collection wrapper when porting `_jlwrap` changes. Wrapper modules load before `juliacall.Base` is assigned, so constructors must look it up through `juliacall` when called. +- For numeric Python fixtures, use `pycomplex(real, imag)` when a Python complex + value is needed. Passing a Julia `Complex` directly to a Python constructor can + produce a Julia wrapper, which NumPy may treat as a sequence rather than a scalar. +- External-interpreter Julia CI jobs do not install NumPy/pandas development + dependencies. Keep synthetic conversion tests independent of them, and guard + library-specific checks with module availability (without hiding import errors + when a module is installed). Conda-backed jobs exercise those optional checks. - The benchmark suite is shared across the compared revisions: `main` uses `pydel!`, while `v1` uses `unsafe_pydel`. - `main` does not have `@pyregion`. Benchmark region wrappers must select a plain-body fallback before macro expansion; a runtime conditional around an unavailable macro still fails when loading the suite. - Julia CI caches can preserve Conda's build metadata without its root environment. Rebuild `Conda` before building `PyCall` or running tests to recreate that directory. - In restricted sandboxes, both suites need approval to write to the shared caches (`~/.julia` and `~/.cache/uv`). +- Julia also writes `~/.julia/logs/manifest_usage.toml` during dependency discovery, + including PythonCall startup through CondaPkg. Cache permissions must include + `~/.julia/logs`, in addition to compiled packages, artifacts and registries. +- Notebook tests start a Jupyter kernel using local sockets. A sandbox that blocks + sockets can run the ordinary Python tests but cannot run `--nbval` successfully. +- Loopback TCP permission alone may be insufficient for notebook tests: ZeroMQ + queries network interfaces via `getifaddrs`, which on Linux needs + `AF_NETLINK`/`NETLINK_ROUTE` sockets. In the tested sandbox, loopback binding + succeeded but creating a route Netlink socket failed with `EPERM`. +- Use the absolute installed Julia executable in a restricted sandbox to avoid + Juliaup's configuration lock. Installed channels may differ from those listed above. +- `JULIA_PKG_OFFLINE=true` lets JuliaPkg resolve the development Python test + environment using installed packages without refreshing registries or cloning + uncached package versions. +- `JULIA_CONDAPKG_ENV` must not point at an existing `.CondaPkg` directory. + To run Aqua against an installed interpreter without creating a fresh Conda + environment, use `JULIA_CONDAPKG_BACKEND=Null` and set `JULIA_PYTHONCALL_EXE` + to that interpreter's absolute path. +- `test/runtests.jl` infers optional development dependencies from the CondaPkg + backend. With the Null backend and an explicit Conda interpreter, PySide6 may + be installed even though `Setup.devdeps` is false, causing the GUI test to + incorrectly expect an import failure. Focused conversion tests are unaffected. +- Interrupted sandbox runs can leave a CondaPkg lock containing a PID from a + different PID namespace. A reused small PID can make later runs wait for that + lock indefinitely. Prefer an explicit installed Python/Null backend for + focused conversion tests; do not remove a lock without checking its owner. - `pysrc/juliacall/juliapkg.json` is tracked; restore its original contents after using the development copy for Python tests. ## Meta instructions diff --git a/benchmark/convert-array-allocations.jl b/benchmark/convert-array-allocations.jl new file mode 100644 index 00000000..0e9489d9 --- /dev/null +++ b/benchmark/convert-array-allocations.jl @@ -0,0 +1,35 @@ +# Attribution only: run after timing trials, with the same environment/Python. +using PythonCall, Profile +array_value(::Val{T},x) where T = pyconvert(T,x,nothing) +label=isempty(ARGS) ? "after" : only(ARGS) +source_root=dirname(pathof(PythonCall)) +np=pyimport("numpy") +array=pyimport("array") +cases=[("ndarray_native",Array,np.arange(1000)), + ("ndarray_wrapper",PyArray,np.arange(1000)), + ("buffer_native",Array,array.array("d",pylist(1:1000))), + ("buffer_wrapper",PyArray,array.array("d",pylist(1:1000)))] +open(joinpath(@__DIR__,"convert-array-allocations-$label.txt"),"w") do io + println(io,"Julia: ",VERSION,"; PythonCall source: ",pathof(PythonCall)) + for (name,T,x) in cases + f=let tag=Val(T), x=x + ()->array_value(tag,x) + end + for _=1:3 + f() + end + GC.gc() + Profile.Allocs.clear() + Profile.Allocs.@profile sample_rate=1.0 f() + println(io,"\n",name) + for a in Profile.Allocs.fetch().allocs + println(io,a.type,": ",a.size," bytes") + for frame in a.stacktrace + file=string(frame.file) + if occursin(source_root,file) + println(io," ",frame) + end + end + end + end +end diff --git a/benchmark/convert-dispatch.jl b/benchmark/convert-dispatch.jl new file mode 100644 index 00000000..06721bf8 --- /dev/null +++ b/benchmark/convert-dispatch.jl @@ -0,0 +1,25 @@ +using PythonCall, BenchmarkTools +const CV=PythonCall.Convert +label = isempty(ARGS) ? "" : "-" * only(ARGS) +x=pylist([pylist(1:5),pylist(6:10)]) +scoped=pylist(1:10) +np=pyimport("numpy") +cases = [ + ("no_preference", ()->pyconvert(Any,x)), + ("one_preference", ()->pyconvert(Any,x;prefer=("builtins:list"=>PyList,))), + ("multiple_preferences", ()->pyconvert(Any,x;prefer=("builtins:dict"=>PyDict,"builtins:list"=>PyList))), + ("unmatched_preferences", ()->pyconvert(Any,x;prefer=("builtins:dict"=>PyDict,))), + ("nested_context", ()->pyconvert(PyConvertContext(;prefer=("builtins:list"=>PyList,)),Vector,x)), + ("ndarray_native", let a=np.arange(1000); ()->pyconvert(Any,a) end), + ("ndarray_wrapper", let a=np.arange(1000); ()->pyconvert(PyArray,a) end), + ("bytes_explicit", let a=pybytes(pylist([1,2,3])); ()->pyconvert(Vector{UInt8},a) end), + ("scope_filtered", ()->pyconvert(Vector{Int},scoped)), +] +open(joinpath(@__DIR__,"convert-dispatch$label.tsv"),"w") do io + println(io,"case\tns\tbytes\tallocs") + for (name,f) in cases + m=minimum(@benchmark $f() seconds=1.0 samples=5000) + println(io,"$name\t$(m.time)\t$(m.memory)\t$(m.allocs)") + end +end +# Cache misses/invalidation are measured with untimed setup in convert-expanded.jl. diff --git a/benchmark/convert-expanded.jl b/benchmark/convert-expanded.jl new file mode 100644 index 00000000..d7a30e8b --- /dev/null +++ b/benchmark/convert-expanded.jl @@ -0,0 +1,184 @@ +# Shared harness for the pre-redesign revision and current v1. +# julia --project=benchmark benchmark/convert-expanded.jl after +# Optional second argument "warm" skips unchanged cold-path experiments. +using PythonCall, BenchmarkTools, SHA +const ExpandedConvert = PythonCall.Convert +const EXPANDED_NEW_API = isdefined(PythonCall, :PyConvertContext) + +expanded_decline1(T,x) = ExpandedConvert.pyconvert_unconverted() +expanded_decline2(T,x) = ExpandedConvert.pyconvert_unconverted() +expanded_decline3(T,x) = ExpandedConvert.pyconvert_unconverted() +expanded_success(T,x) = ExpandedConvert.pyconvert_return(42) +struct ExpandedDestination{I} end +struct ExpandedScopedDecliner{I} <: Function end +(::ExpandedScopedDecliner)(T,x) = ExpandedConvert.pyconvert_unconverted() + +function expanded_register(source,T,f;scope=T) + if EXPANDED_NEW_API + pyconvert_add_rule(source,T,f;scope) + else + pyconvert_add_rule(source,T,f) + end +end +expanded_value(::Val{T},x) where T = pyconvert(T,x,nothing) +function expanded_case(name,::Type{T},x,valid) where T + tag=Val(T) + (;name, target=T, run=()->expanded_value(tag,x), valid) +end +function expanded_cache(T) + if EXPANDED_NEW_API + ExpandedConvert.pyconvert_rules_cache(typeof(PyConvertContext()),T) + else + ExpandedConvert.pyconvert_rules_cache(T) + end +end +function expanded_invalidate!() + if EXPANDED_NEW_API + ExpandedConvert.pyconvert_invalidate!() + else + foreach(empty!,values(ExpandedConvert.PYCONVERT_RULES_CACHE)) + end +end + +label=isempty(ARGS) ? "after" : first(ARGS) +warm_only=length(ARGS)>=2 && ARGS[2]=="warm" +ns=pyimport("types").ModuleType("convert_expanded_benchmark") +pyimport("sys").modules["convert_expanded_benchmark"]=ns +pyexec(""" +import collections.abc +import array +import numpy +class Success: pass +class Declines: pass +class ThreeDeclines: pass +class Scoped: pass +class Preference: pass +class VirtualMapping: pass +collections.abc.Mapping.register(VirtualMapping) +class InterfaceOnly: + def __init__(self, owner): self.owner = owner + @property + def __array_interface__(self): return self.owner.__array_interface__ +class ArrayOnly: + def __init__(self, owner): self.owner = owner + def __array__(self): return self.owner +""",ns.__dict__) +prefix="convert_expanded_benchmark:" +expanded_register(prefix*"Success",Int,expanded_success;scope=Any) +expanded_register(prefix*"Declines",Int,expanded_decline1;scope=Any) +expanded_register(prefix*"Declines",Int,expanded_success;scope=Any) +for f in (expanded_decline1,expanded_decline2,expanded_decline3,expanded_success) + expanded_register(prefix*"ThreeDeclines",Int,f;scope=Any) +end +for i=1:16 + # These rules legitimately decline for this value. Old v1 has no scope + # filtering and must try them; the redesign excludes their owned scopes. + expanded_register(prefix*"Scoped",Any,ExpandedScopedDecliner{i}();scope=ExpandedDestination{i}) +end +expanded_register(prefix*"Scoped",Int,expanded_success;scope=Any) +expanded_register(prefix*"Preference",String,expanded_decline1;scope=Any) +expanded_register(prefix*"Preference",Int,expanded_success;scope=Any) +expanded_register("collections.abc:Mapping",Int,expanded_success) + +buffer=ns.array.array("d",pylist(1:1000)) +owner=ns.numpy.arange(1000) +interface=ns.InterfaceOnly(owner) +arrayonly=ns.ArrayOnly(owner) +@assert PythonCall.C.PyType_CheckBuffer(pytype(buffer)) +@assert !any(pyhasattr(pytype(buffer),a) for a in ("__array_struct__","__array_interface__","__array__")) +@assert !PythonCall.C.PyType_CheckBuffer(pytype(interface)) +@assert !PythonCall.C.PyType_CheckBuffer(pytype(arrayonly)) +if EXPANDED_NEW_API + eligible=filter(r->r.eligible,pyconvert_rules(Array,buffer)) + @assert first(eligible).source=="" +else + eligible=filter(r->typeintersect(r.type,Array)!==Union{}, + ExpandedConvert._pyconvert_get_rules(pytype(buffer))) + @assert first(eligible).func===PythonCall.Wrap.pyconvert_rule_array +end +validint=x->x===42 +validarray=x->x isa Array && size(x)==(1000,) && eltype(x)===Int64 && first(x)==0 && last(x)==999 +validbuffer=x->x isa Array && size(x)==(1000,) && eltype(x)===Float64 && first(x)==1.0 && last(x)==1000.0 +validwrapper=x->x isa PyArray && size(x)==(1000,) && first(x)==0 && last(x)==999 +validbufferwrapper=x->x isa PyArray && size(x)==(1000,) && first(x)==1.0 && last(x)==1000.0 +cases=[ + expanded_case("single_rule",Int,ns.Success(),validint), + expanded_case("decline_then_success",Int,ns.Declines(),validint), + expanded_case("three_declines_then_success",Int,ns.ThreeDeclines(),validint), + expanded_case("sixteen_scoped_decliners",Int,ns.Scoped(),validint), + expanded_case("virtual_mapping_abc",Int,ns.VirtualMapping(),validint), + expanded_case("buffer_only_native",Array,buffer,validbuffer), + expanded_case("buffer_only_wrapper",PyArray,buffer,validbufferwrapper), + expanded_case("interface_only_native",Array,interface,validarray), + expanded_case("array_method_only_native",Array,arrayonly,validarray), + expanded_case("ndarray_native",Array,owner,validarray), + expanded_case("ndarray_wrapper",PyArray,owner,validwrapper), +] +# Verify the dispatch fixtures themselves, outside the measured regions. +for (i,expected) in enumerate(EXPANDED_NEW_API ? (1,2,4,1) : (1,2,4,17)) + case=cases[i] + # The first fixtures are distinct Python classes, each with the indicated + # number of eligible rules. This also detects accidental rule deduplication. + t=(ns.Success,ns.Declines,ns.ThreeDeclines,ns.Scoped)[i] + if EXPANDED_NEW_API + @assert count(r->r.eligible,pyconvert_rules(Int,t()))==expected case.name + else + @assert length(ExpandedConvert.pyconvert_get_rules(Int,t))==expected case.name + end +end +preference_names=("two_matching_first_success","two_matching_first_declines", + "two_matching_decline_then_ordinary") +preference_cases=if EXPANDED_NEW_API + x=ns.Preference() + source=prefix*"Preference" + [(;name=preference_names[1],run=()->pyconvert(Any,x;prefer=(source=>Int,source=>String))), + (;name=preference_names[2],run=()->pyconvert(Any,x;prefer=(source=>String,source=>Int))), + (;name=preference_names[3],run=()->pyconvert(Any,x;prefer=(source=>String,source=>Float64)))] +else + [] +end + +open(joinpath(@__DIR__,"convert-expanded-$label.tsv"),"w") do io + println(io,"case\tmode\tstatus\tmin_ns\tmedian_ns\tbytes\tallocs") + for case in cases + f=case.run + @assert case.valid(f()) case.name + cache=expanded_cache(case.target) + for mode in (warm_only ? ("warm",) : ("warm","selection_miss","invalidated")) + trial=if mode=="warm" + @benchmark $f() seconds=1.0 samples=5000 + elseif mode=="selection_miss" + @benchmark $f() setup=(empty!($cache)) seconds=0.4 samples=200 evals=1 + else + @benchmark $f() setup=(expanded_invalidate!()) seconds=0.4 samples=40 evals=1 + end + m,d=minimum(trial),median(trial) + println(io,"$(case.name)\t$mode\tmeasured\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)") + end + flush(io) + end + for case in preference_cases + f=case.run + @assert f()===42 case.name + trial=@benchmark $f() seconds=1.0 samples=5000 + m,d=minimum(trial),median(trial) + println(io,"$(case.name)\twarm\tmeasured\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)") + end + if !EXPANDED_NEW_API + for name in preference_names + println(io,"$name\twarm\tunsupported\tNA\tNA\tNA\tNA") + end + end +end +open(joinpath(@__DIR__,"convert-expanded-$label-environment.txt"),"w") do io + println(io,"Julia: ",VERSION) + println(io,"Python: ",pyconvert(String,pyimport("sys").version)) + println(io,"Python executable: ",pyconvert(String,pyimport("sys").executable)) + println(io,"NumPy: ",pyconvert(String,ns.numpy.__version__)) + println(io,"BenchmarkTools: ",pkgversion(BenchmarkTools)) + println(io,"PythonCall source: ",pathof(PythonCall)) + println(io,"New API: ",EXPANDED_NEW_API) + for file in ("Convert/pyconvert.jl","Convert/rules.jl","Wrap/PyArray.jl") + println(io,file," SHA256: ",bytes2hex(sha256(read(joinpath(dirname(pathof(PythonCall)),file))))) + end +end diff --git a/benchmark/convert-final.jl b/benchmark/convert-final.jl new file mode 100644 index 00000000..47b83dfc --- /dev/null +++ b/benchmark/convert-final.jl @@ -0,0 +1,63 @@ +# Same harness on original v1 and the final working tree. No new-API dependency. +# julia --project=benchmark benchmark/convert-final.jl after-r1 +using PythonCall, BenchmarkTools, SHA + +final_value(::Val{T}, x) where T = pyconvert(T,x,nothing) +label = isempty(ARGS) ? "after-r1" : only(ARGS) +newapi = isdefined(PythonCall,:PyConvertContext) +fixtures = [ + ("int",Int,pyint(10000),x->x===10000), + ("integer",Integer,pyint(42),x->x===42), + ("bigint",BigInt,pyint(big(2)^200),x->x==big(2)^200), + ("float",Float64,pyfloat(1.5),x->x===1.5), + ("bool",Bool,pybool(true),x->x===true), + ("string",String,pystr("hello"),x->x=="hello"), + ("none",Nothing,pybuiltins.None,isnothing), + ("failure",Int,pystr("oops"),isnothing), + ("list10",Vector,pylist(1:10),x->x==collect(1:10)), + ("list10000",Vector,pylist(1:10000),x->x==collect(1:10000)), + ("list10000_concrete",Vector{Int},pylist(1:10000),x->x==collect(1:10000)), + ("mixed_numeric",Vector,pylist([pyint(1),pyfloat(2.5)]),x->x==[1,2.5]), + ("heterogeneous",Vector,pylist([pyint(1),pystr("two"),pybuiltins.None]),x->isequal(x,Any[1,"two",nothing])), + ("nested_concrete",Vector{Vector{Int}},pylist([pylist(1:5),pylist(6:10)]),x->x==[collect(1:5),collect(6:10)]), + # Any intentionally changes from wrapping to native recursive conversion. + ("nested_any",Any,pylist([pylist(1:5),pylist(6:10)]),x->newapi ? x isa Vector{Vector{Int}} : x isa PyList), + ("dict",Dict,pydict("a"=>1,"b"=>2.5),x->x==Dict("a"=>1,"b"=>2.5)), + ("dict_concrete",Dict{String,Float64},pydict("a"=>1,"b"=>2.5),x->x==Dict("a"=>1.0,"b"=>2.5)), + ("empty_list",Vector,pylist(),x->x isa Vector && isempty(x)), + ("empty_dict",Dict,pydict(),x->x isa Dict && isempty(x)), + ("wrapper",PyList,pylist(1:10),x->x isa PyList && length(x)==10), + ("iterator",Any,pyiter(pylist(1:10)),x->x isa PyIterable), +] +open(joinpath(@__DIR__,"convert-final-$label.tsv"),"w") do io + println(io,"case\tmode\tmin_ns\tmedian_ns\tbytes\tallocs") + for (i,(name,T,x,valid)) in enumerate(fixtures) + tag=Val(T) + @assert valid(final_value(tag,x)) name + modes=i<=8 ? ("typed","runtime_target") : ("typed",) + for mode in modes + trial = if mode=="typed" + @benchmark final_value($tag,$x) seconds=1.0 samples=5000 + else + @benchmark pyconvert($T,$x,nothing) seconds=1.0 samples=5000 + end + m,d=minimum(trial),median(trial) + println(io,"$name\t$mode\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)") + end + flush(io) + end +end +open(joinpath(@__DIR__,"convert-final-$label-environment.txt"),"w") do io + println(io,"Julia: ",VERSION) + println(io,"Python: ",pyconvert(String,pyimport("sys").version)) + println(io,"Python executable: ",pyconvert(String,pyimport("sys").executable)) + println(io,"BenchmarkTools: ",pkgversion(BenchmarkTools)) + println(io,"PythonCall source: ",pathof(PythonCall)) + println(io,"New API: ",newapi) + for file in ("Convert/pyconvert.jl","Convert/rules.jl","Wrap/PyArray.jl") + println(io,file," SHA256: ",bytes2hex(sha256(read(joinpath(dirname(pathof(PythonCall)),file))))) + end + for (name,T,x,_) in fixtures + println(io,name," result type: ",typeof(final_value(Val(T),x))) + end +end diff --git a/benchmark/convert-redesign.jl b/benchmark/convert-redesign.jl new file mode 100644 index 00000000..83494f01 --- /dev/null +++ b/benchmark/convert-redesign.jl @@ -0,0 +1,24 @@ +# Run with julia --project=benchmark benchmark/convert-redesign.jl baseline|redesign +using PythonCall, BenchmarkTools +label = isempty(ARGS) ? "redesign" : ARGS[1] +const CV = PythonCall.Convert +cases = [ + ("int", Int, pyint(42)), ("integer", Integer, pyint(42)), + ("bigint", BigInt, pyint(big(2)^200)), ("float", Float64, pyfloat(1.5)), + ("bool", Bool, pybool(true)), ("string", String, pystr("hello")), + ("none", Nothing, pybuiltins.None), ("failure", Int, pystr("oops")), + ("list10", Vector, pylist(1:10)), ("list10000", Vector, pylist(1:10000)), + ("mixed", Vector, pylist([1, 2.5, "three", nothing])), + ("nested", Any, pylist([pylist(1:5), pylist(6:10)])), + ("dict", Dict, pydict("a"=>1, "b"=>2.5)), + ("wrapper", PyList, pylist(1:10)), + ("iterator", Any, pyiter(pylist(1:10))), +] +open(joinpath(@__DIR__, "convert-$label.tsv"), "w") do io + println(io, "case\tns\tbytes\tallocs") + for (name, T, x) in cases + trial = @benchmark pyconvert($T, $x, nothing) seconds=0.3 samples=1000 + m = minimum(trial) + println(io, "$name\t$(m.time)\t$(m.memory)\t$(m.allocs)") + end +end diff --git a/benchmark/convert-results.jl b/benchmark/convert-results.jl new file mode 100644 index 00000000..83822dff --- /dev/null +++ b/benchmark/convert-results.jl @@ -0,0 +1,54 @@ +# Isolate result representation from Python dispatch and iteration. +# julia --project=benchmark benchmark/convert-results.jl +using BenchmarkTools, PythonCall, InteractiveUtils +label = isempty(ARGS) ? "" : "-" * only(ARGS) +struct UnconvertedBaseline end +@noinline function old_result(::Type{T}, x, success) where T + (success ? (x::T) : UnconvertedBaseline())::Union{T,UnconvertedBaseline} +end +@noinline function new_result(::Type{T}, x, success) where T + (success ? Some(x::T) : nothing)::some_result_returntype(T) +end +# This candidate intentionally wraps every success, independently of the public API. +function some_result_bound(T) + T === Any && return Union{Nothing,Some} + T isa Union && return Union{some_result_bound(T.a),some_result_bound(T.b)} + isconcretetype(T) ? Union{Nothing,Some{T}} : Union{Nothing,Some{<:T}} +end +@generated some_result_returntype(::Type{T}) where T = QuoteNode(some_result_bound(T)) +@noinline function normalized_result(::Type{T}, x, success) where T + (success ? PythonCall.Convert.pyconvert_return(x::T) : nothing)::PythonCall.Convert.pyconvert_returntype(T) +end +@noinline escaping(x) = Ref{Any}(x) +function consume(f,::Type{T},values) where T + result = 0 + for x in values + r = f(T,x,true) + result += r isa Some ? something(r) : r + end + result +end +cases = [(Int64,1), (Float64,1.5), (String,"hello"), (Vector{Int},[1,2]), + (Integer,1), (AbstractArray,[1,2]), (Union{Int64,Nothing},1), (Any,1)] +open(joinpath(@__DIR__,"convert-results$label.tsv"),"w") do io + println(io,"type\trepresentation\tsuccess\tescaping\tns\tbytes\tallocs") + for (T,x) in cases, (name,f) in (("raw",old_result),("some",new_result),("normalized",normalized_result)), success in (true,false), escape in (true,false) + b = escape ? (@benchmark escaping($f($T,$x,$success)) seconds=0.1 samples=500) : + (@benchmark $f($T,$x,$success) seconds=0.1 samples=500) + m=minimum(b) + println(io,"$T\t$name\t$success\t$escape\t$(m.time)\t$(m.memory)\t$(m.allocs)") + end + for (name,f) in (("raw",old_result),("some",new_result),("normalized",normalized_result)) + values=collect(1:100) + m=minimum(@benchmark consume($f,Int,$values) seconds=0.1 samples=500) + println(io,"loop100\t$name\ttrue\tfalse\t$(m.time)\t$(m.memory)\t$(m.allocs)") + end +end +open(joinpath(@__DIR__,"convert-inference$label.txt"),"w") do io + for T in (Int,Integer,Union{Int,Nothing},Any) + println(io,"\npytryconvert($T, ::Py)") + inferred=IOBuffer() + code_warntype(inferred,pytryconvert,Tuple{Type{T},Py};debuginfo=:none) + println(io,rstrip(String(take!(inferred)))) + end +end diff --git a/benchmark/convert-scalars.jl b/benchmark/convert-scalars.jl new file mode 100644 index 00000000..211bfaa2 --- /dev/null +++ b/benchmark/convert-scalars.jl @@ -0,0 +1,80 @@ +# Run in the benchmark environment with an optional output label. +# Val keeps the requested type visible to the benchmark caller's inference. +using PythonCall, BenchmarkTools, InteractiveUtils, Profile +const ScalarConvert = PythonCall.Convert +scalar_convert(::Val{T}, x) where T = pyconvert(T, x, nothing) +scalar_sink(::Val{T}, x) where T = (pyconvert(T, x, nothing); nothing) +scalar_try(::Val{T}, x) where T = pytryconvert(T, x) +function scalar_plain(::Val{T}, x) where T + @pyregion ScalarConvert.pytryconvert_plain(PyConvertContext(), T, x) +end +function scalar_first(::Val{T}, x) where T + @pyregion begin + ctx = PyConvertContext() + selection = ScalarConvert.pyconvert_selection(ctx, T, PythonCall.C.Py_Type(x)) + ScalarConvert.pyconvert_first(ctx, T, x, selection) + end +end + +label = isempty(ARGS) ? "current" : first(ARGS) +fixtures = [("int", Int, pyint(42)), ("float", Float64, pyfloat(1.5)), + ("bigint", BigInt, pyint(big(2)^200)), + ("none", Nothing, pybuiltins.None)] +open(joinpath(@__DIR__, "convert-scalars-$label.tsv"), "w") do io + println(io, "case\tpath\tmin_ns\tmedian_ns\tbytes\tallocs") + for (name, T, x) in fixtures + tag = Val(T) + scalar_convert(tag, x) + dynamic = @benchmark pyconvert($T, $x, nothing) seconds=0.5 samples=2000 + m, d = minimum(dynamic), median(dynamic) + println(io, "$name\tdynamic\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)") + for (path, f) in (("typed", scalar_convert), ("discard", scalar_sink), + ("try", scalar_try), ("plain", scalar_plain), + ("first", scalar_first)) + trial = @benchmark $f($tag, $x) seconds=0.5 samples=2000 + m, d = minimum(trial), median(trial) + println(io, "$name\t$path\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)") + end + flush(io) + end +end +open(joinpath(@__DIR__, "convert-scalars-$label-inference.txt"), "w") do io + for (name, T, x) in fixtures, f in (scalar_convert, scalar_plain, scalar_first) + println(io, "\n$name: $f") + inferred = IOBuffer() + code_warntype(inferred, f, Tuple{Val{T},Py}; debuginfo=:none) + println(io, rstrip(String(take!(inferred)))) + end +end +open(joinpath(@__DIR__, "convert-scalars-$label-allocations.txt"), "w") do io + for (name, T, x) in fixtures + tag = Val(T) + scalar_convert(tag,x) + Profile.Allocs.clear() + Profile.Allocs.@profile sample_rate=1.0 scalar_convert(tag,x) + println(io, "\n$name") + for a in Profile.Allocs.fetch().allocs + println(io, "$(a.type): $(a.size) bytes") + for frame in a.stacktrace + file = string(frame.file) + if occursin("PythonCall",file) || occursin("gmp",file) || occursin("boot.jl",file) + println(io, " ", frame) + end + end + end + end +end + +# Interleave the important public calls so one unusually fast/slow short trial +# does not determine the comparison of None with Int. Include an Int outside +# Julia's small boxed-integer cache to expose otherwise hidden boxing. +open(joinpath(@__DIR__, "convert-scalars-$label-repeat.tsv"), "w") do io + println(io,"round\tcase\tmin_ns\tmedian_ns\tbytes\tallocs") + repeated=[fixtures; [("int_large",Int,pyint(10000))]] + for round=1:3, (name,T,x) in circshift(repeated,round-1) + tag=Val(T) + trial=@benchmark scalar_convert($tag,$x) seconds=0.5 samples=5000 + m,d=minimum(trial),median(trial) + println(io,"$round\t$name\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)") + end +end diff --git a/benchmark/run-convert-final.sh b/benchmark/run-convert-final.sh new file mode 100644 index 00000000..bb9fbe9d --- /dev/null +++ b/benchmark/run-convert-final.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Sequential runs: no overlapping benchmark/test processes launched by this script. +# Supply the absolute Julia executable and an environment developed against the +# original v1 revision, with BenchmarkTools installed. The installed Python +# executable/backend are inherited from the environment and shared by all runs. +set -euo pipefail +if [[ $# != 2 ]]; then + printf 'Usage: bash benchmark/run-convert-final.sh JULIA_EXECUTABLE BASELINE_ENV\n' >&2 + exit 2 +fi +julia_exe=$1 +baseline_env=$2 +repo_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +for round in 1 2 3; do + if [[ $round == 2 ]]; then + revisions=(after before) + else + revisions=(before after) + fi + for revision in "${revisions[@]}"; do + project=$baseline_env + if [[ $revision == after ]]; then + project=$repo_dir/benchmark + fi + printf 'Measuring %s round %s\n' "$revision" "$round" + "$julia_exe" --project="$project" "$repo_dir/benchmark/convert-final.jl" "$revision-r$round" + "$julia_exe" --project="$project" "$repo_dir/benchmark/convert-expanded.jl" "final-$revision-r$round" + done +done +"$julia_exe" --project="$repo_dir/benchmark" "$repo_dir/benchmark/convert-scalars.jl" final +"$julia_exe" --project="$repo_dir/benchmark" "$repo_dir/benchmark/convert-results.jl" final +"$julia_exe" --project="$repo_dir/benchmark" "$repo_dir/benchmark/convert-dispatch.jl" final +printf 'Final conversion benchmarks complete\n' diff --git a/benchmark/summarize-convert-final.py b/benchmark/summarize-convert-final.py new file mode 100644 index 00000000..779d7628 --- /dev/null +++ b/benchmark/summarize-convert-final.py @@ -0,0 +1,56 @@ +"""Aggregate three serial before/after rounds; keep per-trial TSVs as raw data.""" +import csv +import statistics +import sys +from pathlib import Path + +root = Path(__file__).parent +experiment = sys.argv[1] if len(sys.argv) > 1 else "" +common_prefix = "convert-final" + ("-" + experiment if experiment else "") +expanded_prefix = "convert-expanded-" + (experiment or "final") +fields = ("group", "case", "mode", "before_ns", "after_ns", "change_percent", + "before_p50_ns", "after_p50_ns", "before_low_ns", "before_high_ns", + "after_low_ns", "after_high_ns", "before_bytes", "before_allocs", + "after_bytes", "after_allocs") +summary = [] +for group, prefix in (("common", common_prefix), ("expanded", expanded_prefix)): + data = {} + for side in ("before", "after"): + rounds = [] + environments = [] + for round_number in (1, 2, 3): + stem = f"{prefix}-{side}-r{round_number}" + with (root / f"{stem}.tsv").open() as source: + rounds.append({(r["case"], r["mode"]): r for r in + csv.DictReader(source, delimiter="\t") + if r.get("status", "measured") == "measured"}) + environments.append((root / f"{stem}-environment.txt").read_text()) + assert all(e == environments[0] for e in environments), (group, side) + assert all(set(r) == set(rounds[0]) for r in rounds), (group, side) + data[side] = rounds + keys = list(data["before"][0]) + keys.extend(k for k in data["after"][0] if k not in keys) + for key in keys: + result = dict.fromkeys(fields, "") + result.update(group=group, case=key[0], mode=key[1]) + for side in ("before", "after"): + if key not in data[side][0]: + continue + rows = [r[key] for r in data[side]] + times = [float(r["min_ns"]) for r in rows] + result[side + "_ns"] = statistics.median(times) + result[side + "_p50_ns"] = statistics.median(float(r["median_ns"]) for r in rows) + result[side + "_low_ns"] = min(times) + result[side + "_high_ns"] = max(times) + for field in ("bytes", "allocs"): + assert len({r[field] for r in rows}) == 1, (group, side, key, field) + result[side + "_" + field] = int(rows[0][field]) + if result["before_ns"] != "" and result["after_ns"] != "": + result["change_percent"] = 100 * (result["after_ns"] / result["before_ns"] - 1) + summary.append(result) + +with (root / ("convert-" + (experiment or "final") + "-summary.tsv")).open("w") as destination: + writer = csv.DictWriter(destination, fields, delimiter="\t", lineterminator="\n") + writer.writeheader() + writer.writerows(summary) +print(f"Summarized {len(summary)} cases/modes from three before/after rounds") diff --git a/docs/make.jl b/docs/make.jl index ac3a4662..67e0730a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -10,7 +10,7 @@ write("docs/src/releasenotes.md", changelog) makedocs( sitename = "PythonCall & JuliaCall", modules = [PythonCall], - format = Documenter.HTML(assets = ["assets/favicon.ico"]), + format = Documenter.HTML(assets = ["assets/favicon.ico"], edit_link = "v1"), warnonly = [:missing_docs], # avoid raising error when docs are missing pages = [ "Home" => "index.md", diff --git a/docs/src/conversion-to-julia.md b/docs/src/conversion-to-julia.md index 5c65a2aa..b8013739 100644 --- a/docs/src/conversion-to-julia.md +++ b/docs/src/conversion-to-julia.md @@ -1,74 +1,169 @@ # [Conversion to Julia](@id py2jl) -## [Conversion Rules](@id py2jl-conversion) - -The following table specifies the conversion rules used whenever converting a Python object to a Julia object. If the initial Python type matches the "From" column and the desired type `T` intersects with the "To" column, then that conversion is attempted. Conversions are tried in priority order, then in specificity order. - -From Julia, one can convert Python objects to a desired type using `pyconvert(T, x)` for example. - -From Python, the arguments to a Julia function will be converted according to these rules with `T=Any`. - -| From | To | -| :----------------------------------------------------------------------------------------------------------- | :---------------------------------------------------------- | -| **Top priority (wrapped values).** | | -| `juliacall.Jl` | `Any` | -| **Very high priority (arrays).** | | -| Objects satisfying the buffer or array interface (inc. `bytes`, `bytearray`, `array.array`, `numpy.ndarray`) | `PyArray` | -| **High priority (canonical conversions).** | | -| `None` | `Nothing` | -| `bool` | `Bool` | -| `numbers.Integral` (inc. `int`) | `Integer` (prefers `Int`, or `BigInt` on overflow) | -| `float` | `Float64` | -| `complex` | `Complex{Float64}` | -| `range` | `StepRange` | -| `str` | `String` | -| `tuple` | `Tuple` | -| `collections.abc.Mapping` (inc. `dict`) | `PyDict` | -| `collections.abc.Sequence` (inc. `list`) | `PyList` | -| `collections.abc.Set` (inc. `set`, `frozenset`) | `PySet` | -| `io.IOBase` (includes open files) | `PyIO` | -| `BaseException` | `PyException` | -| `datetime.date`/`datetime.time`/`datetime.datetime` | `Date`/`Time`/`DateTime` | -| `datetime.timedelta` | `Microsecond` (or `Millisecond` or `Second` on overflow) | -| `numpy.intXX`/`numpy.uintXX`/`numpy.floatXX` | `IntXX`/`UIntXX`/`FloatXX` | -| `numpy.datetime64` | `NumpyDates.DateTime64` | -| `numpy.timedelta64` | `NumpyDates.TimeDelta64` | -| **Standard priority (other reasonable conversions).** | | -| `None` | `Missing` | -| `bytes` | `Vector{UInt8}`, `Vector{Int8}`, `String` | -| `str` | `String`, `Symbol`, `Char`, `Vector{UInt8}`, `Vector{Int8}` | -| `range` | `UnitRange` | -| `collections.abc.Mapping` | `Dict` | -| `collections.abc.Iterable` | `Vector`, `Set`, `Tuple`, `NamedTuple`, `Pair` | -| `datetime.timedelta` | `Dates.CompoundPeriod` | -| `numbers.Integral` | `Integer`, `Rational`, `Real`, `Number` | -| `numbers.Real` | `AbstractFloat`, `Number`, `Missing`/`Nothing` (if NaN) | -| `numbers.Complex` | `Complex`, `Number` | -| `ctypes.c_int` and other integers | `Integer`, `Rational`, `Real`, `Number` | -| `ctypes.c_float`/`ctypes.c_double` | `Cfloat`/`Cdouble`, `AbstractFloat`, `Real`, `Number` | -| `ctypes.c_voidp` | `Ptr{Cvoid}`, `Ptr` | -| `ctypes.c_char_p` | `Cstring`, `Ptr{Cchar}`, `Ptr` | -| `ctypes.c_wchar_p` | `Cwstring`, `Ptr{Cwchar}`, `Ptr` | -| `numpy.bool_`/`numpy.intXX`/`numpy.uintXX`/`numpy.floatXX` | `Bool`, `Integer`, `Rational`, `Real`, `Number` | -| `numpy.datetime64` | `NumpyDates.InlineDateTime64`, `Dates.DateTime` | -| `numpy.timedelta64` | `NumpyDates.InlineTimeDelta64`, `Dates.Period` | -| Objects satisfying the buffer or array interface | `Array`, `AbstractArray` | -| **Low priority (fallback to `Py`).** | | -| Anything | `Py` | -| **Bottom priority (must be explicitly specified by excluding `Py`).** | | -| Objects satisfying the buffer interface | `PyBuffer` | -| Anything | `PyRef` | - -See [here](@ref python-wrappers) for an explanation of the `Py*` wrapper types (`PyList`, `PyIO`, etc). +## [Conversion rules](@id py2jl-conversion) + +Use `pyconvert(T, x)` to convert a Python object to a Julia type. With +`pyconvert(T, x, default)`, a declined conversion returns `default`. +Exceptions raised during conversion propagate in either form. +JuliaCall uses `T=Any` when passing Python arguments into Julia. + +`Py` is the identity/fallback representation: `pyconvert(Py, x::Py)` returns +`x` directly. Rules should produce Julia representations rather than transform +one Python object into another through a `Py` target. + +Canonical conversions prefer native Julia values: integers become `Int` or +`BigInt` according to their value; floats, complex numbers, strings and +`None` become `Float64`, `ComplexF64`, `String` and `nothing`. +Numeric conversion uses each supported source's representation: integral and +rational sources convert through Julia integers and rationals, while Python +`float` and `complex` use their stored floating-point representations. Supported +NumPy scalars have dedicated rules. Other numeric types need explicit rules; +membership in `numbers.Real` or `numbers.Complex`, or implementing `__float__` +or `__complex__`, does not by itself enable conversion. An exact conversion +that declines cannot be retried through those potentially inexact methods. + +Lists become vectors, dictionaries become dictionaries, sets become sets, +and tuples become tuples (named tuples retain their names). +Array and buffer protocols produce native arrays before generic iterable +conversion. Explicit `PyList`, `PyDict`, `PySet` and `PyArray` targets +select Python-backed wrappers. Generic iterables remain lazy `PyIterable` +wrappers; canonical conversion never consumes an arbitrary iterator. + +Native vector element types are inferred using Julia's `Base.promote_typejoin`, +without converting the individual values to a promoted numeric type. +Dictionary keys and values are inferred independently. Empty unconstrained +containers use `Any`. Explicit element types are honored; abstract bounds, +such as `Vector{<:Real}`, constrain each recursive conversion and the inferred +element type. Cyclic containers are unsupported and may overflow the stack. + +## Preferences and context + +```julia +pyconvert(Any, x; prefer=( + "list" => PyList, + "dict" => PyDict, + "numpy:ndarray" => PyArray, +)) +ctx = PyConvertContext(; prefer=("list" => PyList,)) +pyconvert(ctx, Vector, x) +``` + +Preferences are tried in supplied order. Each matching source attempts normal +conversion to the intersection of the preferred type and the requested type. +A decline continues to the next preference; an exception propagates. +Ordinary conversion follows all preferences and does not reapply them at +that level. Preferences do not alter the registry or source ordering. +They are most useful for representation families; a narrow scalar preference +such as `Int8` can override a more appropriate canonical conversion. + +Context propagation is explicit: a rule calls `pyconvert(ctx, T, child)` +to propagate preferences, or `pyconvert(T, child)` for independent conversion. +Built-in native vector and dictionary rules propagate context. A rule ignoring +context does not propagate preferences to its children. ## [Custom rules](@id py2jl-conversion-custom) -To add a custom conversion rule, you must define a function to do the conversion and call -`pyconvert_add_rule` to register it. +A rule advertises a Python source `t`, Julia output `T`, destination +`scope=S` (default `T`), and conversion function. Register rules in +`__init__`, rather than during precompilation. + +```julia +function from_python(ctx, ::Type{MyArray}, x) + child = pytryconvert(ctx, Vector, x) + child === nothing && return nothing + MyArray(something(child)) +end +id = pyconvert_add_rule("builtins:list", MyArray, from_python; scope=MyArray) +pyconvert_replace_rule(id, "builtins:list", MyArray, from_python; scope=MyArray) +pyconvert_remove_rule(id) +``` + +Every add creates a unique, increasing ID, even for identical rules. +Replacement preserves the ID and creation position. Unknown IDs throw +`KeyError`. Additions, removals, replacements and resolution of pending +rules invalidate conversion caches. + +A rule is eligible for requested type `R` when `typeintersect(T,R)` +is nonempty and `R <: S`. For normalized unions, at least one arm must +be a subtype of `S`. Julia's union normalization can erase an explicit +subtype opt-in: `Union{AbstractArray,MyArray}` is just `AbstractArray`. +Abstract and UnionAll output and scope types are supported. + +Canonical rules use `scope=Any`. Only the owner of the Python source +should register them; PythonCall owns Python built-ins and standard-library +types for this purpose. Other packages must own their destination scope. +For example, a `list -> MyArray` rule scoped to `MyArray` cannot affect +`pyconvert(Any,x)` or `pyconvert(AbstractArray,x)`. + +Rules return `nothing` to decline, an ordinary value to succeed, or +`Some(value)` for explicit success. Use `Some(nothing)` to successfully +return `nothing`, and `Some(Some(value))` to return an actual `Some`. +Prefer returning ordinary values directly. An unnecessary `Some` can allocate +when its concrete type crosses a dispatch boundary; built-in rules wrap only +successful `nothing` and `Some` values. +`pytryconvert([ctx,] T,x)` returns `nothing` on failure and normalizes successes +to the same convention: ordinary values directly, `Some(nothing)` for a +successful nothing, and `Some(Some(value))` for an actual Some value. +Check `result === nothing` for failure; `something(result)` unwraps a success, +and `something(result, default)` handles both outcomes with a default. +Both `f(ctx,T,x)` and `f(T,x)` rules are +supported; the context-aware form is preferred when applicable to that target. +A rule receives a non-union target: each arm of the output/request +intersection is tried before moving to the next rule. Union arm order is +unspecified. Declines continue through all eligible rules. Exceptions propagate. +There is an implicit final `Py` fallback only when the requested type admits it. + +### Source ordering + +Registered source types form a global DAG using strict Python `issubclass` +relationships, including ABC relationships. More-specific sources precede +their ancestors. Ordering never depends on the converted object's MRO or on +Julia output-type specificity. Per-call rule selection filters this global +order without reordering it. + +Independent virtual matchers ``, ``, +`` and `` test the corresponding type attributes or buffer +support. A supporting registered real type precedes the protocol; protocols +precede `object`. No array protocol falsely implies buffer conformance. + +Internal source priorities put Julia wrappers first, then array protocols, +then buffers, then ordinary types. Subtypes inherit the maximum ancestor +priority. Within a priority, stable topological sorting respects subtype +edges and breaks ties by source registration order. Rules on the same source +run in creation order. Incomparable array matchers use definition order: +array struct, array interface, then array. A specific ndarray rule can +override its generic protocol rules. + +### Lazy resolution and diagnostics + +Source names in rules and preferences use `"module:name"`, where `name` is an +attribute path (possibly dotted) looked up on the loaded module. It need not +match the type's `__module__` or `__qualname__`: a public re-export such as +`"mypackage:PublicType"` works even if the type is defined in an internal module. +A bare name abbreviates a built-in: `"list"` means `"builtins:list"`, so +`prefer=("list" => PyList,)` opts into list wrappers. Aliases resolving to the +same Python type share source ordering; their rules retain creation order. +The special shorthand `"None"` means `"types:NoneType"`; for example, +`pyconvert(Any, x; prefer=("None" => Missing,))` converts Python None to missing. + +Optional modules are never imported merely to resolve rules or preferences. +Pending rules enter the global order when their module is +available in `sys.modules` and the named type can be resolved. Encountering +a new Python source module checks pending rules; cache hits do not. +Modules should define their relevant types and complete ABC registrations +before conversion begins. Arbitrary later `ABC.register()` mutations are +unsupported. Cached Python type identities retain owning references. -You must not do this while precompiling, so these calls will normally be in the `__init__` -function of your module. +`pyconvert_rules(T,x)` lists source-matching rules in global order, including +ID, source, advertised output, scope, effective source priority and destination +eligibility. The internal `PythonCall.Convert.pyconvert_preferred_type(t)` +reports the first canonical candidate, which may decline for a particular value. ```@docs PythonCall.pyconvert_add_rule +PythonCall.PyConvertContext +PythonCall.pytryconvert +PythonCall.pyconvert_remove_rule +PythonCall.pyconvert_replace_rule +PythonCall.pyconvert_rules ``` diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index 25357887..dcffb9e5 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -2,6 +2,37 @@ Use this guide to help with migrating code from v0.9 to v1. +## Conversion rules + +`pyconvert(Any, x)` now prefers native vectors, dictionaries, sets and arrays for +materialized Python containers. Generic iterators remain lazy `PyIterable` +wrappers. Request `PyList`, `PyDict` or `PyArray` explicitly, or use source-specific +preferences such as `prefer=("builtins:list" => PyList,)` to select wrappers. + +Generic `numbers.Real` and `numbers.Complex` conversions through `__float__` and +`__complex__` have been removed: those methods can lose information before the +requested Julia conversion is attempted. Integral and rational sources keep +their exact representation rules; Python `float`, `complex` and supported NumPy +scalars keep dedicated rules. Other numeric types must register their own rules. + +Rule priorities have been removed. Register rules with +`pyconvert_add_rule(source, T, f; scope=T)`. Use `scope=Any` for canonical rules +only when you own the Python source type. Otherwise scope the rule to a Julia +destination type you own. Each registration returns a unique ID; use +`pyconvert_remove_rule(id)` or `pyconvert_replace_rule(id, source, T, f; scope=T)` +to remove or update it. + +Rules now return `nothing` to decline, a value to succeed, or `Some(value)` for +explicit success. Replace `pyconvert_unconverted()` with `nothing` and +`pyconvert_return(x)` with `Some(x)` (or simply `x` when it is neither `nothing` +nor a `Some`). `pytryconvert(T,x)` uses the same normalized convention: failure is +`nothing`, ordinary successes are raw values, and successful `nothing`/`Some` +values are wrapped once. Test `r === nothing` for failure and use `something(r)` +to recover a success, or `something(r, default)` for a default on failure. +Rule functions may accept `(ctx,T,x)` to propagate preferences explicitly with +`pyconvert(ctx,T,child)`. See [conversion to Julia](@ref py2jl) for scope, ordering +and context details. + ## Core functionality Comparisons (`==`, `<`, etc.) between Python objects `Py`, diff --git a/pytest/test_convert_v1.py b/pytest/test_convert_v1.py new file mode 100644 index 00000000..ba12e957 --- /dev/null +++ b/pytest/test_convert_v1.py @@ -0,0 +1,17 @@ +def test_native_containers_at_juliacall_boundary(): + from juliacall import Main as jl + + jl.jl_eval("using PythonCall") + assert bool(jl.jl_eval("x -> x isa Vector{Int}")([1, 2])) + assert bool(jl.jl_eval("x -> x isa Dict{String,Int}")({"a": 1})) + assert bool(jl.jl_eval("x -> x isa Vector{Real}")([1, 2.5])) + assert bool(jl.jl_eval("x -> x isa PyIterable")(iter([1, 2]))) + + +def test_native_conversion_does_not_consume_iterators(): + from juliacall import Main as jl + + jl.jl_eval("using PythonCall") + iterator = iter([1, 2, 3]) + assert bool(jl.jl_eval("x -> x isa PyIterable")(iterator)) + assert next(iterator) == 1 diff --git a/src/API/exports.jl b/src/API/exports.jl index d2a048fc..dae455d0 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -103,16 +103,14 @@ export pyxor export unsafe_pydel # Convert +export PyConvertContext export @pyconvert export pyconvert export pyconvert_add_rule -export PYCONVERT_PRIORITY_ARRAY -export PYCONVERT_PRIORITY_CANONICAL -export PYCONVERT_PRIORITY_FALLBACK -export PYCONVERT_PRIORITY_NORMAL -export PYCONVERT_PRIORITY_WRAP -export pyconvert_return -export pyconvert_unconverted +export pytryconvert +export pyconvert_remove_rule +export pyconvert_replace_rule +export pyconvert_rules # Wrap export PyArray diff --git a/src/API/functions.jl b/src/API/functions.jl index 77492f7a..f8b6e90c 100644 --- a/src/API/functions.jl +++ b/src/API/functions.jl @@ -102,8 +102,10 @@ function unsafe_pydel end # Convert function pyconvert end function pyconvert_add_rule end -function pyconvert_return end -function pyconvert_unconverted end +function pytryconvert end +function pyconvert_remove_rule end +function pyconvert_replace_rule end +function pyconvert_rules end # JlWrap function pybinaryio end diff --git a/src/API/types.jl b/src/API/types.jl index 14acece8..099eb80e 100644 --- a/src/API/types.jl +++ b/src/API/types.jl @@ -1,11 +1,15 @@ # Convert -@enum PyConvertPriority begin - PYCONVERT_PRIORITY_WRAP = 400 - PYCONVERT_PRIORITY_ARRAY = 300 - PYCONVERT_PRIORITY_CANONICAL = 200 - PYCONVERT_PRIORITY_NORMAL = 0 - PYCONVERT_PRIORITY_FALLBACK = -100 +""" + PyConvertContext(; prefer=()) + +Explicit conversion context. Preferences are ordered `"module:name" => JuliaType` +pairs, resolved by attribute lookup in loaded modules. A bare name abbreviates +`"builtins:name"`, except `"None"` abbreviates `"types:NoneType"`. +Rules propagate preferences by calling `pyconvert(ctx, T, child)` explicitly. +""" +struct PyConvertContext{P} + prefer::P end # Core diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index fc506fc6..2f95accb 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -14,21 +14,22 @@ using ..NumpyDates using Dates: Date, Time, DateTime, Second, Millisecond, Microsecond, Nanosecond import ..PythonCall: + PyConvertContext, @pyconvert, pyconvert_add_rule, - pyconvert_return, - pyconvert_unconverted, pyconvert, - PyConvertPriority + pytryconvert, + pyconvert_remove_rule, + pyconvert_replace_rule, + pyconvert_rules -export pyconvert_isunconverted, - pyconvert_result, +export pyconvert_return, + pyconvert_isunconverted, pyconvert_result, pyconvert_tryconvert, pyconvert_unconverted, pyconvertarg - include("pyconvert.jl") include("rules.jl") include("ctypes.jl") diff --git a/src/Convert/numpy.jl b/src/Convert/numpy.jl index 73383d54..309eb4e2 100644 --- a/src/Convert/numpy.jl +++ b/src/Convert/numpy.jl @@ -112,7 +112,7 @@ function init_numpy() rule = pyconvert_rule_numpysimplevalue{T,false}() saferule = pyconvert_rule_numpysimplevalue{T,true}() - pyconvert_add_rule(name, T, saferule, PYCONVERT_PRIORITY_ARRAY) + pyconvert_add_rule(name, T, saferule; scope=Any) isuint && pyconvert_add_rule(name, UInt, sizeof(T) ≤ sizeof(UInt) ? saferule : rule) isuint && pyconvert_add_rule(name, Int, sizeof(T) < sizeof(Int) ? saferule : rule) isint && @@ -131,7 +131,7 @@ function init_numpy() "numpy:datetime64", DateTime64, pyconvert_rule_datetime64, - PYCONVERT_PRIORITY_ARRAY, + ; scope=Any, ) pyconvert_add_rule("numpy:datetime64", InlineDateTime64, pyconvert_rule_datetime64) pyconvert_add_rule( @@ -147,7 +147,7 @@ function init_numpy() "numpy:timedelta64", TimeDelta64, pyconvert_rule_timedelta64, - PYCONVERT_PRIORITY_ARRAY, + ; scope=Any, ) pyconvert_add_rule("numpy:timedelta64", InlineTimeDelta64, pyconvert_rule_timedelta64) pyconvert_add_rule( diff --git a/src/Convert/pandas.jl b/src/Convert/pandas.jl index 65786f1c..aedb603e 100644 --- a/src/Convert/pandas.jl +++ b/src/Convert/pandas.jl @@ -2,11 +2,12 @@ pyconvert_rule_pandas_na(::Type{Nothing}, x::Py) = pyconvert_return(nothing) pyconvert_rule_pandas_na(::Type{Missing}, x::Py) = pyconvert_return(missing) function init_pandas() + # pandas.NA is public across supported versions, even where NAType is not. pyconvert_add_rule( - "pandas._libs.missing:NAType", + "pandas:NA.__class__", Missing, pyconvert_rule_pandas_na, - PYCONVERT_PRIORITY_CANONICAL, + ; scope=Any, ) - pyconvert_add_rule("pandas._libs.missing:NAType", Nothing, pyconvert_rule_pandas_na) + pyconvert_add_rule("pandas:NA.__class__", Nothing, pyconvert_rule_pandas_na) end diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 3df5f7f6..c260c776 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -1,519 +1,544 @@ struct PyConvertRule + id::Int + source::String type::Type + scope::Type func::Function - priority::PyConvertPriority end -const PYCONVERT_RULES = Dict{String,Vector{PyConvertRule}}() -const PYCONVERT_EXTRATYPES = Py[] - -""" - pyconvert_add_rule(tname::String, T::Type, func::Function, priority::PyConvertPriority=PYCONVERT_PRIORITY_NORMAL) - -Add a new conversion rule for `pyconvert`. - -### Arguments - -- `tname` is a string of the form `"__module__:__qualname__"` identifying a Python type `t`, - such as `"builtins:dict"` or `"sympy.core.symbol:Symbol"`. This rule only applies to - Python objects of this type. -- `T` is a Julia type, such that this rule only applies when the target type intersects - with `T`. -- `func` is the function implementing the rule. -- `priority` determines whether to prioritise this rule above others. - -When `pyconvert(R, x)` is called, all rules such that `typeintersect(T, R) != Union{}` -and `pyisinstance(x, t)` are considered. These rules are sorted first by priority, -then by the specificity of `t` (e.g. `bool` is more specific than `int` is more specific -than `object`) then by the order they were added. The rules are tried in turn until one -succeeds. - -### Implementing `func` - -`func` is called as `func(S, x::Py)` for some `S <: T`. +# Source nodes retain their first registration position, including across replacement. +mutable struct PyConvertSource + name::String + type::Py + index::Int + priority::Int +end +const PYCONVERT_RULES = PyConvertRule[] +const PYCONVERT_SOURCES = PyConvertSource[] +const PYCONVERT_PENDING = Set{String}() +const PYCONVERT_ORDER = PyConvertRule[] +const PYCONVERT_NEXT_ID = Ref(0) +const PYCONVERT_DIRTY = Ref(true) +const PYCONVERT_MODULES = Set{String}() +const PYCONVERT_PROTOCOLS = ("", "", "", "") + +# Normalize at registration/context construction, rather than during dispatch. +function pyconvert_source_name(name::String) + name == "None" && return "types:NoneType" + name in PYCONVERT_PROTOCOLS || ':' in name ? name : "builtins:" * name +end -It must return one of: -- `pyconvert_return(ans)` where `ans` is the result of the conversion (and must be an `S`). -- `pyconvert_unconverted()` if the conversion was not possible (e.g. converting a `list` to - `Vector{Int}` might fail if some of the list items are not integers). +PyConvertContext(; prefer=()) = PyConvertContext(map(p -> pyconvert_source_name(first(p)) => Val(last(p)), prefer)) +const PYCONVERT_DEFAULT_CONTEXT = PyConvertContext() -The target type `S` is never a union or the empty type, i.e. it is always a data type or -union-all. +function pyconvert_invalidate!() + empty!.(values(PYCONVERT_RULES_CACHE)) + empty!(PYCONVERT_MODULES) + PYCONVERT_DIRTY[] = true +end -### Priority +function pyconvert_source!(name::String) + any(s -> s.name == name, PYCONVERT_SOURCES) && return + push!(PYCONVERT_SOURCES, PyConvertSource(name, PyNULL, length(PYCONVERT_SOURCES)+1, 0)) + name in PYCONVERT_PROTOCOLS || push!(PYCONVERT_PENDING, name) +end -Most rules should have priority `PYCONVERT_PRIORITY_NORMAL` (the default) which is for any -reasonable conversion rule. +""" + pyconvert_add_rule(source, T, f; scope=T) -> id + +Register a new rule. Source names use `"module:name"` (or just `"name"` for +builtins). Resolution lazily looks up attributes, including dotted paths, in +loaded modules; names need not match the type's __module__ or __qualname__. +A rule is eligible when its output intersects the requested type R and R is a +subtype of scope (or a normalized union arm is). Own the source for scope=Any, +or own the destination scope. Rules return nothing to decline, a value to succeed, +or Some(value) to explicitly succeed (required for a successful nothing/Some). +Functions may accept `(ctx, T, x)` or `(T, x)`; the former is preferred. +""" +function pyconvert_add_rule(source::String, T::Type, f::Function; scope::Type=T) + source = pyconvert_source_name(source) + pyconvert_source!(source) + id = (PYCONVERT_NEXT_ID[] += 1) + push!(PYCONVERT_RULES, PyConvertRule(id, source, T, scope, f)) + pyconvert_invalidate!() + id +end -Use priority `PYCONVERT_PRIORITY_CANONICAL` for **canonical** conversion rules. Immutable -objects may be canonically converted to their corresponding Julia type, such as `int` to -`Integer`. Mutable objects **must** be converted to a wrapper type, such that the original -Python object can be retrieved. For example a `list` is canonically converted to `PyList` -and not to a `Vector`. There should not be more than one canonical conversion rule for a -given Python type. +"""Remove a conversion rule by ID. Unknown IDs throw KeyError.""" +function pyconvert_remove_rule(id::Integer) + i = findfirst(r -> r.id == id, PYCONVERT_RULES) + i === nothing && throw(KeyError(id)) + deleteat!(PYCONVERT_RULES, i) + pyconvert_invalidate!() + nothing +end -Other priorities are reserved for internal use. -""" -function pyconvert_add_rule( - pytypename::String, - type::Type, - func::Function, - priority::PyConvertPriority = PYCONVERT_PRIORITY_NORMAL, -) - @nospecialize type func - push!( - get!(Vector{PyConvertRule}, PYCONVERT_RULES, pytypename), - PyConvertRule(type, func, priority), - ) - empty!.(values(PYCONVERT_RULES_CACHE)) - return +"""Replace a rule, preserving its ID and creation position.""" +function pyconvert_replace_rule(id::Integer, source::String, T::Type, f::Function; scope::Type=T) + i = findfirst(r -> r.id == id, PYCONVERT_RULES) + i === nothing && throw(KeyError(id)) + source = pyconvert_source_name(source) + pyconvert_source!(source) + PYCONVERT_RULES[i] = PyConvertRule(id, source, T, scope, f) + pyconvert_invalidate!() + id end -# Alternative ways to represent the result of conversion. -if true - # Returns either the result or Unconverted(). - struct Unconverted end - @inline pyconvert_return(x) = x - @inline pyconvert_unconverted() = Unconverted() - @inline pyconvert_returntype(::Type{T}) where {T} = Union{T,Unconverted} - @inline pyconvert_isunconverted(r) = r === Unconverted() - @inline pyconvert_result(::Type{T}, r) where {T} = r::T -elseif false - # Stores the result in PYCONVERT_RESULT. - # This is global state, probably best avoided. - const PYCONVERT_RESULT = Ref{Any}(nothing) - @inline pyconvert_return(x) = (PYCONVERT_RESULT[] = x; true) - @inline pyconvert_unconverted() = false - @inline pyconvert_returntype(::Type{T}) where {T} = Bool - @inline pyconvert_isunconverted(r::Bool) = !r - @inline pyconvert_result(::Type{T}, r::Bool) where {T} = - (ans = PYCONVERT_RESULT[]::T; PYCONVERT_RESULT[] = nothing; ans) -else - # Same as the previous scheme, but with special handling for bits types. - # This is global state, probably best avoided. - const PYCONVERT_RESULT = Ref{Any}(nothing) - const PYCONVERT_RESULT_ISBITS = Ref{Bool}(false) - const PYCONVERT_RESULT_TYPE = Ref{Type}(Union{}) - const PYCONVERT_RESULT_BITSLEN = 1024 - const PYCONVERT_RESULT_BITS = fill(0x00, PYCONVERT_RESULT_BITSLEN) - function pyconvert_return(x::T) where {T} - if isbitstype(T) && sizeof(T) ≤ PYCONVERT_RESULT_BITSLEN - unsafe_store!(Ptr{T}(pointer(PYCONVERT_RESULT_BITS)), x) - PYCONVERT_RESULT_ISBITS[] = true - PYCONVERT_RESULT_TYPE[] = T - else - PYCONVERT_RESULT[] = x - PYCONVERT_RESULT_ISBITS[] = false - end - return true - end - @inline pyconvert_unconverted() = false - @inline pyconvert_returntype(::Type{T}) where {T} = Bool - @inline pyconvert_isunconverted(r::Bool) = !r - function pyconvert_result(::Type{T}, r::Bool) where {T} - if isbitstype(T) - if sizeof(T) ≤ PYCONVERT_RESULT_BITSLEN - @assert PYCONVERT_RESULT_ISBITS[] - @assert PYCONVERT_RESULT_TYPE[] == T - return unsafe_load(Ptr{T}(pointer(PYCONVERT_RESULT_BITS)))::T - end - elseif PYCONVERT_RESULT_ISBITS[] - t = PYCONVERT_RESULT_TYPE[] - @assert isbitstype(t) - @assert sizeof(t) ≤ PYCONVERT_RESULT_BITSLEN - @assert t <: T - @assert isconcretetype(t) - return unsafe_load(Ptr{t}(pointer(PYCONVERT_RESULT_BITS)))::T - end - # general case - ans = PYCONVERT_RESULT[]::T - PYCONVERT_RESULT[] = nothing - return ans::T +# Rule results only need a wrapper when the successful value is ambiguous. +# The same encoding is used for normalized public pytryconvert results. +@inline pyconvert_return(x) = x +@inline pyconvert_return(x::Union{Nothing,Some}) = Some(x) +@inline pyconvert_unconverted() = nothing +@inline pyconvert_isunconverted(x) = x === nothing +@inline pyconvert_result(::Type{T}, x::Some) where T = something(x)::T +@inline pyconvert_result(x::Some) = something(x) +@inline pyconvert_result(::Type{T}, x) where T = x::T +@inline pyconvert_result(x) = x +function _pyconvert_returntype(T::Type) + T === Any && return Any + T === Union{} && return Nothing + T isa Union && return Union{_pyconvert_returntype(T.a),_pyconvert_returntype(T.b)} + T <: Nothing && return Union{Nothing,Some{Nothing}} + if T <: Some + return isconcretetype(T) ? Union{Nothing,Some{T}} : Union{Nothing,Some{<:T}} end + Union{Nothing,T} +end +@generated pyconvert_returntype(::Type{T}) where T = QuoteNode(_pyconvert_returntype(T)) +# Avoid allocating an escaping Some for every element of an inferred container. +# Public/rule results wrap only ambiguous successes. Dispatch uses this private +# sentinel, with an escape for values that happen to use our internal types. +struct PyConvertFailure end +struct PyConvertEscape{T} + value::T +end +const PYCONVERT_FAILED = PyConvertFailure() +@inline pyconvert_pack(x) = x isa Union{PyConvertFailure,PyConvertEscape} ? PyConvertEscape(x) : x +@inline pyconvert_unpack(x) = x +@inline pyconvert_unpack(x::PyConvertEscape) = x.value +# Check before returning across the dispatch boundary, so a concrete bits value +# can use Julia's inferred return convention instead of being boxed as Any. +@inline function pyconvert_check_raw(::Type{T}, x) where T + x === PYCONVERT_FAILED && return PYCONVERT_FAILED + T === Any ? x : pyconvert_pack(pyconvert_unpack(x)::T) +end +@inline pyconvert_raw(::Nothing) = PYCONVERT_FAILED +@inline pyconvert_raw(x::Some) = pyconvert_pack(something(x)) +@inline pyconvert_raw(x) = pyconvert_pack(x) +@inline function pyconvert_raw(::Type{T}, x) where T + x === nothing && return PYCONVERT_FAILED + value = x isa Some ? something(x) : x + pyconvert_pack(value::T) end -pyconvert_result(r) = pyconvert_result(Any, r) - -pyconvert_tryconvert(::Type{T}, x::T) where {T} = pyconvert_return(x) -pyconvert_tryconvert(::Type{T}, x) where {T} = +pyconvert_tryconvert(::Type{T}, x::T) where T = pyconvert_return(x) +function pyconvert_tryconvert(::Type{T}, x) where T try pyconvert_return(convert(T, x)::T) - catch - pyconvert_unconverted() + catch exc + # Expected numeric/representation failures only. User errors propagate. + (exc isa Union{InexactError,OverflowError} || + (exc isa MethodError && exc.f === convert)) || rethrow() + nothing end - -function pyconvert_typename(t::Py) - m = pygetattr(t, "__module__", "") - n = pygetattr(t, "__name__", "") - return "$m:$n" end -function _pyconvert_get_rules(pytype::Py) - pyisin(x, ys) = any(pyis(x, y) for y in ys) +pyconvert_typename(t::Py) = "$(pygetattr(t, "__module__", "")):$(pygetattr(t, "__qualname__", ""))" +function pyconvert_protocol(name, t::Py) + name == "" && return C.PyType_CheckBuffer(t) + name == "" && return pyhasattr(t, "__array__") + name == "" && return pyhasattr(t, "__array_interface__") + name == "" && return pyhasattr(t, "__array_struct__") + false +end - # get the MROs of all base types we are considering - omro = collect(pytype.__mro__) - basetypes = Py[pytype] - basemros = Vector{Py}[omro] - for xtype in PYCONVERT_EXTRATYPES - # find the topmost supertype of - xbase = PyNULL - for base in omro - if pyissubclass(base, xtype) - xbase = base - end - end - if !pyisnull(xbase) - push!(basetypes, xtype) - xmro = collect(xtype.__mro__) - pyisin(xbase, xmro) || pushfirst!(xmro, xbase) - push!(basemros, xmro) - end +# Never import optional modules as a side effect of conversion. +function pyconvert_resolve(name::String) + name = pyconvert_source_name(name) + name == "builtins:NoneType" && return pytype(pybuiltins.None) + parts = split(name, ':'; limit=2) + length(parts) == 2 || throw(ArgumentError("invalid Python source name: $name")) + m = pygetitem(pyimport("sys").modules, parts[1], PyNULL) + pyisnull(m) && return PyNULL + for part in split(parts[2], '.') + m = pygetattr(m, part, PyNULL) + pyisnull(m) && return PyNULL end - for xbase in basetypes[2:end] - push!(basemros, [xbase]) - end - - # merge the MROs - # this is a port of the merge() function at the bottom of: - # https://www.python.org/download/releases/2.3/mro/ - mro = Py[] - while !isempty(basemros) - # find the first head not contained in any tail - ok = false - b = PyNULL - for bmro in basemros - b = bmro[1] - if all(bmro -> !pyisin(b, bmro[2:end]), basemros) - ok = true - break + pyisinstance(m, pybuiltins.type) || throw(ArgumentError("$name is not a Python type")) + m +end +function pyconvert_resolve_pending!(encountered=PyNULL) + changed = false + for s in PYCONVERT_SOURCES + s.name in PYCONVERT_PENDING || continue + t = pyconvert_resolve(s.name) + # Types made with exec need not be exported by their named module. + # Resolve registered real ancestors by identity, without using MRO order. + if pyisnull(t) && !pyisnull(encountered) + for base in encountered.__mro__ + if pyconvert_typename(base) == s.name + t = base + break + end end end - ok || error( - "Fatal inheritance error: could not merge MROs (mro=$mro, basemros=$basemros)", - ) - # add it to the list - push!(mro, b) - # remove it from consideration - for bmro in basemros - filter!(t -> !pyis(t, b), bmro) - end - # remove empty lists - filter!(x -> !isempty(x), basemros) + pyisnull(t) && continue + s.type = t + delete!(PYCONVERT_PENDING, s.name) + changed = true end - # check the original MRO is preserved - omro_ = filter(t -> pyisin(t, omro), mro) - @assert length(omro) == length(omro_) - @assert all(pyis(x, y) for (x, y) in zip(omro, omro_)) + changed && pyconvert_invalidate!() +end - # get the names of the types in the MRO of pytype - xmro = [String[pyconvert_typename(t)] for t in mro] +function pyconvert_subtype(a::PyConvertSource, b::PyConvertSource) + a === b && return false + av, bv = a.name in PYCONVERT_PROTOCOLS, b.name in PYCONVERT_PROTOCOLS + av && return !bv && b.name == "builtins:object" + bv && return pyconvert_protocol(b.name, a.type) + pyissubclass(a.type, b.type) && !pyissubclass(b.type, a.type) +end - # add special names corresponding to certain interfaces - # these get inserted just above the topmost type satisfying the interface - for (t, x) in reverse(collect(zip(mro, xmro))) - if pyhasattr(t, "__array_struct__") - push!(x, "") - break - end +# Stable Kahn ordering. All strict subtype edges participate, not just MRO edges. +function pyconvert_toposort(edges, priorities, indices) + remaining = collect(eachindex(priorities)) + order = Int[] + while !isempty(remaining) + ready = filter(i -> !any(j -> edges[j,i], remaining), remaining) + isempty(ready) && error("cyclic pyconvert source relationships") + i = first(sort!(ready; by=i -> (-priorities[i], indices[i]))) + push!(order, i) + deleteat!(remaining, findfirst(==(i), remaining)) end - for (t, x) in reverse(collect(zip(mro, xmro))) - if pyhasattr(t, "__array_interface__") - push!(x, "") - break + order +end +function pyconvert_order!() + PYCONVERT_DIRTY[] || return + sources = filter(s -> !(s.name in PYCONVERT_PENDING) && + (s.name in PYCONVERT_PROTOCOLS || s.name in ("builtins:object","juliacall:JlBase") || + any(r -> r.source == s.name, PYCONVERT_RULES)), PYCONVERT_SOURCES) + edges = [pyconvert_subtype(a,b) for a in sources, b in sources] + priorities = [s.name == "juliacall:JlBase" ? 30 : + s.name == "" ? 10 : s.name in PYCONVERT_PROTOCOLS ? 20 : 0 for s in sources] + # Public re-exports are the same source type, including its built-in priority. + for i in eachindex(sources), j in eachindex(sources) + if !pyisnull(sources[i].type) && pyis(sources[i].type,sources[j].type) + priorities[i] = max(priorities[i],priorities[j]) end end - for (t, x) in reverse(collect(zip(mro, xmro))) - if pyhasattr(t, "__array__") - push!(x, "") - break - end + # Propagate maximum priority from every ancestor to every descendant. + for _ in eachindex(sources), i in eachindex(sources), j in eachindex(sources) + edges[i,j] && (priorities[i] = max(priorities[i], priorities[j])) end - for (t, x) in reverse(collect(zip(mro, xmro))) - if C.PyType_CheckBuffer(t) - push!(x, "") - break - end + order = pyconvert_toposort(edges, priorities, [s.index for s in sources]) + empty!(PYCONVERT_ORDER) + emitted = Set{String}() + for i in order + sources[i].priority = priorities[i] + sources[i].name in emitted && continue + aliases = [s.name for s in sources if s.name == sources[i].name || + (!pyisnull(s.type) && !pyisnull(sources[i].type) && pyis(s.type,sources[i].type))] + union!(emitted,aliases) + append!(PYCONVERT_ORDER, filter(r -> r.source in aliases, PYCONVERT_RULES)) end - - # flatten to get the MRO as a list of strings - mro = String[x for xs in xmro for x in xs] - - # get corresponding rules - rules = PyConvertRule[ - rule for tname in mro for - rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES, tname) - ] - - # order the rules by priority, then by original order - order = sort(axes(rules, 1), by = i -> (rules[i].priority, -i), rev = true) - rules = rules[order] - - @debug "pyconvert" pytype mro = join(mro, " ") - return rules + PYCONVERT_DIRTY[] = false +end +pyconvert_scope(R, S) = R isa Union ? pyconvert_scope(R.a,S) || pyconvert_scope(R.b,S) : R <: S +pyconvert_eligible(r, R) = pyconvert_scope(R,r.scope) && typeintersect(r.type,R) !== Union{} +function pyconvert_matches(name, t) + name in PYCONVERT_PROTOCOLS && return pyconvert_protocol(name,t) + s = findfirst(s -> s.name == name, PYCONVERT_SOURCES) + s === nothing && return (u = pyconvert_resolve(name); !pyisnull(u) && pyissubclass(t,u)) + u = PYCONVERT_SOURCES[s].type + pyisnull(u) && (u = pyconvert_resolve(name)) + !pyisnull(u) && pyissubclass(t,u) end -const PYCONVERT_PREFERRED_TYPE = Dict{Py,Type}() - -pyconvert_preferred_type(pytype::Py) = - get!(PYCONVERT_PREFERRED_TYPE, pytype) do - if pyissubclass(pytype, pybuiltins.int) - Union{Int,BigInt} - else - _pyconvert_get_rules(pytype)[1].type +# Closures capture the calling convention once, for this context and target type. +function pyconvert_fix(ctx, ::Type{T}, f) where T + if applicable(f, ctx, T, PyNULL) + (ctx, x) -> pyconvert_raw(T,f(ctx,T,x)) + else + (ctx, x) -> pyconvert_raw(T,f(T,x)) + end +end +struct PyConvertSelection + type::Py # Own a reference: cached pointers can never be recycled underneath us. + rules::Vector{Function} + firstfunc::Any + firsttarget::Type +end +const PYCONVERT_RULES_CACHE = Dict{Tuple{Type,Type},Dict{C.PyPtr,PyConvertSelection}}() +@generated pyconvert_rules_cache(::Type{CTX}, ::Type{T}) where {CTX,T} = + get!(Dict{C.PyPtr,PyConvertSelection}, PYCONVERT_RULES_CACHE, (CTX,T)) +function pyconvert_selection(ctx, ::Type{R}, tptr) where R + cache = pyconvert_rules_cache(typeof(ctx), R) + haskey(cache,tptr) && return cache[tptr] + t = pynew(incref(tptr)) + mod = pystr(String, pygetattr(t,"__module__","")) + if !(mod in PYCONVERT_MODULES) + pyconvert_resolve_pending!(t) + push!(PYCONVERT_MODULES,mod) + end + pyconvert_order!() + rules = Function[] + firstfunc, firsttarget = nothing, Union{} + for r in PYCONVERT_ORDER + pyconvert_eligible(r,R) && pyconvert_matches(r.source,t) || continue + for T in Utils.explode_union(typeintersect(r.type,R)) + T === Union{} && continue + if isempty(rules) + firstfunc, firsttarget = r.func, T + end + push!(rules,pyconvert_fix(ctx,T,r.func)) end end - -function pyconvert_get_rules(type::Type, pytype::Py) - @nospecialize type - - # this could be cached - rules = _pyconvert_get_rules(pytype) - - # intersect rules with type - rules = PyConvertRule[ - PyConvertRule(typeintersect(rule.type, type), rule.func, rule.priority) for - rule in rules - ] - - # explode out unions - rules = [ - PyConvertRule(type, rule.func, rule.priority) for rule in rules for - type in Utils.explode_union(rule.type) - ] - - # filter out empty rules - rules = [rule for rule in rules if rule.type != Union{}] - - # filter out repeated rules - rules = [ - rule for (i, rule) in enumerate(rules) if !any( - (rule.func === rules[j].func) && ((rule.type) <: (rules[j].type)) for - j = 1:(i-1) - ) - ] - - @debug "pyconvert" type rules - return Function[pyconvert_fix(rule.type, rule.func) for rule in rules] + cache[tptr] = PyConvertSelection(t,rules,firstfunc,firsttarget) end -pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) - -const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}() - -@generated pyconvert_rules_cache(::Type{T}) where {T} = - get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T) - -function pyconvert_rule_fast(::Type{T}, x::Py) where {T} - if T isa Union - a = pyconvert_rule_fast(T.a, x)::pyconvert_returntype(T.a) - pyconvert_isunconverted(a) || return a - b = pyconvert_rule_fast(T.b, x)::pyconvert_returntype(T.b) - pyconvert_isunconverted(b) || return b - elseif (T == Nothing) | (T == Missing) - pyisnone(x) && return pyconvert_return(T()) - elseif (T == Bool) - pyisFalse(x) && return pyconvert_return(false) - pyisTrue(x) && return pyconvert_return(true) - elseif (T == Int) | (T == BigInt) - pyisint(x) && return pyconvert_rule_int(T, x) - elseif (T == Float64) - pyisfloat(x) && return pyconvert_return(T(pyfloat_asdouble(x))) - elseif (T == ComplexF64) - pyiscomplex(x) && return pyconvert_return(T(pycomplex_ascomplex(x))) - elseif (T == String) | (T == Char) | (T == Symbol) - pyisstr(x) && return pyconvert_rule_str(T, x) - elseif (T == Vector{UInt8}) | (T == Base.CodeUnits{UInt8,String}) - pyisbytes(x) && return pyconvert_rule_bytes(T, x) - elseif (T <: StepRange) | (T <: UnitRange) - pyisrange(x) && return pyconvert_rule_range(T, x) +# Direct calls only when the cache proves this is the first selected rule and +# target. Registry mutations clear the proof together with the selected rules. +@inline function pyconvert_first(ctx, ::Type{T}, x, selection) where T + f = selection.firstfunc + if selection.firsttarget === T + if T === Int || T === BigInt || T === Integer + f === pyconvert_rule_int && return pyconvert_check_raw(T,pyconvert_rule_int_raw(T,x)) + elseif T === Float64 + f === pyconvert_rule_float && return pyconvert_check_raw(T,pyconvert_raw(pyconvert_rule_float(T,x))) + elseif T === Bool + f === pyconvert_rule_bool && return pyconvert_check_raw(T,pyconvert_raw(pyconvert_rule_bool(T,x))) + elseif T === String + f === pyconvert_rule_str && return pyconvert_check_raw(T,pyconvert_raw(pyconvert_rule_str(T,x))) + elseif T === Nothing + f === pyconvert_rule_none && return nothing + end end - pyconvert_unconverted() + pyconvert_check_raw(T,@pyregionbreak(selection.rules[1](ctx,x))) end -function pytryconvert(::Type{T}, x_) where {T} - # Convert the input to a Py +function pytryconvert_plain(ctx, ::Type{T}, x::Py) where T + selection = pyconvert_selection(ctx,T,C.Py_Type(x)) + if !isempty(selection.rules) + ans = pyconvert_first(ctx,T,x,selection) + ans === PYCONVERT_FAILED || return ans + for i in 2:length(selection.rules) + ans = @pyregionbreak(selection.rules[i](ctx,x)) + ans === PYCONVERT_FAILED || return pyconvert_check_raw(T, ans) + end + end + Py <: T ? x : PYCONVERT_FAILED +end +pyconvert_preference_type(::Val{T}) where T = T +function pytryconvert_raw(ctx::PyConvertContext, ::Type{T}, x_) where T x = Py(x_) - - # We can optimize the conversion for some types by overloading pytryconvert_fast. - # It MUST give the same results as via the slower route using rules. - ans1 = pyconvert_rule_fast(T, x)::pyconvert_returntype(T) - pyconvert_isunconverted(ans1) || return ans1 - - # get rules from the cache - # TODO: we should hold weak references and clear the cache if types get deleted - tptr = C.Py_Type(x) - trules = pyconvert_rules_cache(T) - rules = get!(trules, tptr) do - t = pynew(incref(tptr)) - ans = pyconvert_get_rules(T, t)::Vector{Function} - unsafe_pydel(t) - ans + for (source, preferred) in ctx.prefer + pyconvert_matches(source,pytype(x)) || continue + Q = typeintersect(T,pyconvert_preference_type(preferred)) + Q === Union{} && continue + ans = pytryconvert_plain(ctx,Q,x) + ans === PYCONVERT_FAILED || return ans end + pytryconvert_plain(ctx,T,x) +end +""" + pytryconvert(T, x; prefer=()) + pytryconvert(ctx::PyConvertContext, T, x) - # apply the rules - for rule in rules - # Conversion rules are user-extensible Julia code and may yield. - ans2 = @pyregionbreak(rule(x))::pyconvert_returntype(T) - pyconvert_isunconverted(ans2) || return ans2 +Return `nothing` on decline and otherwise the converted value directly, except +successful `nothing` and `Some` values are wrapped in `Some`. After checking for +failure, `something(result)` recovers the successful value, which is a `T`. +Exceptions from conversion rules propagate. +""" +function pytryconvert(ctx::PyConvertContext, ::Type{T}, x) where T + @pyregion begin + ans = pytryconvert_raw(ctx,T,x) + ans === PYCONVERT_FAILED && return nothing + pyconvert_return(pyconvert_unpack(ans)::T)::pyconvert_returntype(T) end - - return pyconvert_unconverted() +end +pytryconvert(::Type{T}, x; prefer=()) where T = pytryconvert(PyConvertContext(;prefer),T,x) + +"""Inspect source-matching rules in global order, including destination eligibility.""" +function pyconvert_rules(R::Type, x) + t = pytype(Py(x)) + pyconvert_resolve_pending!() + pyconvert_order!() + [(; r.id, r.source, output=r.type, r.scope, + priority=only(s.priority for s in PYCONVERT_SOURCES if s.name==r.source), + order=i, eligible=pyconvert_eligible(r,R)) + for (i,r) in enumerate(PYCONVERT_ORDER) if pyconvert_matches(r.source,t)] +end +function pyconvert_preferred_type(t::Py) + pyconvert_resolve_pending!() + pyconvert_order!() + for r in PYCONVERT_ORDER + pyconvert_eligible(r,Any) && pyconvert_matches(r.source,t) && return r.type + end + Py end """ @pyconvert(T, x, [onfail]) + @pyconvert(ctx, T, x, onfail) -Convert the Python object `x` to a `T`. - -On failure, evaluates to `onfail`, which defaults to `return pyconvert_unconverted()` (mainly useful for writing conversion rules). +Convert a child in a rule, returning failure from the rule unless onfail is given. """ -macro pyconvert(T, x, onfail = :(return $pyconvert_unconverted())) +macro pyconvert(args...) + if length(args) == 4 + ctx,T,x,onfail = args + else + T,x = args[1:2] + ctx = :($PYCONVERT_DEFAULT_CONTEXT) + onfail = length(args)==3 ? args[3] : :(return nothing) + end quote @pyregion begin - T = $(esc(T)) - x = $(esc(x)) - ans = pytryconvert(T, x) - if pyconvert_isunconverted(ans) - $(esc(onfail)) - else - pyconvert_result(T, ans) - end + target = $(esc(T)) + ans = pytryconvert($(esc(ctx)),target,$(esc(x))) + ans === nothing ? $(esc(onfail)) : pyconvert_result(target,ans) end end end """ - pyconvert(T, x, [d]) - -Convert the Python object `x` to a `T`. - -If `d` is specified, it is returned on failure instead of throwing an error. -""" -pyconvert(::Type{T}, x) where {T} = @autopy x @pyconvert T x_ error( - "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", -) -pyconvert(::Type{T}, x, d) where {T} = @autopy x @pyconvert T x_ d + pyconvert(T, x, [default]; prefer=()) + pyconvert(ctx::PyConvertContext, T, x, [default]) +Convert to T, preferring native containers. Source-specific preferences run in +order before ordinary conversion. Failure throws, or returns default if supplied. """ - pyconvertarg(T, x, name) - -Convert the Python object `x` to a `T`. - -On failure, throws a Python `TypeError` saying that the argument `name` could not be converted. -""" -pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin - errset( - pybuiltins.TypeError, - "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", - ) - pythrow() +function pyconvert(ctx::PyConvertContext, ::Type{T}, x) where T + @pyregion begin + ans = pytryconvert_raw(ctx,T,x) + ans === PYCONVERT_FAILED && error("cannot convert Python object to Julia '$T'") + pyconvert_unpack(ans)::T + end +end +function pyconvert(ctx::PyConvertContext, ::Type{T}, x, default) where T + @pyregion begin + ans = pytryconvert_raw(ctx,T,x) + ans === PYCONVERT_FAILED ? default : pyconvert_unpack(ans)::T + end +end +pyconvert(::Type{T}, x; prefer=()) where T = pyconvert(PyConvertContext(;prefer),T,x) +pyconvert(::Type{T}, x, default; prefer=()) where T = pyconvert(PyConvertContext(;prefer),T,x,default) +function pyconvertarg(::Type{T}, x, name) where T + @autopy x @pyconvert T x_ begin + errset(pybuiltins.TypeError,"Cannot convert argument '$name' to a Julia '$T'") + pythrow() + end end function init_pyconvert() - push!(PYCONVERT_EXTRATYPES, pyimport("io" => "IOBase")) - push!( - PYCONVERT_EXTRATYPES, - pyimport("numbers" => ("Number", "Complex", "Real", "Rational", "Integral"))..., - ) - push!( - PYCONVERT_EXTRATYPES, - pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))..., - ) - - priority = PYCONVERT_PRIORITY_CANONICAL - pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) - pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool, priority) - pyconvert_add_rule("builtins:float", Float64, pyconvert_rule_float, priority) + pyimport("types") + pyimport("numbers") + pyimport("collections.abc") + pyimport("io") + pyconvert_source!("builtins:object") + pyconvert_add_rule("types:NoneType", Nothing, pyconvert_rule_none; scope=Any) + pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool; scope=Any) + pyconvert_add_rule("builtins:float", Float64, pyconvert_rule_float; scope=Any) pyconvert_add_rule( "builtins:complex", Complex{Float64}, - pyconvert_rule_complex, - priority, + pyconvert_rule_complex; + scope = Any, ) - pyconvert_add_rule("numbers:Integral", Integer, pyconvert_rule_int, priority) - pyconvert_add_rule("builtins:str", String, pyconvert_rule_str, priority) + pyconvert_add_rule("numbers:Integral", Integer, pyconvert_rule_int; scope=Any) + pyconvert_add_rule("builtins:str", String, pyconvert_rule_str; scope=Any) pyconvert_add_rule( "builtins:bytes", Base.CodeUnits{UInt8,String}, - pyconvert_rule_bytes, - priority, + pyconvert_rule_bytes; + scope = Any, ) pyconvert_add_rule( "builtins:range", StepRange{<:Integer,<:Integer}, - pyconvert_rule_range, - priority, + pyconvert_rule_range; + scope = Any, ) pyconvert_add_rule( "numbers:Rational", Rational{<:Integer}, - pyconvert_rule_fraction, - priority, + pyconvert_rule_fraction; + scope = Any, ) - pyconvert_add_rule("builtins:tuple", NamedTuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("builtins:tuple", Tuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) - pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority) - pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority) + pyconvert_add_rule("builtins:tuple", NamedTuple, pyconvert_rule_iterable; scope=Any) + pyconvert_add_rule("builtins:tuple", Tuple, pyconvert_rule_iterable; scope=Any) + pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime; scope=Any) + pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date; scope=Any) + pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time; scope=Any) pyconvert_add_rule( "datetime:timedelta", Microsecond, - pyconvert_rule_timedelta, - priority, + pyconvert_rule_timedelta; + scope = Any, ) pyconvert_add_rule( "builtins:BaseException", PyException, - pyconvert_rule_exception, - priority, + pyconvert_rule_exception; + scope = Any, ) - priority = PYCONVERT_PRIORITY_NORMAL - pyconvert_add_rule("builtins:NoneType", Missing, pyconvert_rule_none, priority) - pyconvert_add_rule("builtins:bool", Number, pyconvert_rule_bool, priority) - pyconvert_add_rule("numbers:Real", Number, pyconvert_rule_float, priority) - pyconvert_add_rule("builtins:float", Nothing, pyconvert_rule_float, priority) - pyconvert_add_rule("builtins:float", Missing, pyconvert_rule_float, priority) - pyconvert_add_rule("numbers:Complex", Number, pyconvert_rule_complex, priority) - pyconvert_add_rule("numbers:Integral", Number, pyconvert_rule_int, priority) - pyconvert_add_rule("builtins:str", Symbol, pyconvert_rule_str, priority) - pyconvert_add_rule("builtins:str", Char, pyconvert_rule_str, priority) - pyconvert_add_rule("builtins:bytes", Vector{UInt8}, pyconvert_rule_bytes, priority) + pyconvert_add_rule("types:NoneType", Missing, pyconvert_rule_none) + pyconvert_add_rule("builtins:bool", Number, pyconvert_rule_bool) + # Real/Complex ABC membership does not promise lossless __float__/__complex__. + # Integral and Rational sources have their own exact representation rules. + pyconvert_add_rule("builtins:float", Number, pyconvert_rule_float) + pyconvert_add_rule("builtins:float", Nothing, pyconvert_rule_float) + pyconvert_add_rule("builtins:float", Missing, pyconvert_rule_float) + pyconvert_add_rule("builtins:complex", Number, pyconvert_rule_complex) + pyconvert_add_rule("numbers:Integral", Number, pyconvert_rule_int) + pyconvert_add_rule("builtins:str", Symbol, pyconvert_rule_str) + pyconvert_add_rule("builtins:str", Char, pyconvert_rule_str) + pyconvert_add_rule("builtins:bytes", Vector{UInt8}, pyconvert_rule_bytes) pyconvert_add_rule( "builtins:range", UnitRange{<:Integer}, - pyconvert_rule_range, - priority, + pyconvert_rule_range; + scope = AbstractRange, ) - pyconvert_add_rule("numbers:Rational", Number, pyconvert_rule_fraction, priority) + pyconvert_add_rule("numbers:Rational", Number, pyconvert_rule_fraction) pyconvert_add_rule( "collections.abc:Iterable", Vector, pyconvert_rule_iterable, - priority, ) - pyconvert_add_rule("collections.abc:Iterable", Tuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Iterable", Pair, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Iterable", Set, pyconvert_rule_iterable, priority) + pyconvert_add_rule("collections.abc:Iterable", Tuple, pyconvert_rule_iterable) + pyconvert_add_rule("collections.abc:Iterable", Pair, pyconvert_rule_iterable) + pyconvert_add_rule("collections.abc:Iterable", Set, pyconvert_rule_iterable) pyconvert_add_rule( "collections.abc:Sequence", Vector, pyconvert_rule_iterable, - priority, ) - pyconvert_add_rule("collections.abc:Sequence", Tuple, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Set", Set, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Mapping", Dict, pyconvert_rule_mapping, priority) + pyconvert_add_rule("collections.abc:Sequence", Tuple, pyconvert_rule_iterable) + pyconvert_add_rule("collections.abc:Set", Set, pyconvert_rule_iterable) + pyconvert_add_rule("collections.abc:Mapping", Dict, pyconvert_rule_mapping) pyconvert_add_rule( "datetime:timedelta", Millisecond, pyconvert_rule_timedelta, - priority, ) - pyconvert_add_rule("datetime:timedelta", Second, pyconvert_rule_timedelta, priority) - pyconvert_add_rule("datetime:timedelta", Nanosecond, pyconvert_rule_timedelta, priority) + pyconvert_add_rule("datetime:timedelta", Second, pyconvert_rule_timedelta) + pyconvert_add_rule("datetime:timedelta", Nanosecond, pyconvert_rule_timedelta) + + pyconvert_add_rule("builtins:list", Vector, pyconvert_rule_iterable; scope=Any) + pyconvert_add_rule("builtins:dict", Dict, pyconvert_rule_mapping; scope=Any) + for source in ("builtins:set", "builtins:frozenset") + pyconvert_add_rule(source, Set, pyconvert_rule_iterable; scope=Any) + end + # Independent protocol matchers retain their definition-order precedence. + for source in PYCONVERT_PROTOCOLS + pyconvert_add_rule(source, Array, pyconvert_rule_array; scope=Any) + end +end - priority = PYCONVERT_PRIORITY_FALLBACK - pyconvert_add_rule("builtins:object", Py, pyconvert_rule_object, priority) +# Wrap calls this after registering explicit PyArray rules. Within each protocol +# source, wrapper requests must try those before the generic AbstractArray rule. +function init_pyconvert_array_fallbacks() + for source in PYCONVERT_PROTOCOLS + pyconvert_add_rule(source, AbstractArray, pyconvert_rule_array) + end end diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index 7d514c7f..4a1bf18f 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -58,13 +58,20 @@ pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = ### int -pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = @pyregion begin +@inline function pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} + result = pyconvert_rule_int_raw(T, x) + result === PYCONVERT_FAILED ? nothing : pyconvert_return(pyconvert_unpack(result)) +end + +# The dispatcher can use the same numeric conversion without making a Some +# escape the kernel's return boundary (notably Some{BigInt}). +pyconvert_rule_int_raw(::Type{T}, x::Py) where {T<:Number} = @pyregion begin # First try to convert to Clonglong (or Culonglong if unsigned). Conversion rules # run outside their caller's region because user-defined rules may yield. v = T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : C.PyLong_AsLongLong(x) if !iserrset_ambig(v) # success - return pyconvert_tryconvert(T, v) + return pyconvert_raw(pyconvert_tryconvert(T, v)) elseif errmatches(pybuiltins.OverflowError) # overflows Clonglong or Culonglong errclear() @@ -84,14 +91,14 @@ pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = @pyregion begin typemin(typeof(v)) ≤ typemin(T) && typemax(T) ≤ typemax(typeof(v)) # definitely overflows S, give up now - return pyconvert_unconverted() + return PYCONVERT_FAILED else # try converting -> int -> str -> BigInt -> T x_int = pyint(x) x_str = pystr(String, x_int) unsafe_pydel(x_int) v = parse(BigInt, x_str) - return pyconvert_tryconvert(T, v) + return pyconvert_raw(pyconvert_tryconvert(T, v)) end else # other error @@ -194,29 +201,45 @@ function pyconvert_rule_fraction(::Type{T}, x::Py) where {T<:Number} pyconvert_tryconvert(T, @pyconvert(Rational{<:Integer}, x)) end +### arrays + +function pyconvert_rule_array(::Type{A}, x::Py) where {A<:AbstractArray} + r = PythonCall.Wrap.pyarray_make(PyArray, x) + if pyconvert_isunconverted(r) + return pyconvert_unconverted() + else + return pyconvert_tryconvert(A, pyconvert_result(PyArray, r)) + end +end + ### collections # Vector -function _pyconvert_rule_iterable(ans::Vector{T0}, it::Py, ::Type{T1}) where {T0,T1} +_pyconvert_join(A, B, bound) = typeintersect(Base.promote_typejoin(A, B), bound) + +function _pyconvert_rule_iterable(ctx, ans::Vector{T0}, it::Py, ::Type{T1}) where {T0,T1} @label again x_ = unsafe_pynext(it) if pyisnull(x_) unsafe_pydel(it) - return pyconvert_return(ans) + return pyconvert_return(T0 === Union{} ? Vector{T1}() : ans) end - x = @pyconvert(T1, x_) + result = pytryconvert_raw(ctx, T1, x_) + result === PYCONVERT_FAILED && return nothing + x = pyconvert_unpack(result)::T1 if x isa T0 push!(ans, x) @goto again end - T2 = Utils._promote_type_bounded(T0, typeof(x), T1) + T2 = _pyconvert_join(T0, typeof(x), T1) ans2 = Vector{T2}(ans) push!(ans2, x) - return _pyconvert_rule_iterable(ans2, it, T1) + return _pyconvert_rule_iterable(ctx, ans2, it, T1) end function pyconvert_rule_iterable( + ctx::PyConvertContext, ::Type{R}, x::Py, ::Type{Vector{T0}} = Utils._type_lb(R), @@ -224,9 +247,12 @@ function pyconvert_rule_iterable( ) where {R<:Vector,T0,T1} it = pyiter(x) ans = Vector{T0}() - return _pyconvert_rule_iterable(ans, it, T1) + return _pyconvert_rule_iterable(ctx, ans, it, T1) end +pyconvert_rule_iterable(ctx::PyConvertContext, ::Type{R}, x::Py) where {R} = + pyconvert_rule_iterable(R, x) + # Set function _pyconvert_rule_iterable(ans::Set{T0}, it::Py, ::Type{T1}) where {T0,T1} @@ -261,6 +287,7 @@ end # Dict function _pyconvert_rule_mapping( + ctx::PyConvertContext, ans::Dict{K0,V0}, x::Py, it::Py, @@ -271,23 +298,28 @@ function _pyconvert_rule_mapping( k_ = unsafe_pynext(it) if pyisnull(k_) unsafe_pydel(it) - return pyconvert_return(ans) + return pyconvert_return(isempty(ans) ? Dict{K0 === Union{} ? K1 : K0,V0 === Union{} ? V1 : V0}() : ans) end v_ = pygetitem(x, k_) - k = @pyconvert(K1, k_) - v = @pyconvert(V1, v_) + kr = pytryconvert_raw(ctx, K1, k_) + kr === PYCONVERT_FAILED && return nothing + vr = pytryconvert_raw(ctx, V1, v_) + vr === PYCONVERT_FAILED && return nothing + k = pyconvert_unpack(kr)::K1 + v = pyconvert_unpack(vr)::V1 if k isa K0 && v isa V0 push!(ans, k => v) @goto again end - K2 = Utils._promote_type_bounded(K0, typeof(k), K1) - V2 = Utils._promote_type_bounded(V0, typeof(v), V1) + K2 = _pyconvert_join(K0, typeof(k), K1) + V2 = _pyconvert_join(V0, typeof(v), V1) ans2 = Dict{K2,V2}(ans) push!(ans2, k => v) - return _pyconvert_rule_mapping(ans2, x, it, K1, V1) + return _pyconvert_rule_mapping(ctx, ans2, x, it, K1, V1) end function pyconvert_rule_mapping( + ctx::PyConvertContext, ::Type{R}, x::Py, ::Type{Dict{K0,V0}} = Utils._type_lb(R), @@ -295,7 +327,7 @@ function pyconvert_rule_mapping( ) where {R<:Dict,K0,V0,K1,V1} it = pyiter(x) ans = Dict{K0,V0}() - return _pyconvert_rule_mapping(ans, x, it, K1, V1) + return _pyconvert_rule_mapping(ctx, ans, x, it, K1, V1) end # Tuple diff --git a/src/Core/Py.jl b/src/Core/Py.jl index b769a250..cf7f4d76 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -28,6 +28,7 @@ py_finalizer(x::Py) = GC.enqueue(getptr(x)) ispy(::Py) = true getptr(x::Py) = C.PyPtr(getfield(x, :ptr)) +# Py is the identity/fallback representation, not a Python-object transform. pyconvert(::Type{Py}, x::Py) = x setptr!(x::Py, ptr::C.PyPtr) = (setfield!(x, :ptr, Ptr{Cvoid}(ptr)); x) diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index e736bdb0..6f56692d 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -26,8 +26,7 @@ function init_base() pyjuliacallmodule._init_jlwrap() # conversion rule - priority = PYCONVERT_PRIORITY_WRAP - pyconvert_add_rule("juliacall:JlBase", Any, pyconvert_rule_jlvalue, priority) + pyconvert_add_rule("juliacall:JlBase", Any, pyconvert_rule_jlvalue; scope=Any) end pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index dc87f16f..0d20777d 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -18,14 +18,12 @@ end pyconvert_rule_array_nocopy(::Type{A}, x::Py) where {A<:PyArray} = pyarray_make(A, x, copy = false) -function pyconvert_rule_array(::Type{A}, x::Py) where {A<:AbstractArray} - r = pyarray_make(PyArray, x) - if pyconvert_isunconverted(r) - return pyconvert_unconverted() - else - return pyconvert_tryconvert(A, pyconvert_result(PyArray, r)) - end +# Expected unsupported representations may decline; arbitrary errors must propagate. +struct PyArrayUnsupported <: Exception + message::String end +Base.showerror(io::IO, exc::PyArrayUnsupported) = print(io, exc.message) +pyarray_unsupported(message) = throw(PyArrayUnsupported(message)) abstract type PyArraySource end @@ -42,6 +40,7 @@ function pyarray_make( try return pyarray_make(A, x, PyArraySource_ArrayStruct(x, xa)) catch exc + exc isa PyArrayUnsupported || rethrow() @debug "failed to make PyArray from __array_struct__" exc = exc end end @@ -49,6 +48,7 @@ function pyarray_make( try return pyarray_make(A, x, PyArraySource_ArrayInterface(x, xi)) catch exc + exc isa PyArrayUnsupported || rethrow() @debug "failed to make PyArray from __array_interface__" exc = exc end end @@ -56,6 +56,7 @@ function pyarray_make( try return pyarray_make(A, x, PyArraySource_Buffer(x)) catch exc + exc isa PyArrayUnsupported || rethrow() @debug "failed to make PyArray from buffer" exc = exc end end @@ -65,6 +66,7 @@ function pyarray_make( try return pyarray_make(A, y, PyArraySource_ArrayStruct(y, ya)) catch exc + exc isa PyArrayUnsupported || rethrow() @debug "failed to make PyArray from __array__().__array_interface__" exc = exc end @@ -73,6 +75,7 @@ function pyarray_make( try return pyarray_make(A, y, PyArraySource_ArrayInterface(y, yi)) catch exc + exc isa PyArrayUnsupported || rethrow() @debug "failed to make PyArray from __array__().__array_interface__" exc = exc end @@ -93,13 +96,13 @@ function pyarray_make( if T0 == T1 T = T1 R = pyarray_get_R(T) - R′ == R || error("eltype $T is incompatible with array data of type $R′") + R′ == R || pyarray_unsupported("eltype $T is incompatible with array data of type $R′") else T = pyarray_get_T(R′, T0, T1)::DataType - T <: T1 || error("computed eltype $T out of bounds, should be <: $T1") - T >: T0 || error("computed eltype $T out of bounds, should be >: $T0") + T <: T1 || pyarray_unsupported("computed eltype $T out of bounds, should be <: $T1") + T >: T0 || pyarray_unsupported("computed eltype $T out of bounds, should be >: $T0") R = pyarray_get_R(T) - R′ == R || error("computed eltype $T is incompatible with array data of type $R′") + R′ == R || pyarray_unsupported("computed eltype $T is incompatible with array data of type $R′") end # ptr ptr = pyarray_get_ptr(info)::Ptr{Cvoid} @@ -107,7 +110,7 @@ function pyarray_make( N′ = pyarray_get_N(info)::Int if N0 == N1 N = N1 isa Int ? N1 : Int(N1) - N == N′ || error("number of dimensions $N incorrect, expecting $N′") + N == N′ || pyarray_unsupported("number of dimensions $N incorrect, expecting $N′") else N = N′ end @@ -122,11 +125,11 @@ function pyarray_make( Flist = Symbol[] if F0 == F1 F = F1::Tuple{Vararg{Symbol}} - (:mutable in F) && (!M) && error(":mutable flag given but array is not mutable") - (:linear in F) && (!L) && error(":linear flag given but array is not evenly spaced") + (:mutable in F) && (!M) && pyarray_unsupported(":mutable flag given but array is not mutable") + (:linear in F) && (!L) && pyarray_unsupported(":linear flag given but array is not evenly spaced") (:contiguous in F) && (!C) && - error(":contiguous flag given but array is not contiguous") + pyarray_unsupported(":contiguous flag given but array is not contiguous") else M && push!(Flist, :mutable) L && push!(Flist, :linear) @@ -153,9 +156,9 @@ function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__) # offset # TODO: how is the offset measured? offset = pyconvert(Int, @py d.get("offset", 0)) - offset == 0 || error("not supported: non-zero offset") + offset == 0 || pyarray_unsupported("not supported: non-zero offset") # mask - @py("mask" in d) && error("not supported: mask") + @py("mask" in d) && pyarray_unsupported("not supported: mask") # data data = @py d.get("data") if pyistuple(data) @@ -180,14 +183,14 @@ pyarray_typestrdescr_to_type(ts::String, descr::Py) = begin bs = bsc == '<' ? !Utils.islittleendian() : bsc == '>' ? Utils.islittleendian() : - bsc == '|' ? false : error("endianness character not supported: $ts") - bs && error("byte-swapping not supported: $ts") + bsc == '|' ? false : pyarray_unsupported("endianness character not supported: $ts") + bs && pyarray_unsupported("byte-swapping not supported: $ts") # element type etc = ts[2] if etc == 'b' sz = parse(Int, ts[3:end]) sz == sizeof(Bool) && return Bool - error("bool of this size not supported: $ts") + pyarray_unsupported("bool of this size not supported: $ts") elseif etc == 'i' sz = parse(Int, ts[3:end]) sz == 1 && return Int8 @@ -195,7 +198,7 @@ pyarray_typestrdescr_to_type(ts::String, descr::Py) = begin sz == 4 && return Int32 sz == 8 && return Int64 sz == 16 && return Int128 - error("signed int of this size not supported: $ts") + pyarray_unsupported("signed int of this size not supported: $ts") elseif etc == 'u' sz = parse(Int, ts[3:end]) sz == 1 && return UInt8 @@ -203,36 +206,36 @@ pyarray_typestrdescr_to_type(ts::String, descr::Py) = begin sz == 4 && return UInt32 sz == 8 && return UInt64 sz == 16 && return UInt128 - error("unsigned int of this size not supported: $ts") + pyarray_unsupported("unsigned int of this size not supported: $ts") elseif etc == 'f' sz = parse(Int, ts[3:end]) sz == 2 && return Float16 sz == 4 && return Float32 sz == 8 && return Float64 - error("float of this size not supported: $ts") + pyarray_unsupported("float of this size not supported: $ts") elseif etc == 'c' sz = parse(Int, ts[3:end]) sz == 4 && return Complex{Float16} sz == 8 && return Complex{Float32} sz == 16 && return Complex{Float64} - error("complex of this size not supported: $ts") + pyarray_unsupported("complex of this size not supported: $ts") elseif etc == 'U' sz = parse(Int, ts[3:end]) return Utils.StaticString{UInt32,sz} elseif etc == 'O' return UnsafePyObject elseif etc == 'V' - pyisnull(descr) && error("not supported: void dtype with null descr") + pyisnull(descr) && pyarray_unsupported("not supported: void dtype with null descr") sz = parse(Int, ts[3:end]) T = pyarray_descr_to_type(descr) sizeof(T) == sz || - error("size mismatch: itemsize=$sz but sizeof(descr)=$(sizeof(T))") + pyarray_unsupported("size mismatch: itemsize=$sz but sizeof(descr)=$(sizeof(T))") return T elseif etc == 'M' || etc == 'm' m = match(r"^([0-9]+)(\[([0-9]+)?([a-zA-Z]+)\])?$", ts[3:end]) - m === nothing && error("could not parse type: $ts") + m === nothing && pyarray_unsupported("could not parse type: $ts") sz = parse(Int, m[1]) - sz == sizeof(Int64) || error( + sz == sizeof(Int64) || pyarray_unsupported( "$(etc == 'M' ? "datetime" : "timedelta") of this size not supported: $sz", ) s = m[3] === nothing ? 1 : parse(Int, m[3]) @@ -252,14 +255,14 @@ pyarray_typestrdescr_to_type(ts::String, descr::Py) = begin us == "ps" ? NumpyDates.PICOSECONDS : us == "fs" ? NumpyDates.FEMTOSECONDS : us == "as" ? NumpyDates.ATTOSECONDS : - error("not supported: $(etc == 'M' ? "datetime" : "timedelta") unit: $us") + pyarray_unsupported("not supported: $(etc == 'M' ? "datetime" : "timedelta") unit: $us") if etc == 'M' return NumpyDates.InlineDateTime64{s == 1 ? u : (u, s)} else return NumpyDates.InlineTimeDelta64{s == 1 ? u : (u, s)} end else - error("not supported: dtype of kind: $(repr(etc))") + pyarray_unsupported("not supported: dtype of kind: $(repr(etc))") end end @@ -308,9 +311,9 @@ function pyarray_descr_to_type(descr::Py) # TODO: support non-aligned dtypes by packing them into a custom type and reinterpreting T = Tuple{ftypes...} for (i, o) in pairs(foffsets) - fieldoffset(T, i) == o || error("not supported: dtype that is not aligned: $descr") + fieldoffset(T, i) == o || pyarray_unsupported("not supported: dtype that is not aligned: $descr") end - sizeof(T) == curoffset || error("not supported: dtype with end padding: $descr") + sizeof(T) == curoffset || pyarray_unsupported("not supported: dtype with end padding: $descr") # return the tuple type if the field names are f0, f1, ..., else return a named tuple if fnames == [Symbol(:f, i - 1) for i = 1:length(fnames)] return T @@ -371,14 +374,14 @@ end function pyarray_get_R(src::PyArraySource_ArrayStruct) swapped = !Utils.isflagset(src.info.flags, C.NPY_ARRAY_NOTSWAPPED) hasdescr = Utils.isflagset(src.info.flags, C.NPY_ARR_HAS_DESCR) - swapped && error("byte-swapping not supported") + swapped && pyarray_unsupported("byte-swapping not supported") kind = src.info.typekind size = src.info.itemsize if kind == 98 # b = bool if size == sizeof(Bool) return Bool else - error("bool of this size not supported: $size") + pyarray_unsupported("bool of this size not supported: $size") end elseif kind == 105 # i = int if size == 1 @@ -390,7 +393,7 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) elseif size == 8 return Int64 else - error("int of this size not supported: $size") + pyarray_unsupported("int of this size not supported: $size") end elseif kind == 117 # u = uint if size == 1 @@ -402,7 +405,7 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) elseif size == 8 return UInt64 else - error("uint of this size not supported: $size") + pyarray_unsupported("uint of this size not supported: $size") end elseif kind == 102 # f = float if size == 2 @@ -412,7 +415,7 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) elseif size == 8 return Float64 else - error("float of this size not supported: $size") + pyarray_unsupported("float of this size not supported: $size") end elseif kind == 99 # c = complex if size == 4 @@ -423,29 +426,29 @@ function pyarray_get_R(src::PyArraySource_ArrayStruct) return ComplexF64 end elseif kind == 109 # m = timedelta - error("timedelta not supported") + pyarray_unsupported("timedelta not supported") elseif kind == 77 # M = datetime - error("datetime not supported") + pyarray_unsupported("datetime not supported") elseif kind == 79 # O = object if size == sizeof(C.PyPtr) return UnsafePyObject else - error("object pointer of this size not supported: $size") + pyarray_unsupported("object pointer of this size not supported: $size") end elseif kind == 83 # S = byte string - error("byte strings not supported") + pyarray_unsupported("byte strings not supported") elseif kind == 85 # U = unicode string - mod(size, 4) == 0 || error("unicode size must be a multiple of 4: $size") + mod(size, 4) == 0 || pyarray_unsupported("unicode size must be a multiple of 4: $size") return Utils.StaticString{UInt32,div(size, 4)} elseif kind == 86 # V = void (should have descr) - hasdescr || error("not supported: void dtype with no descr") + hasdescr || pyarray_unsupported("not supported: void dtype with no descr") descr = pynew(incref(src.info.descr)) T = pyarray_descr_to_type(descr) sizeof(T) == size || - error("size mismatch: itemsize=$size but sizeof(descr)=$(sizeof(T))") + pyarray_unsupported("size mismatch: itemsize=$size but sizeof(descr)=$(sizeof(T))") return T else - error("not supported: dtype of kind: $(Char(kind))") + pyarray_unsupported("not supported: dtype of kind: $(Char(kind))") end @assert false end @@ -492,7 +495,7 @@ function PyArraySource_Buffer(x::Py) memview = pybuiltins.memoryview(x) buf = C.UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) buf.suboffsets[] == C_NULL || - error("PyArray does not support buffers with non-trivial suboffsets (PIL-style indirect layout)") + pyarray_unsupported("PyArray does not support buffers with non-trivial suboffsets (PIL-style indirect layout)") PyArraySource_Buffer(x, memview, buf) end @@ -534,7 +537,7 @@ const PYARRAY_BUFFERFORMAT_TO_TYPE = let c = Utils.islittleendian() ? '<' : '>' end pyarray_bufferformat_to_type(fmt::String) = get( - () -> error("not implemented: buffer format $(repr(fmt))"), + () -> pyarray_unsupported("not implemented: buffer format $(repr(fmt))"), PYARRAY_BUFFERFORMAT_TO_TYPE, fmt, ) @@ -681,14 +684,14 @@ end if T0 <: Py <: T1 Py else - error("impossible") + pyarray_unsupported("incompatible object element type") end elseif R <: Tuple - error("not implemented") + pyarray_unsupported("tuple array element types are not implemented") elseif T0 <: R <: T1 R else - error("impossible") + pyarray_unsupported("incompatible array element type") end end @@ -724,6 +727,6 @@ end elseif T <: Tuple Tuple{map(pyarray_get_R, T.parameters)...} else - error("unsupported eltype $T") + pyarray_unsupported("unsupported eltype $T") end end diff --git a/src/Wrap/Wrap.jl b/src/Wrap/Wrap.jl index d3b30ff2..f8633dd3 100644 --- a/src/Wrap/Wrap.jl +++ b/src/Wrap/Wrap.jl @@ -32,55 +32,19 @@ include("PyTable.jl") include("PyPandasDataFrame.jl") function __init__() - priority = PYCONVERT_PRIORITY_ARRAY - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - - priority = PYCONVERT_PRIORITY_CANONICAL - pyconvert_add_rule( - "collections.abc:Iterable", - PyIterable, - pyconvert_rule_iterable, - priority, - ) - pyconvert_add_rule( - "collections.abc:Sequence", - PyList, - pyconvert_rule_sequence, - priority, - ) - pyconvert_add_rule("collections.abc:Set", PySet, pyconvert_rule_set, priority) - pyconvert_add_rule("collections.abc:Mapping", PyDict, pyconvert_rule_mapping, priority) - pyconvert_add_rule("io:IOBase", PyIO, pyconvert_rule_io, priority) - pyconvert_add_rule("_io:_IOBase", PyIO, pyconvert_rule_io, priority) - for typename in ["pandas.core.frame:DataFrame", "pandas:DataFrame"] - pyconvert_add_rule( - typename, - PyPandasDataFrame, - pyconvert_rule_pandasdataframe, - priority, - ) - end - for typename in ["pandas.core.arrays.base:ExtensionArray", "pandas.api.extensions:ExtensionArray"] - pyconvert_add_rule( - typename, - PyList, - pyconvert_rule_sequence, - priority, - ) + for source in Convert.PYCONVERT_PROTOCOLS + pyconvert_add_rule(source, PyArray, pyconvert_rule_array_nocopy) end - - priority = PYCONVERT_PRIORITY_NORMAL - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) + Convert.init_pyconvert_array_fallbacks() + pyconvert_add_rule("collections.abc:Iterable", PyIterable, pyconvert_rule_iterable; scope=Any) + pyconvert_add_rule("collections.abc:Sequence", PyList, pyconvert_rule_sequence) + pyconvert_add_rule("collections.abc:Set", PySet, pyconvert_rule_set) + pyconvert_add_rule("collections.abc:Mapping", PyDict, pyconvert_rule_mapping) + pyconvert_add_rule("io:IOBase", PyIO, pyconvert_rule_io; scope=Any) + # This private C base is distinct from the public io.IOBase ABC. + pyconvert_add_rule("_io:_IOBase", PyIO, pyconvert_rule_io; scope=Any) + pyconvert_add_rule("pandas:DataFrame", PyPandasDataFrame, pyconvert_rule_pandasdataframe; scope=Any) + pyconvert_add_rule("pandas.api.extensions:ExtensionArray", PyList, pyconvert_rule_sequence; scope=Any) end end diff --git a/test/Convert.jl b/test/Convert.jl index 2765440f..55fbc491 100644 --- a/test/Convert.jl +++ b/test/Convert.jl @@ -334,6 +334,7 @@ end "$(t.__module__):$(t.__qualname__)", String, (_, _) -> "Hello!!", + ; scope=Any, ) @test pyconvert(String, x) == "Hello!!" @test pyconvert(Any, x) == "Hello!!" # Broken before PR #365 diff --git a/test/ConvertRedesign.jl b/test/ConvertRedesign.jl new file mode 100644 index 00000000..018291aa --- /dev/null +++ b/test/ConvertRedesign.jl @@ -0,0 +1,596 @@ +@testitem "conversion registry, scopes and global ordering" begin + using PythonCall + const CV = PythonCall.Convert + ns = pyimport("types").ModuleType("convert_redesign_test") + pyimport("sys").modules["convert_redesign_test"] = ns + pyexec("""class A: pass +class B: pass +class C(A, B): pass +class D(C): pass +class E(B, A): pass +class Protocol: + __array_interface__ = {} +class Both(Protocol): + def __array__(self): pass +""", ns.__dict__) + ids = Int[] + add(name,T,f;scope=T) = (id=pyconvert_add_rule("convert_redesign_test:"*name,T,f;scope); push!(ids,id); id) + try + b = add("B",String, (_,_) -> "B"; scope=Any) + a = add("A",String, (_,_) -> "A"; scope=Any) + @test pyconvert(Any, ns.C()) == "B" + @test pyconvert(Any, ns.D()) == "B" + @test pyconvert(Any, ns.E()) == "B" # opposite MRO + selected(x) = [r.id for r in pyconvert_rules(Any,x) if r.eligible] + @test selected(ns.C()) == selected(ns.D()) == selected(ns.E()) + c = add("C",String, (_,_) -> "C"; scope=Any) + @test pyconvert(Any,ns.D()) == "C" + @test pyconvert_replace_rule(c,"convert_redesign_test:C",String,(_,_) -> nothing;scope=Any) == c + @test pyconvert(Any,ns.C()) == "B" + duplicate = add("B",String,(_,_) -> "later"; scope=Any) + @test duplicate > c > a > b + @test pyconvert(Any,ns.C()) == "B" + pyconvert_remove_rule(b); filter!(!=(b),ids) + @test pyconvert(Any,ns.C()) == "later" + @test_throws KeyError pyconvert_remove_rule(b) + scoped = add("A",Vector,(_,_) -> [1]; scope=Vector) + @test pyconvert(Vector,ns.A()) == [1] + @test pyconvert(Any,ns.A()) == "A" + @test pytryconvert(AbstractArray,ns.A()) === nothing + @test pyconvert(Union{Vector,Nothing},ns.A()) == [1] + @test pytryconvert(Union{AbstractArray,Vector},ns.A()) === nothing + @test pytryconvert(Union{},ns.A()) === nothing + @test all(!r.eligible for r in pyconvert_rules(Any,ns.A()) if r.id==scoped) + @test CV.pyconvert_protocol("",ns.Both) + @test CV.pyconvert_protocol("",ns.Both) + @test !CV.pyconvert_protocol("",ns.Both) + # Explicit rule outranks its generic virtual protocol parent. + p = add("Both",String,(_,_) -> "specific";scope=Any) + @test pyconvert(Any,ns.Both()) == "specific" + rows = pyconvert_rules(Any,ns.Both()) + @test first(rows).id == p + @test first(rows).priority >= maximum(r.priority for r in rows) + finally + foreach(pyconvert_remove_rule,ids) + pyimport("sys").modules.pop("convert_redesign_test") + end +end + +@testitem "native containers and preferences" begin + using PythonCall + @test parentmodule(PyConvertContext) === PythonCall + # Explicit wrappers must precede the generic native-array fallback even + # though the registrations are initialized by different modules. + CV = PythonCall.Convert + for source in CV.PYCONVERT_PROTOCOLS + rules = filter(r->r.source==source,CV.PYCONVERT_RULES) + @test [r.type for r in rules] == [Array,PyArray,AbstractArray] + @test first(filter(r->CV.pyconvert_eligible(r,PyArray),rules)).type === PyArray + end + cases = [(Any[],Any), ([1,2],Int), (Any[1,2.5],Real), + ([1,"two"],Any), ([nothing,1],Union{Nothing,Int})] + for (values,T) in cases + result = pyconvert(Any,pylist(values)) + @test result isa Vector{T} + @test isequal(result,values) + end + @test pyconvert(Any,pylist([big(2)^200,1])) isa Vector{<:Integer} + @test pyconvert(Vector{Int},pylist([1,2])) == [1,2] + @test pyconvert(Vector{<:Real},pylist(Any[1,2.5])) isa Vector{Real} + @test typeof.(pyconvert(Any,pylist(Any[1,2.5]))) == [Int,Float64] + @test pytryconvert(Vector{Int},pylist([1,"bad"])) === nothing + @test pyconvert(Any,pydict()) isa Dict{Any,Any} + d = pyconvert(Any,pydict("a"=>1,"b"=>2.5)) + @test d isa Dict{String,Real} + @test d == Dict("a"=>1,"b"=>2.5) + @test pyconvert(Any,pylist([pylist([1,2])])) == [[1,2]] + @test pyconvert(Any,pyiter(pylist([1,2]))) isa PyIterable + @test eltype(pyconvert(Any,pyiter(pylist([1,2])))) === Any + pref = ("builtins:list"=>PyList, "builtins:dict"=>PyDict) + @test pyconvert(Any,pylist([1,2]);prefer=pref) isa PyList + @test pyconvert(Any,pydict();prefer=pref) isa PyDict + @test pyconvert(Vector,pylist([1,2]);prefer=pref) isa Vector + @test pyconvert(Any,pylist([1]);prefer=("builtins:dict"=>PyDict,)) isa Vector + ctx = PyConvertContext(;prefer=pref) + @test pyconvert(ctx,Vector,pylist([pylist([1])]))[1] isa PyList + @test pyconvert(Any,pylist([1]);prefer=("builtins:list"=>Int8,)) == [1] + @test pyconvert(Any,pylist([1]);prefer=("builtins:list"=>PyList,"builtins:list"=>Vector)) isa PyList +end + +@testitem "scopes with parametric bounds and union normalization" begin + using PythonCall + CV = PythonCall.Convert + rule(T,S) = CV.PyConvertRule(1,"builtins:list",T,S,(_,_) -> error("ineligible")) + @test CV.pyconvert_eligible(rule(Vector,AbstractVector),Vector{Int}) + @test CV.pyconvert_eligible(rule(Vector,AbstractVector),Vector{<:Real}) + @test !CV.pyconvert_eligible(rule(Vector,Vector),AbstractVector) + @test !CV.pyconvert_eligible(rule(Vector,Vector),Any) + @test CV.pyconvert_eligible(rule(Vector{<:Real},Any),Vector) + @test !CV.pyconvert_eligible(rule(Vector{String},Any),Vector{<:Real}) + nested = Vector{Vector{T}} where T<:Real + @test CV.pyconvert_eligible(rule(nested,nested),Vector{Vector{Int}}) + @test !CV.pyconvert_eligible(rule(nested,nested),Vector{Vector{String}}) + @test CV.pyconvert_eligible(rule(nested,nested),Union{Nothing,Vector{Vector{Int}}}) + @test !CV.pyconvert_eligible(rule(Vector,Vector),Union{AbstractVector,Vector}) + @test !CV.pyconvert_eligible(rule(Union{},Any),Any) + @test !CV.pyconvert_eligible(rule(Any,Any),Union{}) + @test !CV.pyconvert_eligible(rule(Vector,Union{}),Vector) +end + +@testitem "protocol ordering, ABCs and cached scalar overrides" begin + using PythonCall + CV=PythonCall.Convert + ns=pyimport("types").ModuleType("convert_protocol_test") + pyimport("sys").modules["convert_protocol_test"]=ns + pyexec(""" +import abc +class A(metaclass=abc.ABCMeta): pass +class B: pass +A.register(B) +class C(B): pass +class IntSubclass(int): pass +class FloatSubclass(float): pass +""",ns.__dict__) + ids=Int[] + add(source,T,f;scope=Any)=(id=pyconvert_add_rule(source,T,f;scope);push!(ids,id);id) + try + add("convert_protocol_test:A",String,(_,_) -> "ABC") + @test pyconvert(Any,ns.B()) == pyconvert(Any,ns.C()) == "ABC" + # Prime the scalar cache, then register a stricter source rule. + @test pyconvert(Int,ns.IntSubclass(2)) == 2 + bigrule=add("convert_protocol_test:IntSubclass",BigInt,(_,_) -> big(7);scope=BigInt) + @test pyconvert(BigInt,ns.IntSubclass(2)) == big(7) + @test pyconvert(Any,ns.IntSubclass(2)) === 2 + pyconvert_replace_rule(bigrule,"convert_protocol_test:IntSubclass",BigInt,(_,_) -> nothing;scope=BigInt) + @test pyconvert(BigInt,ns.IntSubclass(2)) == big(2) + pyconvert_remove_rule(bigrule);filter!(!=(bigrule),ids) + i=add("convert_protocol_test:IntSubclass",Int,(_,_) -> 3) + @test pyconvert(Int,ns.IntSubclass(2)) == 3 + pyconvert_replace_rule(i,"convert_protocol_test:IntSubclass",Int,(_,_) -> 4;scope=Any) + @test pyconvert(Int,ns.IntSubclass(2)) == 4 + pyconvert_remove_rule(i);filter!(!=(i),ids) + @test pyconvert(Int,ns.IntSubclass(2)) == 2 + @test pyconvert(Float64,ns.FloatSubclass(2)) == 2.0 + add("convert_protocol_test:FloatSubclass",Float64,(_,_) -> 3.0) + @test pyconvert(Float64,ns.FloatSubclass(2)) == 3.0 + @test pyconvert(Any,pyjl([1,2])) == [1,2] + @test first(pyconvert_rules(Any,pyjl([1,2]))).source=="juliacall:JlBase" + if pyis(pyimport("importlib.util").find_spec("numpy"),pybuiltins.None) + @test_skip false # external interpreters need not have development dependencies + else + pyexec("import numpy\nclass Both(numpy.ndarray): pass",ns.__dict__) + a=ns.numpy.arange(3).view(ns.Both) + @test pyconvert(Any,a) isa Vector{<:Integer} + rows=pyconvert_rules(Any,a) + protocols=unique(r.source for r in rows if startswith(r.source,"<")) + @test collect(protocols) == ["","","",""] + i=add("numpy:ndarray",String,(_,_) -> "ndarray") + @test pyconvert(Any,a) == "ndarray" + rows=pyconvert_rules(Any,a) + @test first(rows).id==i + @test first(rows).priority==maximum(r.priority for r in rows) + @test pyconvert(Any,ns.numpy.arange(2);prefer=("numpy:ndarray"=>PyArray,)) isa PyArray + end + finally + foreach(pyconvert_remove_rule,ids) + pyimport("sys").modules.pop("convert_protocol_test") + end +end + +@testitem "conversion exceptions and explicit context boundaries" begin + using PythonCall + ns=pyimport("types").ModuleType("convert_context_test") + pyimport("sys").modules["convert_context_test"]=ns + pyexec(""" +class Value: + child = [1,2] +class Iterator: + def __iter__(self): + yield 1 + raise RuntimeError('iterator failed') +class BadArray: + def __array__(self): + raise RuntimeError('array failed') +""",ns.__dict__) + source="convert_context_test:Value" + id=pyconvert_add_rule(source,Any,(ctx,T,x)->pyconvert(ctx,Any,x.child);scope=Any) + try + pref=(source=>String,"builtins:list"=>PyList) + # Failed preference for String then ordinary conversion propagates context. + # Restrict the advertised output so the String attempt genuinely declines. + pyconvert_replace_rule(id,source,AbstractVector,(ctx,T,x)->pyconvert(ctx,Any,x.child);scope=Any) + @test pyconvert(Any,ns.Value();prefer=pref) isa PyList + pyconvert_replace_rule(id,source,AbstractVector,(T,x)->pyconvert(Any,x.child);scope=Any) + @test pyconvert(Any,ns.Value();prefer=pref) isa Vector + @test_throws "iterator failed" pyconvert(Vector,ns.Iterator()) + @test pyconvert(Any,ns.Iterator()) isa PyIterable + @test_throws "array failed" pyconvert(Any,ns.BadArray()) + pyconvert_replace_rule(id,source,AbstractVector,(T,x)->error("preference failed");scope=Any) + @test_throws "preference failed" pyconvert(Any,ns.Value();prefer=(source=>Vector,)) + finally + pyconvert_remove_rule(id) + pyimport("sys").modules.pop("convert_context_test") + end +end + +@testitem "rule result convention and context signatures" begin + using PythonCall + const CV = PythonCall.Convert + ns = pyimport("types").ModuleType("convert_result_test") + pyimport("sys").modules["convert_result_test"] = ns + pyexec("class Value: pass",ns.__dict__) + source = "convert_result_test:Value" + x = ns.Value() + id = pyconvert_add_rule(source,Any,(_,_) -> Some(nothing);scope=Any) + try + @test pytryconvert(Any,x) === Some(nothing) + @test pyconvert(Any,x) === nothing + pyconvert_replace_rule(id,source,Any,(_,_) -> missing;scope=Any) + @test pyconvert(Any,x) === missing + @test pytryconvert(Any,x) === missing + pyconvert_replace_rule(id,source,Any,(_,_) -> Some(42);scope=Any) + @test pytryconvert(Int,x) === 42 + @test pytryconvert(Any,x) === 42 + pyconvert_replace_rule(id,source,Any,(_,_) -> Some(Some(2));scope=Any) + @test pyconvert(Any,x) === Some(2) + for value in (CV.PYCONVERT_FAILED, CV.PyConvertEscape(1), Some(nothing)) + pyconvert_replace_rule(id,source,Any,(_,_) -> Some(value);scope=Any) + @test pyconvert(Any,x) === value + expected = value isa Union{Nothing,Some} ? Some(value) : value + @test pytryconvert(Any,x) === expected + @test pyconvert(typeof(value),x) === value + end + # Built-in result encoding must preserve ambiguous values exactly once. + for value in (nothing, missing, Some(1), Some(nothing), Some(Some(1)), + [1,2], CV.PYCONVERT_FAILED, CV.PyConvertEscape(1)) + pyconvert_replace_rule(id,source,Any,(_,_) -> CV.pyconvert_return(value);scope=Any) + @test pyconvert(Any,x) === value + expected = value isa Union{Nothing,Some} ? Some(value) : value + @test pytryconvert(typeof(value),x) === expected + @test pyconvert(Any,pyjl(value)) === value + @test pytryconvert(typeof(value),pyjl(value)) === expected + end + @test pyconvert(Any,pylist([pyjl(Some(1)),pyjl(nothing)])) == [Some(1),nothing] + @test pyconvert(Any,pydict("some"=>pyjl(Some(1)),"none"=>pyjl(nothing))) == + Dict("some"=>Some(1),"none"=>nothing) + pyconvert_replace_rule(id,source,Any,(_,_) -> nothing;scope=Any) + @test pyconvert(Any,x) === x # implicit fallback + @test pytryconvert(Any,x) === x + @test pytryconvert(Int,x) === nothing + pyconvert_replace_rule(id,source,Any,(_,_) -> error("rule error");scope=Any) + @test_throws "rule error" pyconvert(Any,x) + function rule(::Type{Int},x) + 1 + end + function rule(ctx,::Type{String},x) + "context" + end + function rule(::Type{String},x) + "legacy" + end + pyconvert_replace_rule(id,source,Union{Int,String},rule;scope=Any) + @test pyconvert(Int,x) == 1 + @test pyconvert(String,x) == "context" + @test pyconvert(Union{Int,String},x) in (1,"context") + calls=Any[] + pyconvert_replace_rule(id,source,Union{Int,String},(T,x)->(push!(calls,T);nothing);scope=Any) + second=pyconvert_add_rule(source,Union{Int,String},(T,x)->(push!(calls,:second);T===Int ? 1 : "ok");scope=Any) + try + @test pyconvert(Union{Int,String},x) in (1,"ok") + @test Set(calls[1:2]) == Set([Int,String]) + @test calls[3] === :second + finally + pyconvert_remove_rule(second) + end + @test 1 isa CV.pyconvert_returntype(Integer) + @test Some(nothing) isa CV.pyconvert_returntype(Union{Int,Nothing}) + @test Some(1) isa CV.pyconvert_returntype(Any) + @test CV.pyconvert_returntype(Int) === Union{Nothing,Int} + @test CV.pyconvert_returntype(Integer) === Union{Nothing,Integer} + @test CV.pyconvert_returntype(Any) === Any + @test CV.pyconvert_returntype(Union{}) === Nothing + @test CV.pyconvert_returntype(Union{Int,Nothing}) === Union{Nothing,Int,Some{Nothing}} + @test CV.pyconvert_returntype(Some{Int}) === Union{Nothing,Some{Some{Int}}} + @test CV.pyconvert_returntype(Union{Int,Some{Int}}) === Union{Nothing,Int,Some{Some{Int}}} + @test Some(Some(1)) isa CV.pyconvert_returntype(Some{<:Integer}) + @test CV.pyconvert_returntype(Vector{<:Real}) == Union{Nothing,Vector{<:Real}} + @test (@inferred pyconvert(Int,pyint(1))) === 1 + @test (@inferred Nothing pytryconvert(Int,pyint(1))) === 1 + @test (@inferred Nothing pytryconvert(Float64,pyfloat(1.5))) === 1.5 + @test pytryconvert(Union{Nothing,Int},pyint(1)) === 1 + @test pytryconvert(Union{Nothing,Int},pybuiltins.None) === Some(nothing) + @test pytryconvert(Some{Int},pyjl(Some(1))) === Some(Some(1)) + @test pytryconvert(Union{Int,Some{Int}},pyjl(Some(1))) === Some(Some(1)) + @test only(Base.return_types(pytryconvert,Tuple{Type{Integer},Py})) === Union{Nothing,Integer} + @test something(pytryconvert(Int,pystr("bad")),:default) === :default + @test something(pytryconvert(Nothing,pybuiltins.None),:default) === nothing + @test something(pytryconvert(Some{Int},pyjl(Some(1)))) === Some(1) + @test pytryconvert(Nothing,pybuiltins.None) === Some(nothing) + finally + pyconvert_remove_rule(id) + pyimport("sys").modules.pop("convert_result_test") + end +end + +@testitem "scalar dispatch preserves singleton rules and overflow behavior" begin + using PythonCall + CV=PythonCall.Convert + original=only(r for r in CV.PYCONVERT_RULES if r.source=="types:NoneType" && r.type===Nothing) + x=pybuiltins.None + @test pyconvert(Nothing,x) === nothing + calls=Ref(0) + try + pyconvert_replace_rule(original.id,original.source,Nothing,(T,x)->(calls[]+=1;Some(nothing));scope=Any) + @test pyconvert(Nothing,x) === nothing + @test calls[]==1 + pyconvert_replace_rule(original.id,original.source,Nothing,(T,x)->nothing;scope=Any) + @test pytryconvert(Nothing,x) === nothing + @test pyconvert(Nothing,x,:declined) === :declined + pyconvert_replace_rule(original.id,original.source,Nothing,(T,x)->error("singleton rule");scope=Any) + @test_throws "singleton rule" pyconvert(Nothing,x) + finally + pyconvert_replace_rule(original.id,original.source,original.type,original.func;scope=original.scope) + end + @test pytryconvert(Nothing,x) === Some(nothing) + @test (@inferred pyconvert(Float64,pyfloat(1.5))) === 1.5 + @test (@inferred pyconvert(Int,pyint(10000))) === 10000 + for n in (big(2)^200,-big(2)^200,big(typemax(Int))+1) + p=pyint(n) + @test pyconvert(BigInt,p)==n + @test pyconvert(Integer,p)==n + @test pyconvert(Any,p)==n + @test pytryconvert(Int,p) === nothing + end + # An exact integer decline must not retry through a rounded float/complex. + below_min=pyint(big(typemin(Int))-1) + @test CV.pyconvert_rule_int(Int,below_min) === nothing + @test pytryconvert(Int,below_min) === nothing + @test pyconvert(BigInt,pyint(42))==42 + @test pyconvert(UInt,pyint(typemax(UInt)))==typemax(UInt) + @test pytryconvert(UInt,pyint(-1)) === nothing +end + +@testitem "numeric conversion uses source representations" begin + using PythonCall + ns = pyimport("types").ModuleType("convert_numeric_source_test") + pyimport("sys").modules["convert_numeric_source_test"] = ns + pyexec(""" +import numbers +from fractions import Fraction +class Integral(int): + def __float__(self): raise AssertionError('integral __float__ called') + def __complex__(self): raise AssertionError('integral __complex__ called') +class Rational(Fraction): + def __float__(self): raise AssertionError('rational __float__ called') + def __complex__(self): raise AssertionError('rational __complex__ called') +class Float(float): + def __float__(self): raise AssertionError('float override called') + def __complex__(self): raise AssertionError('float complex override called') +class Complex(complex): + def __complex__(self): raise AssertionError('complex override called') +class ApproximateReal: + def __float__(self): raise AssertionError('real __float__ called') + def __complex__(self): raise AssertionError('real __complex__ called') +class ApproximateComplex: + def __complex__(self): raise AssertionError('complex __complex__ called') +numbers.Real.register(ApproximateReal) +numbers.Complex.register(ApproximateComplex) +""", ns.__dict__) + try + n = big(typemin(Int))-1 + for x in (pyint(n), ns.Integral(pyint(n)), ns.Rational(pyint(n),1)) + @test pytryconvert(Int,x) === nothing + @test pyconvert(BigInt,x) == n + @test pyconvert(Float64,x) == Float64(n) + @test pyconvert(ComplexF64,x) == ComplexF64(n) + end + @test pyconvert(Int,ns.Integral(3)) === 3 + @test pyconvert(Int,ns.Rational(6,2)) === 3 + @test pyconvert(Rational,ns.Rational(1,3)) == 1//3 + @test pyconvert(Float64,ns.Rational(1,3)) === Float64(1//3) + @test pytryconvert(Int,ns.Rational(pyint(big(2)^63+1),2)) === nothing + for x in (ns.ApproximateReal(),ns.ApproximateComplex()) + for T in (Int,Float64,ComplexF64,Number) + @test pytryconvert(T,x) === nothing + end + @test pyconvert(Any,x) === x + end + @test pyconvert(Int,pyfloat(3.0)) === 3 + @test pytryconvert(Int,pyfloat(3.5)) === nothing + @test pyconvert(Int,pycomplex(3,0)) === 3 + @test pytryconvert(Int,pycomplex(3,1)) === nothing + @test pyconvert(Any,ns.Float(1.5)) === 1.5 + @test pyconvert(ComplexF64,ns.Float(1.5)) === 1.5+0.0im + @test pyconvert(Any,ns.Complex(1,2)) === 1.0+2.0im + id = pyconvert_add_rule("convert_numeric_source_test:ApproximateReal",Float64,(_,_) -> 1.5) + try + @test pyconvert(Float64,ns.ApproximateReal()) === 1.5 + @test pytryconvert(Number,ns.ApproximateReal()) === nothing + finally + pyconvert_remove_rule(id) + end + # Supported NumPy scalars use their own representation-reading rules. + if pyis(pyimport("importlib.util").find_spec("numpy"),pybuiltins.None) + @test_skip false # NumPy is optional for external-interpreter test jobs + else + np = pyimport("numpy") + @test pyconvert(Int,np.int64(3)) === 3 + @test pyconvert(Float64,np.float32(1.5)) === 1.5 + @test pyconvert(ComplexF64,np.complex64(pycomplex(1,2))) === 1.0+2.0im + @test pytryconvert(Int,np.float64(3.5)) === nothing + end + finally + pyimport("sys").modules.pop("convert_numeric_source_test") + end +end + +@testitem "effective source priorities are monotonic" begin + using PythonCall + CV=PythonCall.Convert + pyconvert_rules(Any,pyint(1)) + active=Set(r.source for r in CV.PYCONVERT_ORDER) + sources=filter(s->s.name in active || s.name=="builtins:object",CV.PYCONVERT_SOURCES) + for a in sources, b in sources + CV.pyconvert_subtype(a,b) || continue + @test a.priority >= b.priority + end +end + +@testitem "lazy rule resolution" begin + using PythonCall + id = pyconvert_add_rule("convert_late_test:Value",String,(_,_) -> "late";scope=Any) + try + @test pyconvert(Int,pyint(1)) == 1 + ns = pyimport("types").ModuleType("convert_late_test") + pyexec("class Value: pass",ns.__dict__) + pyimport("sys").modules["convert_late_test"] = ns + @test pyconvert(Any,ns.Value()) == "late" + finally + pyconvert_remove_rule(id) + pyimport("sys").modules.pop("convert_late_test",nothing) + end +end + +@testitem "source names resolve public aliases and built-in shorthand" begin + using PythonCall + CV = PythonCall.Convert + modules = pyimport("sys").modules + types = pyimport("types") + impl = types.ModuleType("convert_names_internal") + api = types.ModuleType("convert_names_public") + modules["convert_names_internal"] = impl + pyexec(""" +class Base: pass +class Outer: + class Internal(Base): pass +class Child(Outer.Internal): pass +""", impl.__dict__) + api.Public = impl.Outer.Internal + api.Namespace = types.SimpleNamespace(Value=impl.Outer.Internal) + ids = Int[] + add(source,T,f;scope=T) = (id=pyconvert_add_rule(source,T,f;scope);push!(ids,id);id) + try + base = add("convert_names_internal:Base",String,(_,_) -> "base";scope=Any) + firstalias = add("convert_names_public:Public",String,(_,_) -> nothing;scope=Any) + secondalias = add("convert_names_public:Namespace.Value",String,(_,_) -> "alias";scope=Any) + # Registration and unrelated conversion do not import an optional module. + @test !pycontains(modules,"convert_names_public") + @test pyconvert(Int,pyint(1)) === 1 + @test "convert_names_public:Public" in CV.PYCONVERT_PENDING + modules["convert_names_public"] = api + @test pyconvert(String,api.Public.__module__) == "convert_names_internal" + @test pyconvert(String,api.Public.__qualname__) == "Outer.Internal" + @test pyconvert(Any,impl.Child()) == "alias" + rows = pyconvert_rules(Any,impl.Child()) + @test [r.id for r in rows if r.id in ids] == [firstalias,secondalias,base] + @test !("convert_names_public:Public" in CV.PYCONVERT_PENDING) + # Preference lookup also works for an unregistered public alias. + api.OtherName = impl.Outer.Internal + @test pyconvert(Any,impl.Child();prefer=("convert_names_public:OtherName"=>Py,)) isa Py + @test pyconvert(Any,impl.Child();prefer=("convert_names_missing:Public"=>Py,)) == "alias" + @test !pycontains(modules,"convert_names_missing") + # Bare builtins work in preferences, contexts, rule adds and replacements. + x = pylist([1,2]) + @test pyconvert(Any,x;prefer=("list"=>PyList,)) isa PyList + ctx = PyConvertContext(;prefer=("list"=>PyList,"dict"=>PyDict)) + @test pyconvert(ctx,Any,pydict()) isa PyDict + @test pyconvert(ctx,Vector,pylist([x]))[1] isa PyList + @test pyconvert(Vector,x;prefer=("list"=>PyList,)) == [1,2] + @test pyconvert(Any,x;prefer=("dict"=>PyDict,)) == [1,2] + id = add("list",String,(_,_) -> "list rule") + @test pyconvert(String,x) == "list rule" + @test only(r.source for r in pyconvert_rules(String,x) if r.id==id) == "builtins:list" + @test pyconvert(Any,x) == [1,2] + @test pyconvert_replace_rule(id,"dict",String,(_,_) -> "dict rule") == id + @test pytryconvert(String,x) === nothing + @test pyconvert(String,pydict()) == "dict rule" + @test only(r.source for r in pyconvert_rules(String,pydict()) if r.id==id) == "builtins:dict" + @test pyis(CV.pyconvert_resolve("list"),pybuiltins.list) + @test pyis(CV.pyconvert_resolve("NoneType"),pytype(pybuiltins.None)) + # An API alias also retains priorities attached to special source types. + api.JuliaValue = pyimport("juliacall").JlBase + wrapid = add("convert_names_public:JuliaValue",String,(_,_) -> nothing) + rows = pyconvert_rules(String,pyjl([1,2])) + canonical = only(r.priority for r in rows if r.source=="juliacall:JlBase") + @test only(r.priority for r in rows if r.id==wrapid) == canonical + finally + foreach(pyconvert_remove_rule,ids) + modules.pop("convert_names_public",nothing) + modules.pop("convert_names_internal",nothing) + end +end + +@testitem "built-in rules use public source locations" begin + using PythonCall + CV = PythonCall.Convert + @test pyconvert(Any,pybuiltins.None) === nothing + @test first(pyconvert_rules(Any,pybuiltins.None)).source == "types:NoneType" + @test pyis(CV.pyconvert_resolve("types:NoneType"),pytype(pybuiltins.None)) + io = pyimport("io") + privatebase = pyimport("_io")._IOBase + @test !pyis(io.IOBase,privatebase) + @test pyconvert(Any,io.BytesIO()) isa PyIO + @test pyconvert(PyIO,privatebase()) isa PyIO + if pyis(pyimport("importlib.util").find_spec("pandas"),pybuiltins.None) + @test_skip false # pandas is an optional development dependency + else + pd = pyimport("pandas") + @test pyconvert(Any,pd.NA) === missing + @test pytryconvert(Nothing,pd.NA) === Some(nothing) + @test all(r.source=="pandas:NA.__class__" for r in pyconvert_rules(Any,pd.NA)) + df = pd.DataFrame() + @test pyconvert(Any,df) isa PyPandasDataFrame + @test count(r->r.source=="pandas:DataFrame",pyconvert_rules(Any,df)) == 1 + a = pd.array(pylist([1,2]);dtype="Int64") + @test pyconvert(PyList,a) isa PyList + @test count(r->r.source=="pandas.api.extensions:ExtensionArray",pyconvert_rules(PyList,a)) == 1 + @test !any(r->startswith(r.source,"pandas.core.") || startswith(r.source,"pandas._libs."),CV.PYCONVERT_RULES) + end +end + +@testitem "None source shorthand" begin + using PythonCall + CV = PythonCall.Convert + x = pybuiltins.None + pref = ("None"=>Missing,) + @test pyis(CV.pyconvert_resolve("None"),pytype(x)) + @test pyconvert(Any,x;prefer=pref) === missing + @test pytryconvert(Any,x;prefer=pref) === missing + @test pyconvert(Nothing,x;prefer=pref) === nothing + @test pyconvert(Any,pyint(1);prefer=pref) === 1 + ctx = PyConvertContext(;prefer=pref) + @test pyconvert(ctx,Any,x) === missing + @test isequal(pyconvert(ctx,Vector,pylist([x])),[missing]) + @test pyconvert(Any,x) === nothing + id = pyconvert_add_rule("None",String,(_,_) -> "none") + try + @test pyconvert(String,x) == "none" + @test only(r.source for r in pyconvert_rules(String,x) if r.id==id) == "types:NoneType" + @test pyconvert_replace_rule(id,"None",String,(_,_) -> "replaced") == id + @test pyconvert(String,x) == "replaced" + finally + pyconvert_remove_rule(id) + end +end + +@testitem "priority topological sort agrees with grouped sorting" begin + using PythonCall + CV = PythonCall.Convert + # Enumerate DAGs with monotonic priorities; independent tie breaker reference. + for mask in 0:63 + edges = falses(4,4) + for (bit,(i,j)) in enumerate([(i,j) for i=1:4 for j=i+1:4]) + edges[i,j] = (mask >> (bit-1)) & 1 == 1 + end + priorities = [0,10,20,0] + for _=1:4, i=1:4, j=1:4 + edges[i,j] && (priorities[i]=max(priorities[i],priorities[j])) + end + grouped = Int[] + for p in sort(unique(priorities);rev=true) + group = findall(==(p),priorities) + append!(grouped,group[CV.pyconvert_toposort(edges[group,group],zeros(Int,length(group)),group)]) + end + @test grouped == CV.pyconvert_toposort(edges,priorities,collect(1:4)) + end +end