Redesign v1 pyconvert dispatch and native conversions

AGENTS.md

@@ -16,6 +16,10 @@
   the process uses its newer `libstdc++`.
 - `julia --project=docs docs/make.jl` requires a valid Git `origin` so Documenter can infer
   source links; it fails during `makedocs` in checkouts without one.
+- `Documenter.HTML` discovers its default edit-link branch with `git remote show
+  origin`, which contacts the remote. Set `edit_link` explicitly for a local build
+  without that network lookup; v1's docs use `edit_link="v1"`. Disabling source
+  remotes alone does not prevent the HTML constructor's branch lookup.
 
 The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense.
 
@@ -25,10 +29,43 @@ The majority of tests live in the Julia package; Python tests cover functionalit
   `main`'s `ValueBase`/`AnyValue` hierarchy. Keep the collection wrapper when porting
   `_jlwrap` changes. Wrapper modules load before `juliacall.Base` is assigned, so
   constructors must look it up through `juliacall` when called.
+- For numeric Python fixtures, use `pycomplex(real, imag)` when a Python complex
+  value is needed. Passing a Julia `Complex` directly to a Python constructor can
+  produce a Julia wrapper, which NumPy may treat as a sequence rather than a scalar.
+- External-interpreter Julia CI jobs do not install NumPy/pandas development
+  dependencies. Keep synthetic conversion tests independent of them, and guard
+  library-specific checks with module availability (without hiding import errors
+  when a module is installed). Conda-backed jobs exercise those optional checks.
 - The benchmark suite is shared across the compared revisions: `main` uses `pydel!`, while `v1` uses `unsafe_pydel`.
 - `main` does not have `@pyregion`. Benchmark region wrappers must select a plain-body fallback before macro expansion; a runtime conditional around an unavailable macro still fails when loading the suite.
 - Julia CI caches can preserve Conda's build metadata without its root environment. Rebuild `Conda` before building `PyCall` or running tests to recreate that directory.
 - In restricted sandboxes, both suites need approval to write to the shared caches (`~/.julia` and `~/.cache/uv`).
+- Julia also writes `~/.julia/logs/manifest_usage.toml` during dependency discovery,
+  including PythonCall startup through CondaPkg. Cache permissions must include
+  `~/.julia/logs`, in addition to compiled packages, artifacts and registries.
+- Notebook tests start a Jupyter kernel using local sockets. A sandbox that blocks
+  sockets can run the ordinary Python tests but cannot run `--nbval` successfully.
+- Loopback TCP permission alone may be insufficient for notebook tests: ZeroMQ
+  queries network interfaces via `getifaddrs`, which on Linux needs
+  `AF_NETLINK`/`NETLINK_ROUTE` sockets. In the tested sandbox, loopback binding
+  succeeded but creating a route Netlink socket failed with `EPERM`.
+- Use the absolute installed Julia executable in a restricted sandbox to avoid
+  Juliaup's configuration lock. Installed channels may differ from those listed above.
+- `JULIA_PKG_OFFLINE=true` lets JuliaPkg resolve the development Python test
+  environment using installed packages without refreshing registries or cloning
+  uncached package versions.
+- `JULIA_CONDAPKG_ENV` must not point at an existing `.CondaPkg` directory.
+  To run Aqua against an installed interpreter without creating a fresh Conda
+  environment, use `JULIA_CONDAPKG_BACKEND=Null` and set `JULIA_PYTHONCALL_EXE`
+  to that interpreter's absolute path.
+- `test/runtests.jl` infers optional development dependencies from the CondaPkg
+  backend. With the Null backend and an explicit Conda interpreter, PySide6 may
+  be installed even though `Setup.devdeps` is false, causing the GUI test to
+  incorrectly expect an import failure. Focused conversion tests are unaffected.
+- Interrupted sandbox runs can leave a CondaPkg lock containing a PID from a
+  different PID namespace. A reused small PID can make later runs wait for that
+  lock indefinitely. Prefer an explicit installed Python/Null backend for
+  focused conversion tests; do not remove a lock without checking its owner.
 - `pysrc/juliacall/juliapkg.json` is tracked; restore its original contents after using the development copy for Python tests.
 
 ## Meta instructions

benchmark/convert-array-allocations.jl

@@ -0,0 +1,35 @@
+# Attribution only: run after timing trials, with the same environment/Python.
+using PythonCall, Profile
+array_value(::Val{T},x) where T = pyconvert(T,x,nothing)
+label=isempty(ARGS) ? "after" : only(ARGS)
+source_root=dirname(pathof(PythonCall))
+np=pyimport("numpy")
+array=pyimport("array")
+cases=[("ndarray_native",Array,np.arange(1000)),
+       ("ndarray_wrapper",PyArray,np.arange(1000)),
+       ("buffer_native",Array,array.array("d",pylist(1:1000))),
+       ("buffer_wrapper",PyArray,array.array("d",pylist(1:1000)))]
+open(joinpath(@__DIR__,"convert-array-allocations-$label.txt"),"w") do io
+    println(io,"Julia: ",VERSION,"; PythonCall source: ",pathof(PythonCall))
+    for (name,T,x) in cases
+        f=let tag=Val(T), x=x
+            ()->array_value(tag,x)
+        end
+        for _=1:3
+            f()
+        end
+        GC.gc()
+        Profile.Allocs.clear()
+        Profile.Allocs.@profile sample_rate=1.0 f()
+        println(io,"\n",name)
+        for a in Profile.Allocs.fetch().allocs
+            println(io,a.type,": ",a.size," bytes")
+            for frame in a.stacktrace
+                file=string(frame.file)
+                if occursin(source_root,file)
+                    println(io,"  ",frame)
+                end
+            end
+        end
+    end
+end

benchmark/convert-dispatch.jl

@@ -0,0 +1,25 @@
+using PythonCall, BenchmarkTools
+const CV=PythonCall.Convert
+label = isempty(ARGS) ? "" : "-" * only(ARGS)
+x=pylist([pylist(1:5),pylist(6:10)])
+scoped=pylist(1:10)
+np=pyimport("numpy")
+cases = [
+    ("no_preference", ()->pyconvert(Any,x)),
+    ("one_preference", ()->pyconvert(Any,x;prefer=("builtins:list"=>PyList,))),
+    ("multiple_preferences", ()->pyconvert(Any,x;prefer=("builtins:dict"=>PyDict,"builtins:list"=>PyList))),
+    ("unmatched_preferences", ()->pyconvert(Any,x;prefer=("builtins:dict"=>PyDict,))),
+    ("nested_context", ()->pyconvert(PyConvertContext(;prefer=("builtins:list"=>PyList,)),Vector,x)),
+    ("ndarray_native", let a=np.arange(1000); ()->pyconvert(Any,a) end),
+    ("ndarray_wrapper", let a=np.arange(1000); ()->pyconvert(PyArray,a) end),
+    ("bytes_explicit", let a=pybytes(pylist([1,2,3])); ()->pyconvert(Vector{UInt8},a) end),
+    ("scope_filtered", ()->pyconvert(Vector{Int},scoped)),
+]
+open(joinpath(@__DIR__,"convert-dispatch$label.tsv"),"w") do io
+    println(io,"case\tns\tbytes\tallocs")
+    for (name,f) in cases
+        m=minimum(@benchmark $f() seconds=1.0 samples=5000)
+        println(io,"$name\t$(m.time)\t$(m.memory)\t$(m.allocs)")
+    end
+end
+# Cache misses/invalidation are measured with untimed setup in convert-expanded.jl.

benchmark/convert-expanded.jl

@@ -0,0 +1,184 @@
+# Shared harness for the pre-redesign revision and current v1.
+# julia --project=benchmark benchmark/convert-expanded.jl after
+# Optional second argument "warm" skips unchanged cold-path experiments.
+using PythonCall, BenchmarkTools, SHA
+const ExpandedConvert = PythonCall.Convert
+const EXPANDED_NEW_API = isdefined(PythonCall, :PyConvertContext)
+
+expanded_decline1(T,x) = ExpandedConvert.pyconvert_unconverted()
+expanded_decline2(T,x) = ExpandedConvert.pyconvert_unconverted()
+expanded_decline3(T,x) = ExpandedConvert.pyconvert_unconverted()
+expanded_success(T,x) = ExpandedConvert.pyconvert_return(42)
+struct ExpandedDestination{I} end
+struct ExpandedScopedDecliner{I} <: Function end
+(::ExpandedScopedDecliner)(T,x) = ExpandedConvert.pyconvert_unconverted()
+
+function expanded_register(source,T,f;scope=T)
+    if EXPANDED_NEW_API
+        pyconvert_add_rule(source,T,f;scope)
+    else
+        pyconvert_add_rule(source,T,f)
+    end
+end
+expanded_value(::Val{T},x) where T = pyconvert(T,x,nothing)
+function expanded_case(name,::Type{T},x,valid) where T
+    tag=Val(T)
+    (;name, target=T, run=()->expanded_value(tag,x), valid)
+end
+function expanded_cache(T)
+    if EXPANDED_NEW_API
+        ExpandedConvert.pyconvert_rules_cache(typeof(PyConvertContext()),T)
+    else
+        ExpandedConvert.pyconvert_rules_cache(T)
+    end
+end
+function expanded_invalidate!()
+    if EXPANDED_NEW_API
+        ExpandedConvert.pyconvert_invalidate!()
+    else
+        foreach(empty!,values(ExpandedConvert.PYCONVERT_RULES_CACHE))
+    end
+end
+
+label=isempty(ARGS) ? "after" : first(ARGS)
+warm_only=length(ARGS)>=2 && ARGS[2]=="warm"
+ns=pyimport("types").ModuleType("convert_expanded_benchmark")
+pyimport("sys").modules["convert_expanded_benchmark"]=ns
+pyexec("""
+import collections.abc
+import array
+import numpy
+class Success: pass
+class Declines: pass
+class ThreeDeclines: pass
+class Scoped: pass
+class Preference: pass
+class VirtualMapping: pass
+collections.abc.Mapping.register(VirtualMapping)
+class InterfaceOnly:
+    def __init__(self, owner): self.owner = owner
+    @property
+    def __array_interface__(self): return self.owner.__array_interface__
+class ArrayOnly:
+    def __init__(self, owner): self.owner = owner
+    def __array__(self): return self.owner
+""",ns.__dict__)
+prefix="convert_expanded_benchmark:"
+expanded_register(prefix*"Success",Int,expanded_success;scope=Any)
+expanded_register(prefix*"Declines",Int,expanded_decline1;scope=Any)
+expanded_register(prefix*"Declines",Int,expanded_success;scope=Any)
+for f in (expanded_decline1,expanded_decline2,expanded_decline3,expanded_success)
+    expanded_register(prefix*"ThreeDeclines",Int,f;scope=Any)
+end
+for i=1:16
+    # These rules legitimately decline for this value. Old v1 has no scope
+    # filtering and must try them; the redesign excludes their owned scopes.
+    expanded_register(prefix*"Scoped",Any,ExpandedScopedDecliner{i}();scope=ExpandedDestination{i})
+end
+expanded_register(prefix*"Scoped",Int,expanded_success;scope=Any)
+expanded_register(prefix*"Preference",String,expanded_decline1;scope=Any)
+expanded_register(prefix*"Preference",Int,expanded_success;scope=Any)
+expanded_register("collections.abc:Mapping",Int,expanded_success)
+
+buffer=ns.array.array("d",pylist(1:1000))
+owner=ns.numpy.arange(1000)
+interface=ns.InterfaceOnly(owner)
+arrayonly=ns.ArrayOnly(owner)
+@assert PythonCall.C.PyType_CheckBuffer(pytype(buffer))
+@assert !any(pyhasattr(pytype(buffer),a) for a in ("__array_struct__","__array_interface__","__array__"))
+@assert !PythonCall.C.PyType_CheckBuffer(pytype(interface))
+@assert !PythonCall.C.PyType_CheckBuffer(pytype(arrayonly))
+if EXPANDED_NEW_API
+    eligible=filter(r->r.eligible,pyconvert_rules(Array,buffer))
+    @assert first(eligible).source=="<buffer>"
+else
+    eligible=filter(r->typeintersect(r.type,Array)!==Union{},
+                    ExpandedConvert._pyconvert_get_rules(pytype(buffer)))
+    @assert first(eligible).func===PythonCall.Wrap.pyconvert_rule_array
+end
+validint=x->x===42
+validarray=x->x isa Array && size(x)==(1000,) && eltype(x)===Int64 && first(x)==0 && last(x)==999
+validbuffer=x->x isa Array && size(x)==(1000,) && eltype(x)===Float64 && first(x)==1.0 && last(x)==1000.0
+validwrapper=x->x isa PyArray && size(x)==(1000,) && first(x)==0 && last(x)==999
+validbufferwrapper=x->x isa PyArray && size(x)==(1000,) && first(x)==1.0 && last(x)==1000.0
+cases=[
+    expanded_case("single_rule",Int,ns.Success(),validint),
+    expanded_case("decline_then_success",Int,ns.Declines(),validint),
+    expanded_case("three_declines_then_success",Int,ns.ThreeDeclines(),validint),
+    expanded_case("sixteen_scoped_decliners",Int,ns.Scoped(),validint),
+    expanded_case("virtual_mapping_abc",Int,ns.VirtualMapping(),validint),
+    expanded_case("buffer_only_native",Array,buffer,validbuffer),
+    expanded_case("buffer_only_wrapper",PyArray,buffer,validbufferwrapper),
+    expanded_case("interface_only_native",Array,interface,validarray),
+    expanded_case("array_method_only_native",Array,arrayonly,validarray),
+    expanded_case("ndarray_native",Array,owner,validarray),
+    expanded_case("ndarray_wrapper",PyArray,owner,validwrapper),
+]
+# Verify the dispatch fixtures themselves, outside the measured regions.
+for (i,expected) in enumerate(EXPANDED_NEW_API ? (1,2,4,1) : (1,2,4,17))
+    case=cases[i]
+    # The first fixtures are distinct Python classes, each with the indicated
+    # number of eligible rules. This also detects accidental rule deduplication.
+    t=(ns.Success,ns.Declines,ns.ThreeDeclines,ns.Scoped)[i]
+    if EXPANDED_NEW_API
+        @assert count(r->r.eligible,pyconvert_rules(Int,t()))==expected case.name
+    else
+        @assert length(ExpandedConvert.pyconvert_get_rules(Int,t))==expected case.name
+    end
+end
+preference_names=("two_matching_first_success","two_matching_first_declines",
+                  "two_matching_decline_then_ordinary")
+preference_cases=if EXPANDED_NEW_API
+    x=ns.Preference()
+    source=prefix*"Preference"
+    [(;name=preference_names[1],run=()->pyconvert(Any,x;prefer=(source=>Int,source=>String))),
+     (;name=preference_names[2],run=()->pyconvert(Any,x;prefer=(source=>String,source=>Int))),
+     (;name=preference_names[3],run=()->pyconvert(Any,x;prefer=(source=>String,source=>Float64)))]
+else
+    []
+end
+
+open(joinpath(@__DIR__,"convert-expanded-$label.tsv"),"w") do io
+    println(io,"case\tmode\tstatus\tmin_ns\tmedian_ns\tbytes\tallocs")
+    for case in cases
+        f=case.run
+        @assert case.valid(f()) case.name
+        cache=expanded_cache(case.target)
+        for mode in (warm_only ? ("warm",) : ("warm","selection_miss","invalidated"))
+            trial=if mode=="warm"
+                @benchmark $f() seconds=1.0 samples=5000
+            elseif mode=="selection_miss"
+                @benchmark $f() setup=(empty!($cache)) seconds=0.4 samples=200 evals=1
+            else
+                @benchmark $f() setup=(expanded_invalidate!()) seconds=0.4 samples=40 evals=1
+            end
+            m,d=minimum(trial),median(trial)
+            println(io,"$(case.name)\t$mode\tmeasured\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)")
+        end
+        flush(io)
+    end
+    for case in preference_cases
+        f=case.run
+        @assert f()===42 case.name
+        trial=@benchmark $f() seconds=1.0 samples=5000
+        m,d=minimum(trial),median(trial)
+        println(io,"$(case.name)\twarm\tmeasured\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)")
+    end
+    if !EXPANDED_NEW_API
+        for name in preference_names
+            println(io,"$name\twarm\tunsupported\tNA\tNA\tNA\tNA")
+        end
+    end
+end
+open(joinpath(@__DIR__,"convert-expanded-$label-environment.txt"),"w") do io
+    println(io,"Julia: ",VERSION)
+    println(io,"Python: ",pyconvert(String,pyimport("sys").version))
+    println(io,"Python executable: ",pyconvert(String,pyimport("sys").executable))
+    println(io,"NumPy: ",pyconvert(String,ns.numpy.__version__))
+    println(io,"BenchmarkTools: ",pkgversion(BenchmarkTools))
+    println(io,"PythonCall source: ",pathof(PythonCall))
+    println(io,"New API: ",EXPANDED_NEW_API)
+    for file in ("Convert/pyconvert.jl","Convert/rules.jl","Wrap/PyArray.jl")
+        println(io,file," SHA256: ",bytes2hex(sha256(read(joinpath(dirname(pathof(PythonCall)),file)))))
+    end
+end

benchmark/convert-final.jl

@@ -0,0 +1,63 @@
+# Same harness on original v1 and the final working tree. No new-API dependency.
+# julia --project=benchmark benchmark/convert-final.jl after-r1
+using PythonCall, BenchmarkTools, SHA
+
+final_value(::Val{T}, x) where T = pyconvert(T,x,nothing)
+label = isempty(ARGS) ? "after-r1" : only(ARGS)
+newapi = isdefined(PythonCall,:PyConvertContext)
+fixtures = [
+    ("int",Int,pyint(10000),x->x===10000),
+    ("integer",Integer,pyint(42),x->x===42),
+    ("bigint",BigInt,pyint(big(2)^200),x->x==big(2)^200),
+    ("float",Float64,pyfloat(1.5),x->x===1.5),
+    ("bool",Bool,pybool(true),x->x===true),
+    ("string",String,pystr("hello"),x->x=="hello"),
+    ("none",Nothing,pybuiltins.None,isnothing),
+    ("failure",Int,pystr("oops"),isnothing),
+    ("list10",Vector,pylist(1:10),x->x==collect(1:10)),
+    ("list10000",Vector,pylist(1:10000),x->x==collect(1:10000)),
+    ("list10000_concrete",Vector{Int},pylist(1:10000),x->x==collect(1:10000)),
+    ("mixed_numeric",Vector,pylist([pyint(1),pyfloat(2.5)]),x->x==[1,2.5]),
+    ("heterogeneous",Vector,pylist([pyint(1),pystr("two"),pybuiltins.None]),x->isequal(x,Any[1,"two",nothing])),
+    ("nested_concrete",Vector{Vector{Int}},pylist([pylist(1:5),pylist(6:10)]),x->x==[collect(1:5),collect(6:10)]),
+    # Any intentionally changes from wrapping to native recursive conversion.
+    ("nested_any",Any,pylist([pylist(1:5),pylist(6:10)]),x->newapi ? x isa Vector{Vector{Int}} : x isa PyList),
+    ("dict",Dict,pydict("a"=>1,"b"=>2.5),x->x==Dict("a"=>1,"b"=>2.5)),
+    ("dict_concrete",Dict{String,Float64},pydict("a"=>1,"b"=>2.5),x->x==Dict("a"=>1.0,"b"=>2.5)),
+    ("empty_list",Vector,pylist(),x->x isa Vector && isempty(x)),
+    ("empty_dict",Dict,pydict(),x->x isa Dict && isempty(x)),
+    ("wrapper",PyList,pylist(1:10),x->x isa PyList && length(x)==10),
+    ("iterator",Any,pyiter(pylist(1:10)),x->x isa PyIterable),
+]
+open(joinpath(@__DIR__,"convert-final-$label.tsv"),"w") do io
+    println(io,"case\tmode\tmin_ns\tmedian_ns\tbytes\tallocs")
+    for (i,(name,T,x,valid)) in enumerate(fixtures)
+        tag=Val(T)
+        @assert valid(final_value(tag,x)) name
+        modes=i<=8 ? ("typed","runtime_target") : ("typed",)
+        for mode in modes
+            trial = if mode=="typed"
+                @benchmark final_value($tag,$x) seconds=1.0 samples=5000
+            else
+                @benchmark pyconvert($T,$x,nothing) seconds=1.0 samples=5000
+            end
+            m,d=minimum(trial),median(trial)
+            println(io,"$name\t$mode\t$(m.time)\t$(d.time)\t$(m.memory)\t$(m.allocs)")
+        end
+        flush(io)
+    end
+end
+open(joinpath(@__DIR__,"convert-final-$label-environment.txt"),"w") do io
+    println(io,"Julia: ",VERSION)
+    println(io,"Python: ",pyconvert(String,pyimport("sys").version))
+    println(io,"Python executable: ",pyconvert(String,pyimport("sys").executable))
+    println(io,"BenchmarkTools: ",pkgversion(BenchmarkTools))
+    println(io,"PythonCall source: ",pathof(PythonCall))
+    println(io,"New API: ",newapi)
+    for file in ("Convert/pyconvert.jl","Convert/rules.jl","Wrap/PyArray.jl")
+        println(io,file," SHA256: ",bytes2hex(sha256(read(joinpath(dirname(pathof(PythonCall)),file)))))
+    end
+    for (name,T,x,_) in fixtures
+        println(io,name," result type: ",typeof(final_value(Val(T),x)))
+    end
+end

benchmark/convert-redesign.jl

@@ -0,0 +1,24 @@
+# Run with julia --project=benchmark benchmark/convert-redesign.jl baseline|redesign
+using PythonCall, BenchmarkTools
+label = isempty(ARGS) ? "redesign" : ARGS[1]
+const CV = PythonCall.Convert
+cases = [
+    ("int", Int, pyint(42)), ("integer", Integer, pyint(42)),
+    ("bigint", BigInt, pyint(big(2)^200)), ("float", Float64, pyfloat(1.5)),
+    ("bool", Bool, pybool(true)), ("string", String, pystr("hello")),
+    ("none", Nothing, pybuiltins.None), ("failure", Int, pystr("oops")),
+    ("list10", Vector, pylist(1:10)), ("list10000", Vector, pylist(1:10000)),
+    ("mixed", Vector, pylist([1, 2.5, "three", nothing])),
+    ("nested", Any, pylist([pylist(1:5), pylist(6:10)])),
+    ("dict", Dict, pydict("a"=>1, "b"=>2.5)),
+    ("wrapper", PyList, pylist(1:10)),
+    ("iterator", Any, pyiter(pylist(1:10))),
+]
+open(joinpath(@__DIR__, "convert-$label.tsv"), "w") do io
+    println(io, "case\tns\tbytes\tallocs")
+    for (name, T, x) in cases
+        trial = @benchmark pyconvert($T, $x, nothing) seconds=0.3 samples=1000
+        m = minimum(trial)
+        println(io, "$name\t$(m.time)\t$(m.memory)\t$(m.allocs)")
+    end
+end