diff --git a/ConfigHeaderExt.zig b/ConfigHeaderExt.zig deleted file mode 100644 index 8862282..0000000 --- a/ConfigHeaderExt.zig +++ /dev/null @@ -1,82 +0,0 @@ -/// Add config values to a ConfigHeader from a lazy path. -/// The lazy path should contain lines in the format: NAME VALUE -pub fn addFile(config_header: *ConfigHeader, config_results: std.Build.LazyPath) void { - const b = config_header.step.owner; - // This ApplyConfigStep is just a HACK which *should work* but the proper solution is to - // add LazyPath support in ConfigHeader itself. - const apply = b.allocator.create(ApplyConfigStep) catch @panic("OOM"); - apply.* = .{ - .step = std.Build.Step.init(.{ - .id = .custom, - .name = "add configuration from lazy path", - .owner = b, - .makeFn = ApplyConfigStep.make, - }), - .config_header = config_header, - .config_results = config_results, - }; - config_results.addStepDependencies(&apply.step); - config_header.step.dependOn(&apply.step); -} - -pub fn parseValue(str: []const u8) ?ConfigHeader.Value { - if (std.mem.eql(u8, str, "undef")) return .undef; - if (std.mem.eql(u8, str, "defined")) return .defined; - if (std.mem.eql(u8, str, "false")) return .{ .boolean = false }; - if (std.mem.eql(u8, str, "true")) return .{ .boolean = true }; - if (std.fmt.parseInt(i64, str, 10)) |int| return .{ .int = int } else |_| {} - if (std.mem.startsWith(u8, str, ":")) return .{ .ident = str[1..] }; - if (str.len >= 2 and str[0] == '"' and str[str.len - 1] == '"') return .{ .string = str[1 .. str.len - 1] }; - return null; -} - -pub const FileEntry = struct { - name: []const u8, - value: ConfigHeader.Value, - - pub fn parse(line: []const u8) error{ MissingValue, InvalidValue }!FileEntry { - const first_space = std.mem.indexOfScalar(u8, line, ' ') orelse return error.MissingValue; - return .{ - .name = line[0..first_space], - .value = parseValue(line[first_space + 1 ..]) orelse return error.InvalidValue, - }; - } -}; - -const ApplyConfigStep = struct { - step: std.Build.Step, - config_header: *ConfigHeader, - config_results: std.Build.LazyPath, - - fn make(step: *std.Build.Step, options: std.Build.Step.MakeOptions) anyerror!void { - _ = options; - const b = step.owner; - const self: *ApplyConfigStep = @fieldParentPtr("step", step); - - const file_path = try self.config_results.getPath4(b, step); - const content = file_path.root_dir.handle.readFileAlloc( - b.graph.io, - file_path.subPathOrDot(), - b.allocator, - .unlimited, - ) catch |err| return step.fail( - "unable to read config values from '{f}': {t}", - .{ file_path, err }, - ); - var line_it = std.mem.splitScalar(u8, content, '\n'); - var line_num: u32 = 0; - while (line_it.next()) |line| { - line_num += 1; - if (line.len == 0 or line[0] == '#') continue; - if (FileEntry.parse(line)) |entry| { - self.config_header.values.put(b.allocator, entry.name, entry.value) catch @panic("OOM"); - } else |err| { - step.addError("{f}:{d}: {t} '{s}'", .{ file_path, line_num, err, line }) catch @panic("OOM"); - } - } - if (step.result_error_msgs.items.len != 0) return error.MakeFailed; - } -}; - -const std = @import("std"); -const ConfigHeader = std.Build.Step.ConfigHeader; diff --git a/build.zig b/build.zig index 802b257..1cc45de 100644 --- a/build.zig +++ b/build.zig @@ -43,28 +43,6 @@ const Stage2FrozenMods = struct { pub fn build(b: *std.Build) !void { const version: Version = b.option(Version, "version", "Python Version") orelse .latest; - const replace_exe = b.addExecutable(.{ - .name = "replace", - .root_module = b.createModule(.{ - .root_source_file = b.path("replace.zig"), - .target = b.graph.host, - }), - }); - const configquery_exe = b.addExecutable(.{ - .name = "configquery", - .root_module = b.createModule(.{ - .root_source_file = b.path("configquery.zig"), - .target = b.graph.host, - }), - }); - const makesetup_exe = b.addExecutable(.{ - .name = "makesetup", - .root_module = b.createModule(.{ - .root_source_file = b.path("makesetup.zig"), - .target = b.graph.host, - }), - }); - const upstream: *std.Build.Dependency = switch (version) { .@"3.11.13" => if (b.lazyDependency("upstream_3.11.13", .{})) |d| d else noUpstream(b), .@"3.12.11" => if (b.lazyDependency("upstream_3.12.11", .{})) |d| d else noUpstream(b), @@ -78,7 +56,26 @@ pub fn build(b: *std.Build) !void { else => true, }; - const libs_host: Libs = .{ .zlib = null, .openssl = null }; + var hacl: ?*std.Build.Step.Compile = undefined; + if (version == .@"3.12.11") { + hacl = b.addLibrary(.{ + .name = "hacl", + .linkage = .static, + .root_module = b.createModule(.{ + .target = target, + .optimize = optimize, + .link_libc = true, + }), + }); + hacl.?.root_module.addIncludePath(upstream.path("./Modules/_hacl/include")); + hacl.?.root_module.addIncludePath(upstream.path("./Modules/_hacl/")); + hacl.?.root_module.addCSourceFile(.{ .file = upstream.path("./Modules/_hacl/Hacl_Hash_SHA2.c"), .flags = &.{ + "-D_BSD_SOURCE", "-D_DEFAULT_SOURCE", "-fwrapv", + "-std=c11", "-fvisibility=hidden", "-DVPATH=\"\"", + } }); + } else hacl = null; + + const libs_host: Libs = .{ .zlib = null, .openssl = null, .hacl = null }; const libs_target: Libs = .{ .zlib = (b.dependency("zlib", .{ .target = target, @@ -88,23 +85,27 @@ pub fn build(b: *std.Build) !void { .target = target, .optimize = optimize, })) |dep| dep.artifact("openssl") else null) else null, + .hacl = hacl, }; - const makesetup_host = addMakesetup(b, version, upstream, libs_host, .{ - .os_tag = b.graph.host.result.os.tag, - .replace_exe = replace_exe, - .makesetup_exe = makesetup_exe, - }); - const makesetup_target = addMakesetup(b, version, upstream, libs_target, .{ - .os_tag = target.result.os.tag, - .replace_exe = replace_exe, - .makesetup_exe = makesetup_exe, + if (b.graph.needed_lazy_dependencies.count() > 0) return; + + const configquery_exe = b.addExecutable(.{ + .name = "configquery", + .root_module = b.createModule(.{ + .root_source_file = b.path("configquery.zig"), + .target = b.graph.host, + }), }); - const pyconfig_host = try addPyconfig(b, version, upstream, b.graph.host, .{ .zlib = null, .openssl = null }, configquery_exe); + + const makesetup_host = addMakesetup(b, version, upstream, libs_host, b.graph.host.result.os.tag); + const makesetup_target = addMakesetup(b, version, upstream, libs_target, target.result.os.tag); + + const pyconfig_host = try addPyconfig(b, version, upstream, b.graph.host, libs_host, configquery_exe); const freeze_module_exe = addPythonExe(b, upstream, b.graph.host, .Debug, .{ .name = "freeze_module", - .makesetup_out = makesetup_host, + .setup = makesetup_host, .pyconfig = pyconfig_host, .stage = .freeze_module, }); @@ -139,11 +140,11 @@ pub fn build(b: *std.Build) !void { const frozen_headers, const deepfreeze_c = blk_deepfreeze_c: { const bootstrap_python_exe = addPythonExe(b, upstream, b.graph.host, .Debug, .{ .name = "bootstrap_python", - .makesetup_out = makesetup_host, + .setup = makesetup_host, .pyconfig = pyconfig_host, .stage = .{ .bootstrap = stage2_frozen_mods }, }); - const bootstrap_packaged = packagePython(b, version, upstream, bootstrap_python_exe); + const bootstrap_packaged = packagePython(b, version, upstream, bootstrap_python_exe, &.{}); const frozen_headers = b.allocator.alloc(std.Build.LazyPath, frozen_modules.len) catch @panic("OOM"); // don't free @@ -167,49 +168,19 @@ pub fn build(b: *std.Build) !void { })); run.addArg("-o"); const deepfreeze_c = run.addOutputFileArg("deepfreeze.c"); - { - // Need to create a custom step because std.Build.Step.Run doesn't - // have addSuffixedOutputArg. - const AddModules = struct { - step: std.Build.Step, - version: Version, - run_deepfreeze: *std.Build.Step.Run, - headers: []const std.Build.LazyPath, - }; - const add_modules_make = struct { - fn make(step: *std.Build.Step, options: std.Build.Step.MakeOptions) anyerror!void { - _ = options; - const b2 = step.owner; - const self: *AddModules = @fieldParentPtr("step", step); - for (frozenModuleNames(self.version), self.headers) |mod_name, header| { - self.run_deepfreeze.addArg(b2.fmt("{s}:{s}", .{ header.getPath(b2), mod_name })); - } - } - }.make; - const add_modules = b.allocator.create(AddModules) catch @panic("OOM"); - add_modules.* = .{ - .step = std.Build.Step.init(.{ - .id = .custom, - .name = "add modules to run deepfreeze.py", - .owner = b, - .makeFn = &add_modules_make, - }), - .version = version, - .run_deepfreeze = run, - .headers = frozen_headers, - }; - add_modules.step.dependOn(freeze_step); - run.step.dependOn(&add_modules.step); + for (frozenModuleNames(version), frozen_headers) |mod_name, header| { + const mod_name_part = b.fmt(":{s}", .{mod_name}); + run.addFileArg2(header, .{ .suffix = mod_name_part }); } - b.step("deepfreeze", "").dependOn(&run.step); break :blk_deepfreeze_c .{ frozen_headers, deepfreeze_c }; }; + const final_pyconfig = try addPyconfig(b, version, upstream, target, libs_target, configquery_exe); const final_exe = addPythonExe(b, upstream, target, optimize, .{ .name = "python", - .makesetup_out = makesetup_target, - .pyconfig = try addPyconfig(b, version, upstream, target, libs_target, configquery_exe), + .setup = makesetup_target, + .pyconfig = final_pyconfig, .stage = .{ .final = .{ .stage2 = stage2_frozen_mods, @@ -218,7 +189,7 @@ pub fn build(b: *std.Build) !void { }, }, }); - const final_packaged = packagePython(b, version, upstream, final_exe); + const final_packaged = packagePython(b, version, upstream, final_exe, &.{final_pyconfig.sysconfigdata.?}); const install_final = b.addInstallDirectory(.{ .source_dir = final_packaged.root, .install_dir = .prefix, @@ -229,13 +200,10 @@ pub fn build(b: *std.Build) !void { const ci_step = b.step("ci", "The build/test step to run on the CI"); ci_step.dependOn(b.getInstallStep()); try ci(b, version, ssl_enabled, upstream, ci_step, .{ - .replace_exe = replace_exe, - .makesetup_exe = makesetup_exe, - .configquery_exe = configquery_exe, .stage2_frozen_mods = stage2_frozen_mods, .frozen_headers = frozen_headers, .deepfreeze_c = deepfreeze_c, - }); + }, configquery_exe); } fn noUpstream(b: *std.Build) *std.Build.Dependency { @@ -249,13 +217,9 @@ fn addMakesetup( version: Version, upstream: *std.Build.Dependency, libs: Libs, - args: struct { - os_tag: std.Target.Os.Tag, - replace_exe: *std.Build.Step.Compile, - makesetup_exe: *std.Build.Step.Compile, - }, -) std.Build.LazyPath { - const is_posix = (args.os_tag != .windows); + os_tag: std.Target.Os.Tag, +) makesetup.Setup { + const is_posix = (os_tag != .windows); const stdlib_modules_common = .{ ._ssl = (libs.openssl != null), @@ -276,7 +240,7 @@ fn addMakesetup( ._pickle = true, ._queue = true, ._random = true, - ._socket = (args.os_tag != .windows), + ._socket = (os_tag != .windows), ._statistics = true, ._struct = true, ._zoneinfo = true, @@ -303,9 +267,9 @@ fn addMakesetup( .fcntl = is_posix, ._posixshmem = is_posix, .grp = is_posix, - .ossaudiodev = (args.os_tag == .linux), - .resource = (args.os_tag == .linux), - .spwd = (args.os_tag == .linux), + .ossaudiodev = (os_tag == .linux), + .resource = (os_tag == .linux), + .spwd = (os_tag == .linux), .syslog = is_posix, .termios = is_posix, @@ -351,72 +315,101 @@ fn addMakesetup( }; const @"stdlib_modules_3.12.11" = .{ ._ctypes = false, - ._sha2 = false, + ._sha2 = (libs.hacl != null), .xxsubtype = false, ._xxinterpchannels = false, }; - const setup_bootstrap = blk: { - const replace = b.addRunArtifact(args.replace_exe); - replace.addFileArg(upstream.path("Modules/Setup.bootstrap.in")); - const out = replace.addOutputFileArg("Setup.bootstrap"); + const pwd_enabled = is_posix; + const value_str = if (pwd_enabled) "" else "#"; - const pwd_enabled = is_posix; - const value_str = if (pwd_enabled) "" else "#"; - replace.addArg(b.fmt("MODULE_PWD_TRUE={s}", .{value_str})); - break :blk out; - }; + const dep_root_path = upstream.builder.root.root_dir.path orelse "."; + const bootstrap_in_path = b.pathJoin(&.{ dep_root_path, "Modules/Setup.bootstrap.in" }); + b.dependOnFileContents(upstream.path("Modules/Setup.bootstrap.in")); - const setup_stdlib = blk_stdlib: { - const replace = b.addRunArtifact(args.replace_exe); - replace.addFileArg(upstream.path("Modules/Setup.stdlib.in")); - const out = replace.addOutputFileArg("Setup.stdlib"); - replace.addArg("MODULE_{NAME}_TRUE=MODULE_{NAME}_TRUE"); - replace.addArg("MODULE_BUILDTYPE=static"); - addReplaceModuleArgs(b, replace, @TypeOf(stdlib_modules_common), stdlib_modules_common); - switch (version) { - .@"3.11.13" => addReplaceModuleArgs(b, replace, @TypeOf(@"stdlib_modules_3.11.13"), @"stdlib_modules_3.11.13"), - .@"3.12.11" => { - replace.addArg("MODULE__CTYPES_MALLOC_CLOSURE="); - addReplaceModuleArgs(b, replace, @TypeOf(@"stdlib_modules_3.12.11"), @"stdlib_modules_3.12.11"); - }, - } - break :blk_stdlib out; - }; + // First replace call + const setup_bootstrap = replace.replace(b, bootstrap_in_path, &.{ + b.fmt("MODULE_PWD_TRUE={s}", .{value_str}), + }); + defer b.allocator.free(setup_bootstrap); - const generate = b.addRunArtifact(args.makesetup_exe); - generate.addDirectoryArg(upstream.path(".")); - const makesetup_out = generate.addOutputDirectoryArg("gen"); - generate.addFileArg(setup_bootstrap); - generate.addFileArg(setup_stdlib); - generate.addFileArg(upstream.path("Modules/Setup")); - return makesetup_out; + var replacements: std.ArrayList([]const u8) = .empty; + defer replacements.deinit(b.allocator); + + replacements.append(b.allocator, "MODULE_{NAME}_TRUE=MODULE_{NAME}_TRUE") catch @panic("OOM"); + replacements.append(b.allocator, "MODULE_BUILDTYPE=static") catch @panic("OOM"); + addReplacements(&replacements, b, @TypeOf(stdlib_modules_common), stdlib_modules_common); + switch (version) { + .@"3.11.13" => addReplacements(&replacements, b, @TypeOf(@"stdlib_modules_3.11.13"), @"stdlib_modules_3.11.13"), + .@"3.12.11" => { + replacements.append(b.allocator, "MODULE__CTYPES_MALLOC_CLOSURE=") catch @panic("OOM"); + addReplacements(&replacements, b, @TypeOf(@"stdlib_modules_3.12.11"), @"stdlib_modules_3.12.11"); + }, + } + + // Second replace call + const stdlib_in_path = b.pathJoin(&.{ dep_root_path, "Modules/Setup.stdlib.in" }); + b.dependOnFileContents(upstream.path("Modules/Setup.stdlib.in")); + const setup_stdlib = replace.replace(b, stdlib_in_path, replacements.items); + defer b.allocator.free(setup_stdlib); + + const setup_in_path = b.pathJoin(&.{ dep_root_path, "Modules/Setup" }); + b.dependOnFileContents(upstream.path("Modules/Setup")); + + const setup_file = std.Io.Dir.cwd().readFileAlloc(b.graph.io, setup_in_path, b.allocator, .unlimited) catch |e| + std.debug.panic("open '{s}' failed with {t}", .{ setup_in_path, e }); + var setup = makesetup.makesetup(b, &.{ setup_bootstrap, setup_stdlib, setup_file }, &.{ bootstrap_in_path, stdlib_in_path, setup_in_path }); + + const config_c_in_path = b.pathJoin(&.{ dep_root_path, "Modules/config.c.in" }); + b.dependOnFileContents(upstream.path("Modules/config.c.in")); + const config_c_in = std.Io.Dir.cwd().readFileAlloc(b.graph.io, config_c_in_path, b.allocator, .unlimited) catch |e| + std.debug.panic("open file '{s}' failed with {t}", .{ config_c_in_path, e }); + + const wf = b.addWriteFiles(); + const config_contents = makesetup.writeConfigC(b, setup, config_c_in_path, config_c_in); + defer b.allocator.free(config_contents); + const config_c = wf.add("Modules/config.c", config_contents); + setup.outfile = config_c; + + return setup; } -fn addReplaceModuleArgs(b: *std.Build, replace: *std.Build.Step.Run, comptime Modules: type, modules: Modules) void { - inline for (std.meta.fields(Modules)) |field| { - const name = field.name; +fn addReplacements(list: *std.ArrayList([]const u8), b: *std.Build, comptime Modules: type, modules: Modules) void { + comptime switch (@typeInfo(Modules)) { + .@"struct" => {}, + else => @compileError("Modules type must be a struct"), + }; + inline for (@typeInfo(Modules).@"struct".field_names) |name| { const value = @field(modules, name); var upcase_buf: [100]u8 = undefined; for (name, 0..) |c, i| { upcase_buf[i] = std.ascii.toUpper(c); } const value_str = if (value) "" else "#"; - replace.addArg(b.fmt("MODULE_{s}_TRUE={s}", .{ upcase_buf[0..name.len], value_str })); + list.append(b.allocator, b.fmt("MODULE_{s}_TRUE={s}", .{ upcase_buf[0..name.len], value_str })) catch @panic("OOM"); } } -fn packagePython(b: *std.Build, version: Version, upstream: *std.Build.Dependency, exe: *std.Build.Step.Compile) struct { +const other_libs = struct { + path: std.Build.LazyPath, + name: []const u8, +}; + +fn packagePython(b: *std.Build, version: Version, upstream: *std.Build.Dependency, exe: *std.Build.Step.Compile, libs: []const other_libs) struct { root: std.Build.LazyPath, exe: std.Build.LazyPath, } { const write_files = b.addNamedWriteFiles(exe.name); _ = write_files.addCopyDirectory(upstream.path("Lib"), b.fmt("lib/python{s}", .{version.libName()}), .{}); + for (libs) |lib| { + _ = write_files.addCopyFile(lib.path, b.fmt("lib/python{s}/{s}", .{ version.libName(), lib.name })); + } const empty_dir = b.addWriteFiles().getDirectory(); _ = write_files.addCopyDirectory(empty_dir, b.fmt("lib/python{s}/lib-dynload", .{version.libName()}), .{}); if (exe.producesPdbFile()) { _ = write_files.addCopyFile(exe.getEmittedPdb(), b.fmt("bin/{s}.pdb", .{exe.name})); } + b.getInstallStep().dependOn(&write_files.step); return .{ .root = write_files.getDirectory(), .exe = write_files.addCopyFile( @@ -433,7 +426,7 @@ fn addPythonExe( optimize: std.builtin.OptimizeMode, args: struct { name: []const u8, - makesetup_out: std.Build.LazyPath, + setup: makesetup.Setup, pyconfig: Pyconfig, stage: PythonExeStage, }, @@ -467,10 +460,7 @@ fn addPythonExe( .windows => "\"win32\"", else => std.debug.panic("todo: populate PLATFORM for os '{s}'", .{@tagName(target.result.os.tag)}), }); - switch (args.pyconfig.header) { - .path => |path| exe.root_module.addIncludePath(path.dirname()), - .config_header => |h| exe.root_module.addConfigHeader(h), - } + exe.root_module.addIncludePath(args.pyconfig.header.dirname()); exe.root_module.addIncludePath(upstream.path(".")); exe.root_module.addIncludePath(upstream.path("Include")); exe.root_module.addIncludePath(upstream.path("Include/internal")); @@ -496,67 +486,20 @@ fn addPythonExe( "-std=c11", "-fvisibility=hidden", "-DVPATH=\"\"", + "-DABIFLAGS=\"\"", + "-fno-sanitize=function", }; - { - const AddModules = struct { - step: std.Build.Step, - upstream: *std.Build.Dependency, - exe: *std.Build.Step.Compile, - module_compile_args_file: std.Build.LazyPath, - }; - const add_modules_make = struct { - fn make(step: *std.Build.Step, options: std.Build.Step.MakeOptions) anyerror!void { - _ = options; - const io = step.owner.graph.io; - const self: *AddModules = @fieldParentPtr("step", step); - const file_path = self.module_compile_args_file.getPath2(step.owner, step); - - const module_compile_args = try std.Io.Dir.cwd().readFileAlloc(io, file_path, step.owner.allocator, .unlimited); - defer step.owner.allocator.free(module_compile_args); - - var files: std.ArrayList([]const u8) = .empty; - defer files.deinit(step.owner.allocator); - - var line_it = std.mem.splitScalar(u8, module_compile_args, '\n'); - while (line_it.next()) |line| { - if (line.len == 0) continue; - if (std.mem.startsWith(u8, line, "# ")) continue; - if (std.mem.endsWith(u8, line, ".c")) { - try files.append(step.owner.allocator, line); - } else if (std.mem.startsWith(u8, line, "-I")) { - const path = line[2..]; - const prefix = "$(srcdir)/"; - if (!std.mem.startsWith(u8, path, prefix)) std.debug.panic( - "expected include path to start with '-I{s}' but got: '{s}'", - .{ prefix, line }, - ); - const inc_sub_path = step.owner.dupe(path[prefix.len..]); - self.exe.root_module.addIncludePath(self.upstream.path(inc_sub_path)); - } else std.debug.panic("todo: parse module-compile-args line '{s}'", .{line}); - } - - self.exe.root_module.addCSourceFiles(.{ - .root = self.upstream.path("."), - .files = files.items, - .flags = &flags_common, - }); - } - }.make; - const add_modules = b.allocator.create(AddModules) catch @panic("OOM"); - add_modules.* = .{ - .step = std.Build.Step.init(.{ - .id = .custom, - .name = b.fmt("add module sources/includes to {s} exe", .{exe.name}), - .owner = b, - .makeFn = &add_modules_make, - }), - .upstream = upstream, - .exe = exe, - .module_compile_args_file = args.makesetup_out.path(b, "module-compile-args.txt"), - }; - add_modules.module_compile_args_file.addStepDependencies(&add_modules.step); - exe.step.dependOn(&add_modules.step); + var it = args.setup.modules.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.enabled) { + for (entry.value_ptr.compile_args) |compile_arg| switch (compile_arg) { + .source => |src| exe.root_module.addCSourceFile(.{ + .file = upstream.path(b.fmt("Modules/{s}", .{src})), + }), + .include => |inc| exe.root_module.addIncludePath(upstream.path(b.fmt("{s}", .{inc}))), + }; + } } exe.root_module.addCSourceFiles(.{ @@ -607,7 +550,7 @@ fn addPythonExe( }); exe.root_module.addCSourceFile(.{ - .file = args.makesetup_out.path(b, "config.c"), + .file = args.setup.outfile orelse @panic("Missing config.c file"), .flags = &flags_common, }); @@ -646,6 +589,7 @@ fn addPythonExe( if (args.pyconfig.libs.zlib) |zlib| exe.root_module.linkLibrary(zlib); if (args.pyconfig.libs.openssl) |openssl| exe.root_module.linkLibrary(openssl); + if (args.pyconfig.libs.hacl) |hacl| exe.root_module.linkLibrary(hacl); if (target.result.os.tag == .windows) { exe.root_module.linkSystemLibrary("ws2_32", .{}); @@ -661,15 +605,14 @@ fn addPythonExe( const Pyconfig = struct { version: Version, libs: Libs, - header: union(enum) { - path: std.Build.LazyPath, - config_header: *std.Build.Step.ConfigHeader, - }, + header: std.Build.LazyPath, + sysconfigdata: ?other_libs, }; const Libs = struct { zlib: ?*std.Build.Step.Compile, openssl: ?*std.Build.Step.Compile, + hacl: ?*std.Build.Step.Compile, }; fn addPyconfig( @@ -684,214 +627,67 @@ fn addPyconfig( if (t.os.tag == .windows) return .{ .version = version, .libs = libs, - .header = .{ .path = upstream.path("PC/pyconfig.h") }, + .header = upstream.path("PC/pyconfig.h"), + .sysconfigdata = null, }; - const config_header = b.addConfigHeader(.{ - .style = .{ .autoconf_undef = upstream.path("pyconfig.h.in") }, - .include_path = "pyconfig.h", - }, .{ - .ALIGNOF_LONG = 8, - .ALIGNOF_SIZE_T = 8, - .DOUBLE_IS_LITTLE_ENDIAN_IEEE754 = 1, - .ENABLE_IPV6 = 1, - .MAJOR_IN_SYSMACROS = have(t.os.tag == .linux), - .PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT = 1, - .PTHREAD_SYSTEM_SCHED_SUPPORTED = 1, - .PY_BUILTIN_HASHLIB_HASHES = "md5,sha1,sha256,sha512,sha3,blake2", - .PY_COERCE_C_LOCALE = 1, - .PY_SSL_DEFAULT_CIPHERS = 1, - .PY_SUPPORT_TIER = 2, - .RETSIGTYPE = .void, - .SIZEOF_DOUBLE = 8, - .SIZEOF_FLOAT = 4, - .SIZEOF_FPOS_T = 16, - .SIZEOF_INT = 4, - .SIZEOF_LONG = 8, - .SIZEOF_LONG_DOUBLE = 16, - .SIZEOF_LONG_LONG = 8, - .SIZEOF_OFF_T = 8, - .SIZEOF_PID_T = 4, - .SIZEOF_PTHREAD_KEY_T = 4, - .SIZEOF_PTHREAD_T = 8, - .SIZEOF_SHORT = 2, - .SIZEOF_SIZE_T = 8, - .SIZEOF_TIME_T = 8, - .SIZEOF_UINTPTR_T = 8, - .SIZEOF_VOID_P = 8, - .SIZEOF_WCHAR_T = 4, - .SIZEOF__BOOL = 1, - .STDC_HEADERS = 1, - .SYS_SELECT_WITH_SYS_TIME = 1, - .WITH_DECIMAL_CONTEXTVAR = 1, - .WITH_DOC_STRINGS = 1, - .WITH_FREELISTS = 1, - .WITH_PYMALLOC = 1, - ._DARWIN_C_SOURCE = 1, - ._FILE_OFFSET_BITS = 64, - ._LARGEFILE_SOURCE = 1, - ._NETBSD_SOURCE = 1, - ._POSIX_C_SOURCE = .@"200809L", - ._PYTHONFRAMEWORK = "", - ._REENTRANT = 1, - ._XOPEN_SOURCE = 700, - ._XOPEN_SOURCE_EXTENDED = 1, - .__BSD_VISIBLE = 1, - ._ALL_SOURCE = 1, - ._GNU_SOURCE = 1, - ._POSIX_PTHREAD_SEMANTICS = 1, - ._TANDEM_SOURCE = 1, - .__EXTENSIONS__ = 1, - - .AC_APPLE_UNIVERSAL_BUILD = null, - .AIX_BUILDDATE = null, - .AIX_GENUINE_CPLUSPLUS = null, - .ALT_SOABI = null, - .ANDROID_API_LEVEL = null, - .DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 = null, - .DOUBLE_IS_BIG_ENDIAN_IEEE754 = null, - .GETPGRP_HAVE_ARG = null, - .MAJOR_IN_MKDEV = null, - .MVWDELCH_IS_EXPRESSION = null, - .PACKAGE_BUGREPORT = null, - .PACKAGE_NAME = null, - .PACKAGE_STRING = null, - .PACKAGE_TARNAME = null, - .PACKAGE_URL = null, - .PACKAGE_VERSION = null, - .POSIX_SEMAPHORES_NOT_ENABLED = null, - .PYLONG_BITS_IN_DIGIT = null, - .PY_SQLITE_ENABLE_LOAD_EXTENSION = null, - .PY_SQLITE_HAVE_SERIALIZE = null, - .PY_SSL_DEFAULT_CIPHER_STRING = null, - .Py_DEBUG = null, - .Py_ENABLE_SHARED = null, - .Py_HASH_ALGORITHM = null, - .Py_STATS = null, - .Py_SUNOS_VERSION = null, - .Py_TRACE_REFS = null, - .SETPGRP_HAVE_ARG = null, - .SIGNED_RIGHT_SHIFT_ZERO_FILLS = null, - .THREAD_STACK_SIZE = null, - .TIMEMODULE_LIB = null, - .TM_IN_SYS_TIME = null, - .USE_COMPUTED_GOTOS = null, - .WINDOW_HAS_FLAGS = null, - .WITH_DTRACE = null, - .WITH_DYLD = null, - .WITH_EDITLINE = null, - .WITH_LIBINTL = null, - .WITH_NEXT_FRAMEWORK = null, - .WITH_VALGRIND = null, - .X87_DOUBLE_ROUNDING = null, - ._BSD_SOURCE = null, - ._INCLUDE__STDC_A1_SOURCE = null, - ._LARGE_FILES = null, - ._MINIX = null, - ._POSIX_1_SOURCE = null, - ._POSIX_SOURCE = null, - ._POSIX_THREADS = null, - ._WASI_EMULATED_GETPID = null, - ._WASI_EMULATED_PROCESS_CLOCKS = null, - ._WASI_EMULATED_SIGNAL = null, - .clock_t = null, - .@"const" = null, - .gid_t = null, - .mode_t = null, - .off_t = null, - .pid_t = null, - .signed = null, - .size_t = null, - .socklen_t = null, - .uid_t = null, - .WORDS_BIGENDIAN = null, - - .HAVE_BROKEN_MBSTOWCS = null, - .HAVE_BROKEN_NICE = null, - .HAVE_BROKEN_PIPE_BUF = null, - .HAVE_BROKEN_POLL = null, - .HAVE_BROKEN_POSIX_SEMAPHORES = null, - .HAVE_BROKEN_PTHREAD_SIGMASK = null, - .HAVE_BROKEN_SEM_GETVALUE = null, - .HAVE_BROKEN_UNSETENV = null, - - .HAVE_PTHREAD_STUBS = null, - .HAVE_USABLE_WCHAR_T = null, - .HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION = null, - .HAVE_CHROOT = have(t.os.tag == .linux), - .HAVE_GCC_ASM_FOR_MC68881 = have(t.cpu.arch == .m68k), - .HAVE_GCC_ASM_FOR_X64 = have(t.cpu.arch == .x86_64), - .HAVE_GCC_ASM_FOR_X87 = have(t.cpu.arch == .x86_64 or t.cpu.arch == .x86), - // Readline type (static configuration, not function test) - .HAVE_RL_COMPDISP_FUNC_T = null, + const vers_config_file = switch (version) { + .@"3.11.13" => "config-3.11.13", + .@"3.12.11" => "config-3.12.11", + }; + const vers_config_file_path = b.path(vers_config_file); + b.dependOnFileContents(vers_config_file_path); + b.dependOnFileContents(b.path("config-common")); + + const pyconfig_h_in_path = upstream.path("pyconfig.h.in"); + b.dependOnFileContents(pyconfig_h_in_path); + + const run = b.addRunArtifact(configquery_exe); + run.addArg("--zig-exe"); + run.addArg(b.graph.zig_exe); + run.addArg("--cache-dir"); + run.addDirectoryArg(std.Build.LazyPath.cache_root); + run.addArg("-target"); + run.addArg(try target.query.zigTriple(b.allocator)); + run.addArg("-mcpu"); + run.addArg(try target.query.serializeCpuAlloc(b.allocator)); + run.addArg("-version"); + run.addArg(switch (version) { + .@"3.11.13" => "3.11.13", + .@"3.12.11" => "3.12.11", }); - switch (version) { - .@"3.11.13" => config_header.addValues(.{ - .PY_FORMAT_SIZE_T = "z", - .TIME_WITH_SYS_TIME = 1, - .FLOAT_WORDS_BIGENDIAN = null, - }), - .@"3.12.11" => config_header.addValues(.{ - .ALIGNOF_MAX_ALIGN_T = @as(u32, switch (t.cpu.arch) { - .x86_64, .aarch64 => 16, - .x86, .arm => 8, - else => 8, - }), - - // ncurses library (static configuration, not function test) - .HAVE_NCURSESW = null, - - // Performance trampoline feature - .PY_HAVE_PERF_TRAMPOLINE = null, - - ._HPUX_ALT_XOPEN_SOCKET_API = null, - ._OPENBSD_SOURCE = null, - - // C standard feature test macros (all set to null as they're optional) - .__STDC_WANT_IEC_60559_ATTRIBS_EXT__ = null, - .__STDC_WANT_IEC_60559_BFP_EXT__ = null, - .__STDC_WANT_IEC_60559_DFP_EXT__ = null, - .__STDC_WANT_IEC_60559_FUNCS_EXT__ = null, - .__STDC_WANT_IEC_60559_TYPES_EXT__ = null, - .__STDC_WANT_LIB_EXT2__ = null, - .__STDC_WANT_MATH_SPEC_FUNCS__ = null, - }), - } - - { - const run = b.addRunArtifact(configquery_exe); - run.addArg("--zig-exe"); - run.addArg(b.graph.zig_exe); - if (b.cache_root.path) |cache_root| { - run.addArg("--cache-dir"); - run.addArg(cache_root); - } - run.addArg("-target"); - run.addArg(try target.query.zigTriple(b.allocator)); - run.addArg("-mcpu"); - run.addArg(try target.query.serializeCpuAlloc(b.allocator)); - if (libs.zlib) |zlib| run.addPrefixedDirectoryArg("-I", zlib.getEmittedIncludeTree()); - if (libs.openssl) |openssl| run.addPrefixedDirectoryArg("-I", openssl.getEmittedIncludeTree()); - run.addFileArg(b.path("config-common")); - run.addFileArg(switch (version) { - .@"3.11.13" => b.path("config-3.11.13"), - .@"3.12.11" => b.path("config-3.12.11"), - }); - run.addArg("-o"); - ConfigHeaderExt.addFile(config_header, run.addOutputFileArg("config")); - } + run.addArg("-infile"); + run.addFileArg(pyconfig_h_in_path); + if (libs.zlib) |zlib| run.addPrefixedDirectoryArg("-I", zlib.getEmittedIncludeTree()); + if (libs.openssl) |openssl| run.addPrefixedDirectoryArg("-I", openssl.getEmittedIncludeTree()); + run.addFileArg(b.path("config-common")); + run.addFileArg(vers_config_file_path); + run.addArg("-o"); + const config_header = run.addOutputFileArg("pyconfig.h"); + + const sysconfigdata_contents = makesetup.writeSysConfigData(b); + defer b.allocator.free(sysconfigdata_contents); + + const sysconfigdata_filename = b.fmt("_sysconfigdata__{s}_.py", .{switch (t.os.tag) { + .linux => "linux", + .macos => "darwin", + .windows => "win32", + else => std.debug.panic("todo: populate PLATFORM for os '{s}'", .{@tagName(t.os.tag)}), + }}); + + const wf = b.addWriteFiles(); + const sysconfigdata_file = wf.add(sysconfigdata_filename, sysconfigdata_contents); + const install_step = b.addInstallFile(sysconfigdata_file, b.fmt("lib/python{s}/{s}", .{ version.libName(), sysconfigdata_filename })); + b.getInstallStep().dependOn(&install_step.step); return .{ .version = version, .libs = libs, - .header = .{ .config_header = config_header }, + .header = config_header, + .sysconfigdata = .{ .path = sysconfigdata_file, .name = sysconfigdata_filename }, }; } -fn have(x: bool) ?u1 { - return if (x) 1 else null; -} - const Config = struct { name: []const u8, string: []const u8 }; fn concatConfigs(comptime first: anytype, comptime second: anytype) [std.meta.fields(@TypeOf(first)).len + std.meta.fields(@TypeOf(second)).len]Config { const first_len = std.meta.fields(@TypeOf(first)).len; @@ -1134,13 +930,11 @@ fn ci( upstream: *std.Build.Dependency, ci_step: *std.Build.Step, args: struct { - replace_exe: *std.Build.Step.Compile, - makesetup_exe: *std.Build.Step.Compile, - configquery_exe: *std.Build.Step.Compile, stage2_frozen_mods: Stage2FrozenMods, frozen_headers: []const std.Build.LazyPath, deepfreeze_c: std.Build.LazyPath, }, + configquery_exe: *std.Build.Step.Compile, ) !void { const CiTarget = struct { triple: []const u8, ssl: bool }; const ci_targets = [_]CiTarget{ @@ -1172,6 +966,25 @@ fn ci( const install = b.step(b.fmt("install-{s}", .{ci_target.triple}), ""); ci_step.dependOn(install); + var hacl: ?*std.Build.Step.Compile = undefined; + if (version == .@"3.12.11") { + hacl = b.addLibrary(.{ + .name = "hacl", + .linkage = .static, + .root_module = b.createModule(.{ + .target = target, + .optimize = optimize, + .link_libc = true, + }), + }); + hacl.?.root_module.addIncludePath(upstream.path("./Modules/_hacl/include")); + hacl.?.root_module.addIncludePath(upstream.path("./Modules/_hacl/")); + hacl.?.root_module.addCSourceFile(.{ .file = upstream.path("./Modules/_hacl/Hacl_Hash_SHA2.c"), .flags = &.{ + "-D_BSD_SOURCE", "-D_DEFAULT_SOURCE", "-fwrapv", + "-std=c11", "-fvisibility=hidden", "-DVPATH=\"\"", + } }); + } else hacl = null; + const libs: Libs = .{ .zlib = (b.dependency("zlib", .{ .target = target, @@ -1181,16 +994,13 @@ fn ci( .target = target, .optimize = optimize, })) |dep| dep.artifact("openssl") else null) else null, + .hacl = hacl, }; - const makesetup = addMakesetup(b, version, upstream, libs, .{ - .os_tag = target.result.os.tag, - .replace_exe = args.replace_exe, - .makesetup_exe = args.makesetup_exe, - }); + const makesetups = addMakesetup(b, version, upstream, libs, target.result.os.tag); const exe = addPythonExe(b, upstream, target, optimize, .{ .name = "python", - .makesetup_out = makesetup, - .pyconfig = try addPyconfig(b, version, upstream, target, libs, args.configquery_exe), + .setup = makesetups, + .pyconfig = try addPyconfig(b, version, upstream, target, libs, configquery_exe), .stage = .{ .final = .{ .stage2 = args.stage2_frozen_mods, .frozen_headers = args.frozen_headers, @@ -1222,4 +1032,5 @@ fn concat(allocator: std.mem.Allocator, lists: []const []const []const u8) []con } const std = @import("std"); -const ConfigHeaderExt = @import("ConfigHeaderExt.zig"); +const replace = @import("replace.zig"); +const makesetup = @import("makesetup.zig"); diff --git a/build.zig.zon b/build.zig.zon index 97def5a..a51558f 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,8 +1,8 @@ .{ .name = .cpython, .fingerprint = 0x3224901d1babdd14, - .version = "0.1.0", - .minimum_zig_version = "0.16.0", + .version = "0.2.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ .@"upstream_3.11.13" = .{ .url = "git+https://github.com/python/cpython#498b971ea3673012a1d4b21860b229d55fc6e575", @@ -31,6 +31,8 @@ "makesetup.zig", "replace.zig", "ConfigHeaderExt.zig", - "config-*", + "config-common", + "config-3.11.13", + "config-3.12.11", }, } diff --git a/configquery.zig b/configquery.zig index 4339f9f..000bab7 100644 --- a/configquery.zig +++ b/configquery.zig @@ -5,6 +5,7 @@ pub fn main(init: std.process.Init) !void { var template_files: std.ArrayList([]const u8) = .empty; var output_path: ?[]const u8 = null; var config: Config = .{}; + var log_error_comments = false; { var args = try init.minimal.args.iterateAllocator(arena); @@ -20,6 +21,12 @@ pub fn main(init: std.process.Init) !void { config.target_triple = args.next() orelse fatal("-target requires an argument", .{}); } else if (std.mem.eql(u8, arg, "-mcpu")) { config.mcpu = args.next() orelse fatal("-mcpu requires an argument", .{}); + } else if (std.mem.eql(u8, arg, "-version")) { + config.version = args.next() orelse fatal("-version requires an argument", .{}); + } else if (std.mem.eql(u8, arg, "-infile")) { + config.infile = args.next() orelse fatal("-infile requires an argument", .{}); + } else if (std.mem.eql(u8, arg, "-log_err")) { + log_error_comments = true; } else if (std.mem.startsWith(u8, arg, "-I")) { config.include_dirs.append(arena, arg) catch |e| oom(e); } else { @@ -41,7 +48,10 @@ pub fn main(init: std.process.Init) !void { \\ --zig-exe [exe] Path to zig executable (for compile queries) \\ -target [name] -- see the targets command \\ -mcpu [cpu] Specify target CPU and feature set + \\ -version [version] Specify Python version + \\ -infile [python.h.in] Path to python.h.in file \\ -I[dir] Add directory to include search path + \\ -log_err Log to stderr the failed query output \\ --cache-dir [path] The local cache directory \\ \\Literal Values @@ -68,6 +78,14 @@ pub fn main(init: std.process.Init) !void { std.process.exit(1); } + const config_h_in = std.Io.Dir.cwd().readFileAlloc(io, config.infile.?, arena, .unlimited) catch |err| fatal( + "failed to read '{s}' with {t}", + .{ config.infile.?, err }, + ); + + var config_header: std.array_hash_map.String(ConfigHeader.Value) = .empty; + try initialize_config(io, arena, config, &config_header); + var out_buf: [4096]u8 = undefined; var out_file = if (output_path) |path| std.Io.Dir.cwd().createFile(io, path, .{}) catch |err| fatal( "failed to create '{s}' with {t}", @@ -98,7 +116,12 @@ pub fn main(init: std.process.Init) !void { }, .literal => |lit| { checkUnique(arena, &name_set, io, template_path, line_num, lit.config_name, &error_count); - out.print("{s} {s}\n", .{ lit.config_name, lit.value_str }) catch return out_writer.err.?; + if (parseValue(lit.value_str)) |entry| { + config_header.put(arena, lit.config_name, entry) catch @panic("OOM"); + } else { + error_count += 1; + reportError(io, template_path, line_num, "{s} '{s}'", .{ lit.config_name, line_untrimmed }); + } }, .query => |input| { checkUnique(arena, &name_set, io, template_path, line_num, input.config_name, &error_count); @@ -109,7 +132,7 @@ pub fn main(init: std.process.Init) !void { .line_num = line_num, }) catch |e| oom(e); } else { - out.print("{s} {s}\n", .{ input.config_name, input.pass_text }) catch return out_writer.err.?; + config_header.put(arena, input.config_name, .{ .string = input.pass_text }) catch @panic("OOM"); } }, } @@ -119,7 +142,7 @@ pub fn main(init: std.process.Init) !void { if (error_count > 0) fatal("{} errors in config templates", .{error_count}); if (queries.items.len > 0) { - const zig_exe = config.zig_exe orelse fatal("compile query requires --zig-exe", .{}); + const zig_exe = config.zig_exe orelse fatal("compile query requires zig-exe", .{}); const cache_dir = config.cache_dir orelse fatal("compile query requires --cache-dir", .{}); const argv_template = buildArgvTemplate(arena, zig_exe, &config); @@ -133,24 +156,44 @@ pub fn main(init: std.process.Init) !void { for (queries.items) |*q| { const result = q.result orelse fatal("compile query for '{s}' has no result", .{q.input.config_name}); - const value_text = if (result.success) q.input.pass_text else q.input.fail_text; - out.print("{s}{s} {s}\n", .{ result.error_comments, q.input.config_name, value_text }) catch return out_writer.err.?; if (result.fatal_msg) |msg| { error_count += 1; reportError(io, q.file_path, q.line_num, "{s}", .{msg}); + } else { + const value_text = if (result.success) q.input.pass_text else q.input.fail_text; + + if (parseValue(value_text)) |entry| { + config_header.put(arena, q.input.config_name, entry) catch @panic("OOM"); + } else { + error_count += 1; + reportError(io, q.file_path, q.line_num, "{s} '{s}'", .{ q.input.config_name, value_text }); + } + if (result.error_comments.len > 0 and log_error_comments) { + std.log.debug("{s}{s} {s} : {s}\n", .{ result.error_comments, q.input.config_name, q.input.pass_text, q.input.fail_text }); + } } } } + const all_keys = config_header.keys(); + + var value_map: ValueMap = .empty; + try value_map.ensureTotalCapacity(arena, all_keys.len); + for (all_keys) |key| value_map.putAssumeCapacityNoClobber(key, false); + + try renderAutoConfUndef(config_h_in, out, &config_header, &value_map, config.infile.?); + out.flush() catch return out_writer.err.?; if (error_count > 0) fatal("{} errors", .{error_count}); } -const Config = struct { +pub const Config = struct { zig_exe: ?[]const u8 = null, cache_dir: ?[]const u8 = null, target_triple: ?[]const u8 = null, mcpu: ?[]const u8 = null, + version: ?[]const u8 = null, + infile: ?[]const u8 = null, include_dirs: std.ArrayList([]const u8) = .empty, }; @@ -185,6 +228,255 @@ const Query = struct { }; }; +fn have(x: bool) ?u1 { + return if (x) 1 else null; +} + +fn initialize_config(io: std.Io, alloc: std.mem.Allocator, config: Config, config_header: *std.array_hash_map.String(ConfigHeader.Value)) !void { + const query = try std.Target.Query.parse(.{ .arch_os_abi = config.target_triple.? }); + const t = try std.zig.system.resolveTargetQuery(io, query); + + addValues(alloc, config_header, .{ + .ALIGNOF_LONG = 8, + .ALIGNOF_SIZE_T = 8, + .DOUBLE_IS_LITTLE_ENDIAN_IEEE754 = 1, + .ENABLE_IPV6 = 1, + .MAJOR_IN_SYSMACROS = have(t.os.tag == .linux), + .PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT = 1, + .PTHREAD_SYSTEM_SCHED_SUPPORTED = 1, + .PY_BUILTIN_HASHLIB_HASHES = "md5,sha1,sha256,sha512,sha3,blake2", + .PY_COERCE_C_LOCALE = 1, + .PY_SSL_DEFAULT_CIPHERS = 1, + .PY_SUPPORT_TIER = 2, + .RETSIGTYPE = .void, + .SIZEOF_DOUBLE = 8, + .SIZEOF_FLOAT = 4, + .SIZEOF_FPOS_T = 16, + .SIZEOF_INT = 4, + .SIZEOF_LONG = 8, + .SIZEOF_LONG_DOUBLE = 16, + .SIZEOF_LONG_LONG = 8, + .SIZEOF_OFF_T = 8, + .SIZEOF_PID_T = 4, + .SIZEOF_PTHREAD_KEY_T = 4, + .SIZEOF_PTHREAD_T = 8, + .SIZEOF_SHORT = 2, + .SIZEOF_SIZE_T = 8, + .SIZEOF_TIME_T = 8, + .SIZEOF_UINTPTR_T = 8, + .SIZEOF_VOID_P = 8, + .SIZEOF_WCHAR_T = 4, + .SIZEOF__BOOL = 1, + .STDC_HEADERS = 1, + .SYS_SELECT_WITH_SYS_TIME = 1, + .WITH_DECIMAL_CONTEXTVAR = 1, + .WITH_DOC_STRINGS = 1, + .WITH_FREELISTS = 1, + .WITH_PYMALLOC = 1, + ._DARWIN_C_SOURCE = 1, + ._FILE_OFFSET_BITS = 64, + ._LARGEFILE_SOURCE = 1, + ._NETBSD_SOURCE = 1, + ._POSIX_C_SOURCE = .@"200809L", + ._PYTHONFRAMEWORK = "", + ._REENTRANT = 1, + ._XOPEN_SOURCE = 700, + ._XOPEN_SOURCE_EXTENDED = 1, + .__BSD_VISIBLE = 1, + ._ALL_SOURCE = 1, + ._GNU_SOURCE = 1, + ._POSIX_PTHREAD_SEMANTICS = 1, + ._TANDEM_SOURCE = 1, + .__EXTENSIONS__ = 1, + + .AC_APPLE_UNIVERSAL_BUILD = null, + .AIX_BUILDDATE = null, + .AIX_GENUINE_CPLUSPLUS = null, + .ALT_SOABI = null, + .ANDROID_API_LEVEL = null, + .DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 = null, + .DOUBLE_IS_BIG_ENDIAN_IEEE754 = null, + .GETPGRP_HAVE_ARG = null, + .MAJOR_IN_MKDEV = null, + .MVWDELCH_IS_EXPRESSION = null, + .PACKAGE_BUGREPORT = null, + .PACKAGE_NAME = null, + .PACKAGE_STRING = null, + .PACKAGE_TARNAME = null, + .PACKAGE_URL = null, + .PACKAGE_VERSION = null, + .POSIX_SEMAPHORES_NOT_ENABLED = null, + .PYLONG_BITS_IN_DIGIT = null, + .PY_SQLITE_ENABLE_LOAD_EXTENSION = null, + .PY_SQLITE_HAVE_SERIALIZE = null, + .PY_SSL_DEFAULT_CIPHER_STRING = null, + .Py_DEBUG = null, + .Py_ENABLE_SHARED = null, + .Py_HASH_ALGORITHM = null, + .Py_STATS = null, + .Py_SUNOS_VERSION = null, + .Py_TRACE_REFS = null, + .SETPGRP_HAVE_ARG = null, + .SIGNED_RIGHT_SHIFT_ZERO_FILLS = null, + .THREAD_STACK_SIZE = null, + .TIMEMODULE_LIB = null, + .TM_IN_SYS_TIME = null, + .USE_COMPUTED_GOTOS = null, + .WINDOW_HAS_FLAGS = null, + .WITH_DTRACE = null, + .WITH_DYLD = null, + .WITH_EDITLINE = null, + .WITH_LIBINTL = null, + .WITH_NEXT_FRAMEWORK = null, + .WITH_VALGRIND = null, + .X87_DOUBLE_ROUNDING = null, + ._BSD_SOURCE = null, + ._INCLUDE__STDC_A1_SOURCE = null, + ._LARGE_FILES = null, + ._MINIX = null, + ._POSIX_1_SOURCE = null, + ._POSIX_SOURCE = null, + ._POSIX_THREADS = null, + ._WASI_EMULATED_GETPID = null, + ._WASI_EMULATED_PROCESS_CLOCKS = null, + ._WASI_EMULATED_SIGNAL = null, + .clock_t = null, + .@"const" = null, + .gid_t = null, + .mode_t = null, + .off_t = null, + .pid_t = null, + .signed = null, + .size_t = null, + .socklen_t = null, + .uid_t = null, + .WORDS_BIGENDIAN = null, + + .HAVE_BROKEN_MBSTOWCS = null, + .HAVE_BROKEN_NICE = null, + .HAVE_BROKEN_PIPE_BUF = null, + .HAVE_BROKEN_POLL = null, + .HAVE_BROKEN_POSIX_SEMAPHORES = null, + .HAVE_BROKEN_PTHREAD_SIGMASK = null, + .HAVE_BROKEN_SEM_GETVALUE = null, + .HAVE_BROKEN_UNSETENV = null, + + .HAVE_PTHREAD_STUBS = null, + .HAVE_USABLE_WCHAR_T = null, + .HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION = null, + .HAVE_CHROOT = have(t.os.tag == .linux), + .HAVE_GCC_ASM_FOR_MC68881 = have(t.cpu.arch == .m68k), + .HAVE_GCC_ASM_FOR_X64 = have(t.cpu.arch == .x86_64), + .HAVE_GCC_ASM_FOR_X87 = have(t.cpu.arch == .x86_64 or t.cpu.arch == .x86), + // Readline type (static configuration, not function test) + .HAVE_RL_COMPDISP_FUNC_T = null, + }); + if (std.mem.eql(u8, config.version.?, "3.11.13")) addValues(alloc, config_header, .{ + .PY_FORMAT_SIZE_T = "z", + .TIME_WITH_SYS_TIME = 1, + .FLOAT_WORDS_BIGENDIAN = null, + }); + if (std.mem.eql(u8, config.version.?, "3.12.11")) + addValues(alloc, config_header, .{ + .ALIGNOF_MAX_ALIGN_T = @as(u32, switch (t.cpu.arch) { + .x86_64, .aarch64 => 16, + .x86, .arm => 8, + else => 8, + }), + + // ncurses library (static configuration, not function test) + .HAVE_NCURSESW = null, + + // Performance trampoline feature + .PY_HAVE_PERF_TRAMPOLINE = null, + + ._HPUX_ALT_XOPEN_SOCKET_API = null, + ._OPENBSD_SOURCE = null, + + // C standard feature test macros (all set to null as they're optional) + .__STDC_WANT_IEC_60559_ATTRIBS_EXT__ = null, + .__STDC_WANT_IEC_60559_BFP_EXT__ = null, + .__STDC_WANT_IEC_60559_DFP_EXT__ = null, + .__STDC_WANT_IEC_60559_FUNCS_EXT__ = null, + .__STDC_WANT_IEC_60559_TYPES_EXT__ = null, + .__STDC_WANT_LIB_EXT2__ = null, + .__STDC_WANT_MATH_SPEC_FUNCS__ = null, + }); +} +pub fn addValue(arena: std.mem.Allocator, config_header: *std.array_hash_map.String(ConfigHeader.Value), name: []const u8, comptime T: type, value: T) void { + return addValueInner(arena, config_header, name, T, value) catch @panic("OOM"); +} + +fn addValueInner(arena: std.mem.Allocator, config_header: *std.array_hash_map.String(ConfigHeader.Value), name: []const u8, comptime T: type, value: T) !void { + switch (@typeInfo(T)) { + .null => { + try config_header.put(arena, name, .undef); + }, + .void => { + try config_header.put(arena, name, .defined); + }, + .bool => { + try config_header.put(arena, name, .{ .boolean = value }); + }, + .int => { + try config_header.put(arena, name, .{ .int = value }); + }, + .comptime_int => { + try config_header.put(arena, name, .{ .int = value }); + }, + .@"enum", .enum_literal => { + try config_header.put(arena, name, .{ .ident = @tagName(value) }); + }, + .optional => { + if (value) |x| { + return addValueInner(arena, config_header, name, @TypeOf(x), x); + } else { + try config_header.put(arena, name, .undef); + } + }, + .pointer => |ptr| { + switch (@typeInfo(ptr.child)) { + .array => |array| { + if (ptr.size == .one and array.child == u8) { + try config_header.put(arena, name, .{ .string = value }); + return; + } + }, + .int => { + if (ptr.size == .slice and ptr.child == u8) { + try config_header.put(arena, name, .{ .string = value }); + return; + } + }, + else => {}, + } + + @compileError("unsupported ConfigHeader value type: " ++ @typeName(T)); + }, + else => @compileError("unsupported ConfigHeader value type: " ++ @typeName(T)), + } +} + +pub fn addValues(arena: std.mem.Allocator, config_header: *std.array_hash_map.String(ConfigHeader.Value), values: anytype) void { + const info = @typeInfo(@TypeOf(values)).@"struct"; + inline for (info.field_names, info.field_types) |field_name, field_type| { + addValue(arena, config_header, field_name, field_type, @field(values, field_name)); + } +} + +fn parseValue(ostr: ?[]const u8) ?ConfigHeader.Value { + if (ostr == null) return .undef; + const str = ostr.?; + if (std.mem.eql(u8, str, "undef")) return .undef; + if (std.mem.eql(u8, str, "defined")) return .defined; + if (std.mem.eql(u8, str, "false")) return .{ .boolean = false }; + if (std.mem.eql(u8, str, "true")) return .{ .boolean = true }; + if (std.fmt.parseInt(i64, str, 10)) |int| return .{ .int = int } else |_| {} + if (std.mem.startsWith(u8, str, ":")) return .{ .ident = str[1..] }; + if (str.len >= 2 and str[0] == '"' and str[str.len - 1] == '"') return .{ .string = str[1 .. str.len - 1] }; + return null; +} + fn parseLine(line_untrimmed: []const u8) Line { const line = std.mem.trim(u8, line_untrimmed, &std.ascii.whitespace); if (line.len == 0 or line[0] == '#') return .skip; @@ -194,18 +486,18 @@ fn parseLine(line_untrimmed: []const u8) Line { const value_str = std.mem.trimStart(u8, line[first_space + 1 ..], " "); if (!std.mem.startsWith(u8, value_str, "?")) { - _ = ConfigHeaderExt.parseValue(value_str) orelse + _ = parseValue(value_str) orelse return .{ .err = "invalid config value" }; return .{ .literal = .{ .config_name = config_name, .value_str = value_str } }; } const pass_str, const after_pass = scanConfigValue(value_str, 1); - _ = ConfigHeaderExt.parseValue(pass_str) orelse return .{ .err = "expected PASS_VALUE after ?" }; + _ = parseValue(pass_str) orelse return .{ .err = "expected PASS_VALUE after ?" }; const pipe_index = skipWhitespace(value_str, after_pass); if (pipe_index >= value_str.len or value_str[pipe_index] != '|') return .{ .err = "expected '|' after PASS_VALUE" }; const fail_str, const after_fail = scanConfigValue(value_str, pipe_index + 1); - _ = ConfigHeaderExt.parseValue(fail_str) orelse return .{ .err = "expected FAIL_VALUE after '|'" }; + _ = parseValue(fail_str) orelse return .{ .err = "expected FAIL_VALUE after '|'" }; var rest = std.mem.trimStart(u8, value_str[after_fail..], " "); const defines = parseExprClause(&rest, "define="); const includes = parseExprClause(&rest, "include="); @@ -346,8 +638,6 @@ fn doCompileCheck( const argv = blk: { var a = std.ArrayList([]const u8).initCapacity(arena, argv_template.len + 3) catch |e| oom(e); a.appendSliceAssumeCapacity(argv_template); - a.appendAssumeCapacity("--cache-dir"); - a.appendAssumeCapacity(task_dir); a.appendAssumeCapacity(source_path); break :blk a.items; }; @@ -496,6 +786,101 @@ test parseLine { try std.testing.expect(parseLine("NOVALUE") == .err); } +fn ensureAllValuesUsed( + value_map: *ValueMap, + src_path: []const u8, +) !void { + var any_errors = false; + for (value_map.keys(), value_map.values()) |name, used| { + if (used) continue; + std.log.err("{s}: config header value unused: {s}", .{ src_path, name }); + any_errors = true; + } + if (any_errors) unreachable; +} + +const os_newline = if (@import("builtin").os.tag == .windows) "\r\n" else "\n"; + +fn detectNewline(contents: []const u8) []const u8 { + const lf_index = std.mem.findScalar(u8, contents, '\n') orelse return os_newline; + return if (lf_index > 0 and contents[lf_index - 1] == '\r') "\r\n" else "\n"; +} + +const header_text = "This file was generated by ConfigHeader using the Zig Build System."; +const c_generated_line = "/* " ++ header_text ++ " */"; +const asm_generated_line = "; " ++ header_text; + +fn renderAutoConfUndef( + contents: []const u8, + w: *Writer, + config_header: *const std.array_hash_map.String(ConfigHeader.Value), + value_map: *ValueMap, + src_path: []const u8, +) !void { + const newline = detectNewline(contents); + + try w.writeAll(c_generated_line); + try w.writeAll(newline); + + var any_errors = false; + var line_index: u32 = 0; + var line_it = std.mem.splitScalar(u8, contents, '\n'); + while (line_it.next()) |raw_line| : (line_index += 1) { + const last_line = line_it.index == line_it.buffer.len; + const line = std.mem.trimEnd(u8, raw_line, "\r"); + + if (!std.mem.startsWith(u8, line, "#")) { + try w.writeAll(line); + if (!last_line) try w.writeAll(newline); + continue; + } + var it = std.mem.tokenizeAny(u8, line[1..], " \t\r"); + const undef = it.next().?; + if (!std.mem.eql(u8, undef, "undef")) { + try w.writeAll(line); + if (!last_line) try w.writeAll(newline); + continue; + } + const name = it.next().?; + const index = value_map.getIndex(name) orelse { + std.log.err("{s}:{d}: unspecified config header value: {s}", .{ + src_path, line_index + 1, name, + }); + any_errors = true; + continue; + }; + value_map.values()[index] = true; // Set to used. + try renderValueC(w, newline, name, config_header.get(name).?); + } + + try ensureAllValuesUsed(value_map, src_path); + if (any_errors) return error.MakeFailed; +} + +fn renderValueC(w: *Writer, newline: []const u8, name: []const u8, value: ConfigHeader.Value) !void { + switch (value) { + .undef => try w.print("/* #undef {s} */{s}", .{ name, newline }), + .defined => try w.print("#define {s}{s}", .{ name, newline }), + .boolean => |b| return renderValueCBool(w, newline, name, b), + .int => |i| try w.print("#define {s} {d}{s}", .{ name, i, newline }), + .ident => |ident| return renderValueCIdent(w, newline, name, ident), + .string => |string| try w.print("#define {s} \"{f}\"{s}", .{ name, std.zig.fmtString(string), newline }), + } +} + +fn renderValueCBool(w: *Writer, newline: []const u8, name: []const u8, b: bool) Writer.Error!void { + return w.print("#define {s} {c}{s}", .{ name, @as(u8, '0') + @intFromBool(b), newline }); +} + +fn renderValueCIdent(w: *Writer, newline: []const u8, name: []const u8, ident: []const u8) Writer.Error!void { + try w.print("#define {s}", .{name}); + if (ident.len > 0) { + try w.writeByte(' '); + try w.writeAll(ident); + } + return w.writeAll(newline); +} + fn reportError(io: std.Io, file: []const u8, line: u32, comptime fmt: []const u8, args: anytype) void { var stderr = std.Io.File.stderr().writer(io, &.{}); stderr.interface.print("{s}:{}: " ++ fmt ++ "\n", .{ file, line } ++ args) catch @@ -512,4 +897,7 @@ fn fatal(comptime fmt: []const u8, args: anytype) noreturn { } const std = @import("std"); -const ConfigHeaderExt = @import("ConfigHeaderExt.zig"); +const ConfigHeader = std.Build.Step.ConfigHeader; +const LazyPath = std.Build.LazyPath; +const Writer = std.Io.Writer; +const ValueMap = std.array_hash_map.String(bool); diff --git a/makesetup.zig b/makesetup.zig index 1b8e080..8a7d959 100644 --- a/makesetup.zig +++ b/makesetup.zig @@ -4,96 +4,53 @@ const print = std.debug.print; const ArrayListUnmanaged = std.ArrayListUnmanaged; const Allocator = std.mem.Allocator; -pub fn main(init: std.process.Init) !void { - const arena = init.arena.allocator(); - const all_args = try init.minimal.args.toSlice(arena); - const io = init.io; +pub fn makesetup(b: *std.Build, setup_files: []const []const u8, setup_files_src_paths: []const []const u8) Setup { + const arena = b.allocator; + const setup = try parseSetupFiles(arena, setup_files, setup_files_src_paths); - if (all_args.len <= 1) { - var stderr = std.Io.File.stderr().writer(io, &.{}); - try stderr.interface.writeAll("usage: makesetup UPSTREAM_SRC OUT_DIR SETUP_FILES...\n"); - std.process.exit(0xff); - } - const args = all_args[1..]; - if (args.len < 3) errExit("expected at least 3 cmdline args but got {}", .{args.len}); - - const upstream_src = args[0]; - const out_path = args[1]; - const setup_files = args[2..]; - - const setup = try parseSetupFiles(arena, io, setup_files); - - const config_in_path = std.Io.Dir.path.join(arena, &.{ upstream_src, "Modules", "config.c.in" }) catch |e| oom(e); - const config_in = std.Io.Dir.cwd().readFileAlloc(io, config_in_path, arena, .unlimited) catch |e| - std.debug.panic("open file '{s}' failed with {t}", .{ config_in_path, e }); - - var out_dir = try std.Io.Dir.cwd().createDirPathOpen(io, out_path, .{}); - defer out_dir.close(io); - - { - var file = try out_dir.createFile(io, "config.c", .{}); - defer file.close(io); - var out_file_buf: [4096]u8 = undefined; - var file_writer = file.writer(io, &out_file_buf); - writeConfigC(setup, config_in_path, config_in, &file_writer.interface) catch |err| switch (err) { - error.WriteFailed => return file_writer.err orelse error.Unexpected, - }; - } + return setup; - { - var out_file = try out_dir.createFile(io, "module-compile-args.txt", .{}); - defer out_file.close(io); - var out_file_buf: [4096]u8 = undefined; - var file_writer = out_file.writer(io, &out_file_buf); - writeCompileArgs(setup, &file_writer.interface) catch |err| switch (err) { - error.WriteFailed => return file_writer.err orelse error.Unexpected, - }; - } + // { + // var out_file = try out_dir.createFile(io, "module-compile-args.txt", .{}); + // defer out_file.close(io); + // var out_file_buf: [4096]u8 = undefined; + // var file_writer = out_file.writer(io, &out_file_buf); + // writeCompileArgs(setup, &file_writer.interface) catch |err| switch (err) { + // error.WriteFailed => return file_writer.err orelse error.Unexpected, + // }; + // } } -fn writeConfigC( +pub fn writeConfigC( + b: *std.Build, setup: Setup, config_in_path: []const u8, config_in: []const u8, - writer: *std.Io.Writer, -) error{WriteFailed}!void { - try writer.print("/* Generated automatically from {s} by makesetup. */\n", .{config_in_path}); +) []const u8 { + const alloc = b.allocator; + var list: std.ArrayList(u8) = .empty; + defer list.deinit(alloc); + + list.appendSlice(alloc, b.fmt("/* Generated automatically from {s} by makesetup. */\n", .{config_in_path})) catch @panic("OOM"); var lines = std.mem.splitScalar(u8, config_in, '\n'); while (lines.next()) |line| { if (std.mem.indexOf(u8, line, "MARKER 1")) |_| { var it = setup.modules.iterator(); while (it.next()) |entry| { if (!entry.value_ptr.enabled) continue; - try writer.print("extern PyObject* PyInit_{s}(void);\n", .{entry.key_ptr.*}); + list.appendSlice(alloc, b.fmt("extern PyObject* PyInit_{s}(void);\n", .{entry.key_ptr.*})) catch @panic("OOM"); } } else if (std.mem.indexOf(u8, line, "MARKER 2")) |_| { var it = setup.modules.iterator(); while (it.next()) |entry| { if (!entry.value_ptr.enabled) continue; - try writer.print(" {{\"{s}\", PyInit_{0s}}},\n", .{entry.key_ptr.*}); + list.appendSlice(alloc, b.fmt(" {{\"{s}\", PyInit_{0s}}},\n", .{entry.key_ptr.*})) catch @panic("OOM"); } } - try writer.writeAll(line); - try writer.writeByte('\n'); + list.appendSlice(alloc, line) catch @panic("OOM"); + list.append(alloc, '\n') catch @panic("OOM"); } - try writer.flush(); -} - -fn writeCompileArgs(setup: Setup, writer: *std.Io.Writer) error{WriteFailed}!void { - var it = setup.modules.iterator(); - while (it.next()) |entry| { - const module_name = entry.key_ptr.*; - { - const suffix: []const u8 = if (entry.value_ptr.enabled) "" else " (DISABLED)"; - try writer.print("# Module '{s}'{s}\n", .{ module_name, suffix }); - } - const prefix: []const u8 = if (entry.value_ptr.enabled) "" else "# "; - for (entry.value_ptr.compile_args) |compile_arg| switch (compile_arg) { - .source => |src| try writer.print("{s}Modules/{s}\n", .{ prefix, src }), - .include => |inc| try writer.print("{s}-I{s}\n", .{ prefix, inc }), - }; - } - try writer.flush(); + return list.toOwnedSlice(alloc) catch @panic("OOM"); } const CompileArg = union(enum) { @@ -109,15 +66,14 @@ const Module = struct { compile_args: []const CompileArg, libs: []const []const u8, }; -const Setup = struct { +pub const Setup = struct { modules: std.StringArrayHashMapUnmanaged(Module) = .{}, + outfile: ?std.Build.LazyPath = null, }; -fn parseSetupFiles(arena: std.mem.Allocator, io: Io, setup_files: []const []const u8) !Setup { +fn parseSetupFiles(arena: std.mem.Allocator, setup_files: []const []const u8, setup_files_src_paths: []const []const u8) !Setup { var setup: Setup = .{}; - for (setup_files) |file_path| { - const content = std.Io.Dir.cwd().readFileAlloc(io, file_path, arena, .unlimited) catch |e| - std.debug.panic("open '{s}' failed with {t}", .{ file_path, e }); + for (setup_files, setup_files_src_paths) |content, file_path| { try parseSetupFile(arena, &setup, file_path, content); } return setup; @@ -188,7 +144,7 @@ fn parseSetupFile( if (std.mem.startsWith(u8, part, "#")) { break; } else if (std.mem.endsWith(u8, part, ".c")) { - compile_args.append(arena, .{ .source = part }) catch |e| oom(e); + compile_args.append(arena, .{ .source = arena.dupe(u8, part) catch |e| oom(e) }) catch |e| oom(e); } else if (std.mem.endsWith(u8, part, ".a")) { libraries.append(arena, part) catch |e| oom(e); } else if (std.mem.startsWith(u8, part, "-D")) { @@ -196,7 +152,9 @@ fn parseSetupFile( defines.append(arena, part[2..]) catch |e| oom(e); } else if (std.mem.startsWith(u8, part, "-I")) { if (part.len == 2) errExit("{s}:{}: expected '-IPATH' but just got '-I'", .{ file_path, lineno }); - compile_args.append(arena, .{ .include = part[2..] }) catch |e| oom(e); + if (part.len > 11 and std.mem.startsWith(u8, part, "-I$(srcdir)")) { + compile_args.append(arena, .{ .include = arena.dupe(u8, part[11..]) catch |e| oom(e) }) catch |e| oom(e); + } else compile_args.append(arena, .{ .include = arena.dupe(u8, part[2..]) catch |e| oom(e) }) catch |e| oom(e); } else if (std.mem.startsWith(u8, part, "-l")) { if (part.len == 2) errExit("{s}:{}: expected '-lLIB' but just got '-l'", .{ file_path, lineno }); libs.append(arena, part[2..]) catch |e| oom(e); @@ -287,3 +245,17 @@ fn errExit(comptime fmt: []const u8, args: anytype) noreturn { std.log.err(fmt, args); std.process.exit(0xff); } + +pub fn writeSysConfigData(b: *std.Build) []const u8 { + const alloc = b.allocator; + var list: std.ArrayList(u8) = .empty; + defer list.deinit(alloc); + + list.appendSlice(alloc, "build_time_vars = {'ABIFLAGS': ''") catch @panic("OOM"); + + // TODO: Output the values of the config header in a python format + + list.appendSlice(alloc, "}\n") catch @panic("OOM"); + + return list.toOwnedSlice(alloc) catch @panic("OOM"); +} diff --git a/replace.zig b/replace.zig index adb4c51..6f64329 100644 --- a/replace.zig +++ b/replace.zig @@ -1,22 +1,9 @@ const std = @import("std"); const Io = std.Io; -pub fn main(init: std.process.Init) !void { - const arena = init.arena.allocator(); - const all_args = try init.minimal.args.toSlice(arena); - const io = init.io; - - if (all_args.len <= 1) { - var stderr = std.Io.File.stderr().writer(io, &.{}); - try stderr.interface.writeAll("usage: replace IN_FILE OUT_FILE NAME=VALUE...\n"); - std.process.exit(0xff); - } - const args = all_args[1..]; - if (args.len < 2) errExit("expected at least 2 cmdline args but got {}", .{args.len}); - - const in_path = args[0]; - const out_path = args[1]; - const replacements = args[2..]; +pub fn replace(b: *std.Build, in_path: []const u8, replacements: []const []const u8) []const u8 { + const arena = b.allocator; + const io = b.graph.io; var map: std.StringHashMapUnmanaged(MapNode) = .{}; for (replacements) |r| { @@ -24,20 +11,13 @@ pub fn main(init: std.process.Init) !void { "expected NAME=VALUE cmdline arg but got '{s}'", .{r}, ); - try map.put(arena, r[0..eq_index], .{ .count = 0, .value = r[eq_index + 1 ..] }); + map.put(arena, r[0..eq_index], .{ .count = 0, .value = r[eq_index + 1 ..] }) catch @panic("OOM"); } const in = std.Io.Dir.cwd().readFileAlloc(io, in_path, arena, .unlimited) catch |e| std.debug.panic("open '{s}' failed with {t}", .{ in_path, e }); - var out_file = try std.Io.Dir.cwd().createFile(io, out_path, .{}); - defer out_file.close(io); - - var out_file_buf: [4096]u8 = undefined; - var file_writer = out_file.writer(io, &out_file_buf); - writeFile(in_path, in, &map, &file_writer.interface) catch |err| switch (err) { - error.WriteFailed => return file_writer.err orelse error.Unexpected, - }; + const returnval = writeFile(in_path, in, &map, b) catch |err| std.debug.panic("replace failed with {t}", .{err}); var unused_count: usize = 0; var it = map.iterator(); @@ -51,6 +31,8 @@ pub fn main(init: std.process.Init) !void { std.log.err("{} unused variable(s), the did not appear in template '{s}'", .{ unused_count, in_path }); std.process.exit(0xff); } + + return returnval; } const MapNode = struct { @@ -62,13 +44,17 @@ fn writeFile( in_path: []const u8, in: []const u8, map: *const std.StringHashMapUnmanaged(MapNode), - writer: *std.Io.Writer, -) error{WriteFailed}!void { + b: *std.Build, +) ![]const u8 { + const alloc = b.allocator; + var list: std.ArrayList(u8) = .empty; + defer list.deinit(b.allocator); + var missing_count: usize = 0; var offset: usize = 0; while (true) { const at_index = std.mem.indexOfScalarPos(u8, in, offset, '@') orelse break; - try writer.writeAll(in[offset..at_index]); + try list.appendSlice(alloc, in[offset..at_index]); const end = std.mem.indexOfScalarPos(u8, in, at_index + 1, '@') orelse errExit( "{s}: unterminated '@' at offset {}", .{ in_path, at_index }, @@ -76,17 +62,16 @@ fn writeFile( const name = in[at_index + 1 .. end]; if (map.getEntry(name)) |entry| { entry.value_ptr.count += 1; - try writer.writeAll(entry.value_ptr.value); + try list.appendSlice(alloc, entry.value_ptr.value); } else { std.log.err("undefined variable '{s}'", .{name}); - try writer.print("MISSING_VAR_{s}", .{name}); + try list.appendSlice(alloc, b.fmt("MISSING_VAR_{s}", .{name})); missing_count += 1; } offset = end + 1; } - try writer.writeAll(in[offset..]); - try writer.flush(); if (missing_count > 0) errExit("{} missing variable(s) for template '{s}'", .{ missing_count, in_path }); + return list.toOwnedSlice(alloc); } fn errExit(comptime fmt: []const u8, args: anytype) noreturn {