From fc6b925f5cdc0111cda14987040a2b8401b28656 Mon Sep 17 00:00:00 2001 From: ufik Date: Fri, 13 Nov 2026 21:34:46 +0300 Subject: [PATCH 1/2] cyrillic-notes: export Cyrillic glyphs and extend existing atlases --- README.md | 34 ++++++++++++++++++++++++- extend_font.py | 52 +++++++++++++++++++++++++++++++++++++++ fontgen.cpp | 31 +++++++++++++++++------ tests/fontgen_test.cpp | 30 ++++++++++++++++++++++ tests/test_extend_font.py | 49 ++++++++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 extend_font.py create mode 100644 tests/fontgen_test.cpp create mode 100644 tests/test_extend_font.py diff --git a/README.md b/README.md index f748c90..7285225 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,42 @@ Outputs are written to `Generated Fonts/` beside the executable: - **TGA**: 32-bit BGRA image containing packed glyphs with premultiplied alpha. Dimensions are powers of two chosen to fit the selected glyph set. - **TGF metadata**: - Each font size starts with `HEIGHT ;`. - - Every glyph line follows `GLYPH ` with optional ASCII comment. + - ASCII entries use `GLYPH ` with optional ASCII comment. + - Available Cyrillic glyphs (U+0400 through U+04FF) use `GLYPH_U32 ;`, with a decimal codepoint. - `x`/`y` are the glyph origin inside the atlas, `width` is the advance width in pixels, and `left`/`right` provide side bearings. Refer to `fontgen.cpp` if you need deeper details or want to extend the format. +## Extending an existing atlas + +Regenerating a font can change the original ASCII metrics or pixels when the +installed font or rasterizer differs. To preserve an existing atlas, generate +the extra glyphs at the same heights, then append them with Python and Pillow: + +```text +python -m pip install Pillow +python extend_font.py "original/Liberation Sans.tgf" "Generated Fonts/Liberation Sans.tgf" "extended/Liberation Sans.tgf" +``` + +The output directory receives both the extended TGF and its TGA files. Original +glyph entries and atlas pixels stay unchanged; Unicode entries refer to the +appended portion. Use a separate output directory. The engine must understand +`GLYPH_U32` to display the new glyphs. Font redistribution still requires the +font's license; the generator's license does not replace it. + +## Regression checks + +Build and run `tests/fontgen_test.cpp` from an x86 Visual Studio developer shell +in a scratch working directory. It includes the generator implementation and +requires `user32.lib`, `gdi32.lib`, and `comctl32.lib` when linking. +It writes disposable font assets and checks four representative Cyrillic glyphs. + +```text +python -m unittest discover -s tests -p test_extend_font.py +``` + +The Python tests verify preservation of original pixels and metadata and the +coordinates and pixels of appended glyphs. + ## License Released under the MIT License (see `LICENSE`). diff --git a/extend_font.py b/extend_font.py new file mode 100644 index 0000000..ed5385b --- /dev/null +++ b/extend_font.py @@ -0,0 +1,52 @@ +"""Append generated Unicode glyphs without replacing an existing ASCII atlas.""" +import argparse +from pathlib import Path +import re + +from PIL import Image + + +def sections(path): + result = {} + height = None + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"HEIGHT (\d+);", line) + if match: + height = int(match[1]) + result[height] = [] + elif height is not None: + result[height].append(line) + return result + + +def extend(original, extra, output): + base = sections(original) + additions = sections(extra) + if base.keys() != additions.keys(): + raise ValueError("Font heights must match") + output.parent.mkdir(parents=True, exist_ok=True) + lines = [] + for height, glyphs in base.items(): + with Image.open(original.with_suffix(f".{height}.tga")) as before, Image.open(extra.with_suffix(f".{height}.tga")) as after: + width = max(before.width, after.width) + size = 1 << (before.height + after.height - 1).bit_length() + atlas = Image.new("RGBA", (width, size), (255, 255, 255, 0)) + atlas.paste(before, (0, 0)) + atlas.paste(after, (0, before.height)) + lines.extend([f"HEIGHT {height};", *glyphs]) + for glyph in additions[height]: + match = re.match(r"GLYPH_U32 (\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(-?\d+)\s+(-?\d+);", glyph) + if match: + codepoint, x, y, width, left, right = map(int, match.groups()) + lines.append(f"GLYPH_U32 {codepoint} {x} {y + before.height} {width} {left} {right};") + atlas.save(output.with_suffix(f".{height}.tga"), compression="tga_rle") + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("original", type=Path) + parser.add_argument("extra", type=Path) + parser.add_argument("output", type=Path) + arguments = parser.parse_args() + extend(arguments.original, arguments.extra, arguments.output) diff --git a/fontgen.cpp b/fontgen.cpp index e3ce496..0cea9f0 100644 --- a/fontgen.cpp +++ b/fontgen.cpp @@ -41,6 +41,7 @@ typedef struct { // Glyph data struct glyphDat_s { + unsigned codepoint; unsigned x, y; unsigned width; int spleft; @@ -86,16 +87,26 @@ void glFontGenV2_c::Build(HDC hdc) auto& current_height = heights.emplace_back(); current_height.height = height; - for (unsigned k = 0; k < 128; k++) { + for (unsigned k = 0; k < 0x500; k++) { + if (k >= 128 && k < 0x400) { + continue; + } + if (k >= 128) { + const WCHAR character = static_cast(k); + WORD index = 0xffff; + if (GetGlyphIndicesW(hdc, &character, 1, &index, GGI_MARK_NONEXISTING_GLYPHS) == GDI_ERROR || index == 0xffff) { + continue; + } + } // Get glyph GLYPHMETRICS gm; const MAT2 m2i = { 0, 1, 0, 0, 0, 0, 0, 1 }; - const unsigned bbSize = GetGlyphOutline(hdc, k, GGO_GRAY8_BITMAP, &gm, 0, NULL, &m2i); - if (bbSize < 0) { + const unsigned bbSize = GetGlyphOutlineW(hdc, k, GGO_GRAY8_BITMAP, &gm, 0, NULL, &m2i); + if (bbSize == GDI_ERROR) { return; } std::unique_ptr bb = std::make_unique(bbSize); - GetGlyphOutline(hdc, k, GGO_GRAY8_BITMAP, &gm, bbSize, bb.get(), &m2i); + GetGlyphOutlineW(hdc, k, GGO_GRAY8_BITMAP, &gm, bbSize, bb.get(), &m2i); // Get glyph info const unsigned xs = gm.gmBlackBoxX; @@ -106,6 +117,7 @@ void glFontGenV2_c::Build(HDC hdc) // Initialise glyph data auto& glyph = current_height.glyphs.emplace_back(); + glyph.codepoint = k; glyph.width = xs; glyph.spleft = xo; glyph.spright = glw - (xo + xs); @@ -291,9 +303,14 @@ void glFontGenV2_c::Finish() tgf << "HEIGHT " << height_data.height << ";\n"; for (size_t g = 0; g < height_data.glyphs.size(); g++) { const auto& glyph = height_data.glyphs[g]; - tgf << "GLYPH " << std::setw(3) << glyph.x << " " << std::setw(3) << glyph.y << " " << std::setw(2) << glyph.width << " " << std::setw(2) << glyph.spleft << " " << std::setw(2) << glyph.spright << ";\t// " << g; - if (isprint(g)) { - tgf << " (" << (char)g << ")"; + if (glyph.codepoint < 128) { + tgf << "GLYPH "; + } else { + tgf << "GLYPH_U32 " << glyph.codepoint << " "; + } + tgf << std::setw(3) << glyph.x << " " << std::setw(3) << glyph.y << " " << std::setw(2) << glyph.width << " " << std::setw(2) << glyph.spleft << " " << std::setw(2) << glyph.spright << ";\t// " << glyph.codepoint; + if (glyph.codepoint < 128 && isprint(glyph.codepoint)) { + tgf << " (" << (char)glyph.codepoint << ")"; } tgf << "\n"; } diff --git a/tests/fontgen_test.cpp b/tests/fontgen_test.cpp new file mode 100644 index 0000000..9f36d0b --- /dev/null +++ b/tests/fontgen_test.cpp @@ -0,0 +1,30 @@ +// Regression coverage for Cyrillic Notes spec clause 3. +// Run in a temporary working directory; the generator writes Generated Fonts there. +#include "../fontgen.cpp" +#include + +int main() +{ + HDC dc = CreateCompatibleDC(nullptr); + HFONT font = CreateFontW(20, 0, 0, 0, FW_NORMAL, FALSE, FALSE, FALSE, + DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, CLIP_DEFAULT_PRECIS, DEFAULT_QUALITY, + VARIABLE_PITCH, L"Arial"); + auto previous = SelectObject(dc, font); + glFontGenV2_c generator; + generator.Init("CyrillicTest"); + generator.Build(dc); + generator.Finish(); + SelectObject(dc, previous); + DeleteObject(font); + DeleteDC(dc); + std::ifstream stream("Generated Fonts/CyrillicTest.tgf"); + std::string metadata((std::istreambuf_iterator(stream)), {}); + int failures = 0; + for (unsigned codepoint : { 0x401u, 0x410u, 0x44fu, 0x451u }) { + bool present = metadata.find("GLYPH_U32 " + std::to_string(codepoint) + " ") != std::string::npos; + std::cout << (present ? "PASS" : "FAIL") << ": exports Cyrillic codepoint " << codepoint << "\n"; + failures += !present; + } + std::cout << "4 tests, " << failures << " failures\n"; + return failures != 0; +} diff --git a/tests/test_extend_font.py b/tests/test_extend_font.py new file mode 100644 index 0000000..e2febef --- /dev/null +++ b/tests/test_extend_font.py @@ -0,0 +1,49 @@ +"""Cyrillic Notes clause 3: preserve the existing ASCII layout and pixels.""" +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from PIL import Image + + +def generate(directory): + root = Path(directory) + (root / "original.tgf").write_text("HEIGHT 2;\nGLYPH 1 1 2 -1 2;\t// fixture\n") + (root / "extra.tgf").write_text("HEIGHT 2;\nGLYPH 0 0 3 0 0;\nGLYPH_U32 1040 1 1 2 0 1;\n") + Image.new("RGBA", (4, 4), (97, 29, 11, 191)).save(root / "original.2.tga") + Image.new("RGBA", (4, 4), (13, 41, 83, 157)).save(root / "extra.2.tga") + subprocess.run([sys.executable, str(Path(__file__).resolve().parents[1] / "extend_font.py"), + str(root / "original.tgf"), str(root / "extra.tgf"), str(root / "output.tgf")], check=True, timeout=30) + return root + + +class ExtensionTests(unittest.TestCase): + """Check generated assets through the command-line interface.""" + + def test_preserves_original_glyph_metrics_and_coordinates(self): + with tempfile.TemporaryDirectory() as directory: + root = generate(directory) + self.assertTrue((root / "output.tgf").read_text().startswith((root / "original.tgf").read_text())) + + def test_preserves_original_atlas_pixels(self): + with tempfile.TemporaryDirectory() as directory: + root = generate(directory) + with Image.open(root / "output.2.tga") as atlas, Image.open(root / "original.2.tga") as original: + self.assertEqual(original.tobytes(), atlas.crop((0, 0, 4, 4)).tobytes()) + + def test_offsets_new_glyph_coordinates_below_original_atlas(self): + with tempfile.TemporaryDirectory() as directory: + root = generate(directory) + self.assertIn("GLYPH_U32 1040 1 5 2 0 1;", (root / "output.tgf").read_text()) + + def test_preserves_new_glyph_pixels(self): + with tempfile.TemporaryDirectory() as directory: + root = generate(directory) + with Image.open(root / "output.2.tga") as atlas, Image.open(root / "extra.2.tga") as extra: + self.assertEqual(extra.crop((1, 1, 3, 3)).tobytes(), atlas.crop((1, 5, 3, 7)).tobytes()) + + +if __name__ == "__main__": + unittest.main() From 798f9d2ba9251cce7f6ba92316fd40a541b4247b Mon Sep 17 00:00:00 2001 From: ufik Date: Fri, 13 Nov 2026 22:11:07 +0300 Subject: [PATCH 2/2] cyrillic-notes: extend punctuation without replacing existing glyphs --- extend_font.py | 4 +++- fontgen.cpp | 9 +++++---- tests/fontgen_test.cpp | 7 ++++--- tests/test_extend_font.py | 10 ++++++++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/extend_font.py b/extend_font.py index ed5385b..4e758bb 100644 --- a/extend_font.py +++ b/extend_font.py @@ -34,11 +34,13 @@ def extend(original, extra, output): atlas.paste(before, (0, 0)) atlas.paste(after, (0, before.height)) lines.extend([f"HEIGHT {height};", *glyphs]) + existing = {int(glyph.split()[1]) for glyph in glyphs if glyph.startswith("GLYPH_U32 ")} for glyph in additions[height]: match = re.match(r"GLYPH_U32 (\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(-?\d+)\s+(-?\d+);", glyph) if match: codepoint, x, y, width, left, right = map(int, match.groups()) - lines.append(f"GLYPH_U32 {codepoint} {x} {y + before.height} {width} {left} {right};") + if codepoint not in existing: + lines.append(f"GLYPH_U32 {codepoint} {x} {y + before.height} {width} {left} {right};") atlas.save(output.with_suffix(f".{height}.tga"), compression="tga_rle") output.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/fontgen.cpp b/fontgen.cpp index 0cea9f0..8a1c68e 100644 --- a/fontgen.cpp +++ b/fontgen.cpp @@ -87,10 +87,11 @@ void glFontGenV2_c::Build(HDC hdc) auto& current_height = heights.emplace_back(); current_height.height = height; - for (unsigned k = 0; k < 0x500; k++) { - if (k >= 128 && k < 0x400) { - continue; - } + std::vector characters; + for (unsigned k = 0; k < 128; k++) characters.push_back(k); + for (unsigned k = 0x400; k < 0x500; k++) characters.push_back(k); + characters.insert(characters.end(), { 0xabu, 0xbbu, 0x2013u, 0x2014u, 0x2122u }); + for (unsigned k : characters) { if (k >= 128) { const WCHAR character = static_cast(k); WORD index = 0xffff; diff --git a/tests/fontgen_test.cpp b/tests/fontgen_test.cpp index 9f36d0b..4dcfaf7 100644 --- a/tests/fontgen_test.cpp +++ b/tests/fontgen_test.cpp @@ -20,11 +20,12 @@ int main() std::ifstream stream("Generated Fonts/CyrillicTest.tgf"); std::string metadata((std::istreambuf_iterator(stream)), {}); int failures = 0; - for (unsigned codepoint : { 0x401u, 0x410u, 0x44fu, 0x451u }) { + // Notes repair clause 3: punctuation used by the translated notes. + for (unsigned codepoint : { 0x401u, 0x410u, 0x44fu, 0x451u, 0x2013u, 0x2014u, 0xabu, 0xbbu, 0x2122u }) { bool present = metadata.find("GLYPH_U32 " + std::to_string(codepoint) + " ") != std::string::npos; - std::cout << (present ? "PASS" : "FAIL") << ": exports Cyrillic codepoint " << codepoint << "\n"; + std::cout << (present ? "PASS" : "FAIL") << ": exports Unicode codepoint " << codepoint << "\n"; failures += !present; } - std::cout << "4 tests, " << failures << " failures\n"; + std::cout << "9 tests, " << failures << " failures\n"; return failures != 0; } diff --git a/tests/test_extend_font.py b/tests/test_extend_font.py index e2febef..ba65c11 100644 --- a/tests/test_extend_font.py +++ b/tests/test_extend_font.py @@ -10,8 +10,8 @@ def generate(directory): root = Path(directory) - (root / "original.tgf").write_text("HEIGHT 2;\nGLYPH 1 1 2 -1 2;\t// fixture\n") - (root / "extra.tgf").write_text("HEIGHT 2;\nGLYPH 0 0 3 0 0;\nGLYPH_U32 1040 1 1 2 0 1;\n") + (root / "original.tgf").write_text("HEIGHT 2;\nGLYPH 1 1 2 -1 2;\t// fixture\nGLYPH_U32 1105 0 0 1 0 0;\n") + (root / "extra.tgf").write_text("HEIGHT 2;\nGLYPH 0 0 3 0 0;\nGLYPH_U32 1040 1 1 2 0 1;\nGLYPH_U32 1105 2 2 1 1 1;\n") Image.new("RGBA", (4, 4), (97, 29, 11, 191)).save(root / "original.2.tga") Image.new("RGBA", (4, 4), (13, 41, 83, 157)).save(root / "extra.2.tga") subprocess.run([sys.executable, str(Path(__file__).resolve().parents[1] / "extend_font.py"), @@ -44,6 +44,12 @@ def test_preserves_new_glyph_pixels(self): with Image.open(root / "output.2.tga") as atlas, Image.open(root / "extra.2.tga") as extra: self.assertEqual(extra.crop((1, 1, 3, 3)).tobytes(), atlas.crop((1, 5, 3, 7)).tobytes()) + # Notes repair clause 3: extending again must preserve existing Unicode too. + def test_does_not_override_existing_unicode_glyphs(self): + with tempfile.TemporaryDirectory() as directory: + root = generate(directory) + self.assertEqual(1, (root / "output.tgf").read_text().count("GLYPH_U32 1105 ")) + if __name__ == "__main__": unittest.main()