Export Cyrillic glyphs and preserve existing font atlases

README.md

@@ -28,10 +28,42 @@ Outputs are written to `Generated Fonts/` beside the executable:
 - **TGA**: 32-bit BGRA image containing packed glyphs with premultiplied alpha. Dimensions are powers of two chosen to fit the selected glyph set.
 - **TGF metadata**:
   - Each font size starts with `HEIGHT <px>;`.
-  - Every glyph line follows `GLYPH <x> <y> <width> <left> <right>` with optional ASCII comment.
+  - ASCII entries use `GLYPH <x> <y> <width> <left> <right>` with optional ASCII comment.
+  - Available Cyrillic glyphs (U+0400 through U+04FF) use `GLYPH_U32 <codepoint> <x> <y> <width> <left> <right>;`, with a decimal codepoint.
   - `x`/`y` are the glyph origin inside the atlas, `width` is the advance width in pixels, and `left`/`right` provide side bearings.
 
 Refer to `fontgen.cpp` if you need deeper details or want to extend the format.
 
+## Extending an existing atlas
+
+Regenerating a font can change the original ASCII metrics or pixels when the
+installed font or rasterizer differs. To preserve an existing atlas, generate
+the extra glyphs at the same heights, then append them with Python and Pillow:
+
+```text
+python -m pip install Pillow
+python extend_font.py "original/Liberation Sans.tgf" "Generated Fonts/Liberation Sans.tgf" "extended/Liberation Sans.tgf"
+```
+
+The output directory receives both the extended TGF and its TGA files. Original
+glyph entries and atlas pixels stay unchanged; Unicode entries refer to the
+appended portion. Use a separate output directory. The engine must understand
+`GLYPH_U32` to display the new glyphs. Font redistribution still requires the
+font's license; the generator's license does not replace it.
+
+## Regression checks
+
+Build and run `tests/fontgen_test.cpp` from an x86 Visual Studio developer shell
+in a scratch working directory. It includes the generator implementation and
+requires `user32.lib`, `gdi32.lib`, and `comctl32.lib` when linking.
+It writes disposable font assets and checks four representative Cyrillic glyphs.
+
+```text
+python -m unittest discover -s tests -p test_extend_font.py
+```
+
+The Python tests verify preservation of original pixels and metadata and the
+coordinates and pixels of appended glyphs.
+
 ## License
 Released under the MIT License (see `LICENSE`).

extend_font.py

@@ -0,0 +1,54 @@
+"""Append generated Unicode glyphs without replacing an existing ASCII atlas."""
+import argparse
+from pathlib import Path
+import re
+
+from PIL import Image
+
+
+def sections(path):
+    result = {}
+    height = None
+    for line in path.read_text(encoding="utf-8").splitlines():
+        match = re.fullmatch(r"HEIGHT (\d+);", line)
+        if match:
+            height = int(match[1])
+            result[height] = []
+        elif height is not None:
+            result[height].append(line)
+    return result
+
+
+def extend(original, extra, output):
+    base = sections(original)
+    additions = sections(extra)
+    if base.keys() != additions.keys():
+        raise ValueError("Font heights must match")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    lines = []
+    for height, glyphs in base.items():
+        with Image.open(original.with_suffix(f".{height}.tga")) as before, Image.open(extra.with_suffix(f".{height}.tga")) as after:
+            width = max(before.width, after.width)
+            size = 1 << (before.height + after.height - 1).bit_length()
+            atlas = Image.new("RGBA", (width, size), (255, 255, 255, 0))
+            atlas.paste(before, (0, 0))
+            atlas.paste(after, (0, before.height))
+            lines.extend([f"HEIGHT {height};", *glyphs])
+            existing = {int(glyph.split()[1]) for glyph in glyphs if glyph.startswith("GLYPH_U32 ")}
+            for glyph in additions[height]:
+                match = re.match(r"GLYPH_U32 (\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(-?\d+)\s+(-?\d+);", glyph)
+                if match:
+                    codepoint, x, y, width, left, right = map(int, match.groups())
+                    if codepoint not in existing:
+                        lines.append(f"GLYPH_U32 {codepoint} {x} {y + before.height} {width} {left} {right};")
+            atlas.save(output.with_suffix(f".{height}.tga"), compression="tga_rle")
+    output.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("original", type=Path)
+    parser.add_argument("extra", type=Path)
+    parser.add_argument("output", type=Path)
+    arguments = parser.parse_args()
+    extend(arguments.original, arguments.extra, arguments.output)

fontgen.cpp

@@ -41,6 +41,7 @@ typedef struct {
 
 // Glyph data
 struct glyphDat_s {
+	unsigned codepoint;
 	unsigned x, y;
 	unsigned width;
 	int spleft;
@@ -86,16 +87,27 @@ void glFontGenV2_c::Build(HDC hdc)
 	auto& current_height = heights.emplace_back();
 	current_height.height = height;
 
-	for (unsigned k = 0; k < 128; k++) {
+	std::vector<unsigned> characters;
+	for (unsigned k = 0; k < 128; k++) characters.push_back(k);
+	for (unsigned k = 0x400; k < 0x500; k++) characters.push_back(k);
+	characters.insert(characters.end(), { 0xabu, 0xbbu, 0x2013u, 0x2014u, 0x2122u });
+	for (unsigned k : characters) {
+		if (k >= 128) {
+			const WCHAR character = static_cast<WCHAR>(k);
+			WORD index = 0xffff;
+			if (GetGlyphIndicesW(hdc, &character, 1, &index, GGI_MARK_NONEXISTING_GLYPHS) == GDI_ERROR || index == 0xffff) {
+				continue;
+			}
+		}
 		// Get glyph
 		GLYPHMETRICS gm;
 		const MAT2 m2i = { 0, 1, 0, 0, 0, 0, 0, 1 };
-		const unsigned bbSize = GetGlyphOutline(hdc, k, GGO_GRAY8_BITMAP, &gm, 0, NULL, &m2i);
-		if (bbSize < 0) {
+		const unsigned bbSize = GetGlyphOutlineW(hdc, k, GGO_GRAY8_BITMAP, &gm, 0, NULL, &m2i);
+		if (bbSize == GDI_ERROR) {
 			return;
 		}
 		std::unique_ptr<uint8_t[]> bb = std::make_unique<uint8_t[]>(bbSize);
-		GetGlyphOutline(hdc, k, GGO_GRAY8_BITMAP, &gm, bbSize, bb.get(), &m2i);
+		GetGlyphOutlineW(hdc, k, GGO_GRAY8_BITMAP, &gm, bbSize, bb.get(), &m2i);
 
 		// Get glyph info
 		const unsigned xs = gm.gmBlackBoxX;
@@ -106,6 +118,7 @@ void glFontGenV2_c::Build(HDC hdc)
 
 		// Initialise glyph data
 		auto& glyph = current_height.glyphs.emplace_back();
+		glyph.codepoint = k;
 		glyph.width = xs;
 		glyph.spleft = xo;
 		glyph.spright = glw - (xo + xs);
@@ -291,9 +304,14 @@ void glFontGenV2_c::Finish()
 		tgf << "HEIGHT " << height_data.height << ";\n";
 		for (size_t g = 0; g < height_data.glyphs.size(); g++) {
 			const auto& glyph = height_data.glyphs[g];
-			tgf << "GLYPH " << std::setw(3) << glyph.x << " " << std::setw(3) << glyph.y << " " << std::setw(2) << glyph.width << " " << std::setw(2) << glyph.spleft << " " << std::setw(2) << glyph.spright << ";\t// " << g;
-			if (isprint(g)) {
-				tgf << " (" << (char)g << ")";
+			if (glyph.codepoint < 128) {
+				tgf << "GLYPH ";
+			} else {
+				tgf << "GLYPH_U32 " << glyph.codepoint << " ";
+			}
+			tgf << std::setw(3) << glyph.x << " " << std::setw(3) << glyph.y << " " << std::setw(2) << glyph.width << " " << std::setw(2) << glyph.spleft << " " << std::setw(2) << glyph.spright << ";\t// " << glyph.codepoint;
+			if (glyph.codepoint < 128 && isprint(glyph.codepoint)) {
+				tgf << " (" << (char)glyph.codepoint << ")";
 			}
 			tgf << "\n";
 		}

tests/fontgen_test.cpp

@@ -0,0 +1,31 @@
+// Regression coverage for Cyrillic Notes spec clause 3.
+// Run in a temporary working directory; the generator writes Generated Fonts there.
+#include "../fontgen.cpp"
+#include <iostream>
+
+int main()
+{
+	HDC dc = CreateCompatibleDC(nullptr);
+	HFONT font = CreateFontW(20, 0, 0, 0, FW_NORMAL, FALSE, FALSE, FALSE,
+		DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, CLIP_DEFAULT_PRECIS, DEFAULT_QUALITY,
+		VARIABLE_PITCH, L"Arial");
+	auto previous = SelectObject(dc, font);
+	glFontGenV2_c generator;
+	generator.Init("CyrillicTest");
+	generator.Build(dc);
+	generator.Finish();
+	SelectObject(dc, previous);
+	DeleteObject(font);
+	DeleteDC(dc);
+	std::ifstream stream("Generated Fonts/CyrillicTest.tgf");
+	std::string metadata((std::istreambuf_iterator<char>(stream)), {});
+	int failures = 0;
+	// Notes repair clause 3: punctuation used by the translated notes.
+	for (unsigned codepoint : { 0x401u, 0x410u, 0x44fu, 0x451u, 0x2013u, 0x2014u, 0xabu, 0xbbu, 0x2122u }) {
+		bool present = metadata.find("GLYPH_U32 " + std::to_string(codepoint) + " ") != std::string::npos;
+		std::cout << (present ? "PASS" : "FAIL") << ": exports Unicode codepoint " << codepoint << "\n";
+		failures += !present;
+	}
+	std::cout << "9 tests, " << failures << " failures\n";
+	return failures != 0;
+}

tests/test_extend_font.py

@@ -0,0 +1,55 @@
+"""Cyrillic Notes clause 3: preserve the existing ASCII layout and pixels."""
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from PIL import Image
+
+
+def generate(directory):
+    root = Path(directory)
+    (root / "original.tgf").write_text("HEIGHT 2;\nGLYPH 1 1 2 -1 2;\t// fixture\nGLYPH_U32 1105 0 0 1 0 0;\n")
+    (root / "extra.tgf").write_text("HEIGHT 2;\nGLYPH 0 0 3 0 0;\nGLYPH_U32 1040 1 1 2 0 1;\nGLYPH_U32 1105 2 2 1 1 1;\n")
+    Image.new("RGBA", (4, 4), (97, 29, 11, 191)).save(root / "original.2.tga")
+    Image.new("RGBA", (4, 4), (13, 41, 83, 157)).save(root / "extra.2.tga")
+    subprocess.run([sys.executable, str(Path(__file__).resolve().parents[1] / "extend_font.py"),
+                    str(root / "original.tgf"), str(root / "extra.tgf"), str(root / "output.tgf")], check=True, timeout=30)
+    return root
+
+
+class ExtensionTests(unittest.TestCase):
+    """Check generated assets through the command-line interface."""
+
+    def test_preserves_original_glyph_metrics_and_coordinates(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = generate(directory)
+            self.assertTrue((root / "output.tgf").read_text().startswith((root / "original.tgf").read_text()))
+
+    def test_preserves_original_atlas_pixels(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = generate(directory)
+            with Image.open(root / "output.2.tga") as atlas, Image.open(root / "original.2.tga") as original:
+                self.assertEqual(original.tobytes(), atlas.crop((0, 0, 4, 4)).tobytes())
+
+    def test_offsets_new_glyph_coordinates_below_original_atlas(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = generate(directory)
+            self.assertIn("GLYPH_U32 1040 1 5 2 0 1;", (root / "output.tgf").read_text())
+
+    def test_preserves_new_glyph_pixels(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = generate(directory)
+            with Image.open(root / "output.2.tga") as atlas, Image.open(root / "extra.2.tga") as extra:
+                self.assertEqual(extra.crop((1, 1, 3, 3)).tobytes(), atlas.crop((1, 5, 3, 7)).tobytes())
+
+    # Notes repair clause 3: extending again must preserve existing Unicode too.
+    def test_does_not_override_existing_unicode_glyphs(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = generate(directory)
+            self.assertEqual(1, (root / "output.tgf").read_text().count("GLYPH_U32 1105 "))
+
+
+if __name__ == "__main__":
+    unittest.main()