fix: avoid stack overflow on table separator lines with many columns

flexmark-ext-tables/src/main/java/com/vladsch/flexmark/ext/tables/internal/TableParagraphPreProcessor.java

@@ -121,7 +121,9 @@ public static Pattern getTableHeaderSeparator(int minColumnDashes, String intell
         int minColDash = minColumnDashes >= 2 ? minColumnDashes - 1 : 1;
         int minColDashes = minColumnDashes >= 3 ? minColumnDashes - 2 : 1;
         // to prevent conversion to arabic numbers, using string
-        String COL = String.format(Locale.US, "(?:" + "\\s*-{%d,}\\s*|\\s*:-{%d,}\\s*|\\s*-{%d,}:\\s*|\\s*:-{%d,}:\\s*" + ")", minCol, minColDash, minColDash, minColDashes);
+        // The column repetition below is possessive, so a column never backtracks into another alternative:
+        // the alternatives must stay ordered longest first (":-:" before ":-", "-:" before "-").
+        String COL = String.format(Locale.US, "(?:" + "\\s*+:-{%d,}+:\\s*+|\\s*+:-{%d,}+\\s*+|\\s*+-{%d,}+:\\s*+|\\s*+-{%d,}+\\s*+" + ")", minColDashes, minColDash, minColDash, minCol);
 
         boolean noIntelliJ = intellijDummyIdentifier.isEmpty();
         String add = noIntelliJ ? "" : TableFormatOptions.INTELLIJ_DUMMY_IDENTIFIER;
@@ -132,7 +134,10 @@ public static Pattern getTableHeaderSeparator(int minColumnDashes, String intell
 
         String regex = "\\|" + COL + "\\|?\\s*" + "|" +
                 COL + "\\|\\s*" + "|" +
-                "\\|?" + "(?:" + COL + "\\|)+" + COL + "\\|?\\s*";
+                // Not "(?:COL\\|)+COL": java.util.regex matches a greedy group repetition recursively, one set of stack
+                // frames per column, and a separator line with several hundred columns overflowed the stack.
+                // A possessive repetition is matched in a loop.
+                "\\|?" + COL + "(?:\\|" + COL + ")++" + "\\|?\\s*";
 
         String withIntelliJ = regex.replace("\\s", sp).replace("\\|", pipe).replace("-", ds);
 

flexmark-ext-tables/src/test/java/com/vladsch/flexmark/ext/tables/TableSeparatorStackOverflowTest.java

@@ -0,0 +1,99 @@
+package com.vladsch.flexmark.ext.tables;
+
+import com.vladsch.flexmark.html.HtmlRenderer;
+import com.vladsch.flexmark.parser.Parser;
+import com.vladsch.flexmark.util.data.DataHolder;
+import com.vladsch.flexmark.util.data.MutableDataSet;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class TableSeparatorStackOverflowTest {
+    // The default thread stack size differs per platform, so the tests pin one to stay deterministic.
+    private static final long STACK_SIZE_BYTES = 512 * 1024;
+    private static final int COLUMNS = 5000;
+
+    private static String renderOnSmallStack(String markdown, int minSeparatorDashes) throws InterruptedException {
+        DataHolder options = new MutableDataSet()
+                .set(TablesExtension.MIN_SEPARATOR_DASHES, minSeparatorDashes)
+                .set(Parser.EXTENSIONS, Arrays.asList(TablesExtension.create()))
+                .toImmutable();
+        Parser parser = Parser.builder(options).build();
+        HtmlRenderer renderer = HtmlRenderer.builder(options).build();
+
+        AtomicReference<String> html = new AtomicReference<>();
+        AtomicReference<Throwable> failure = new AtomicReference<>();
+        Thread thread = new Thread(null, () -> {
+            try {
+                html.set(renderer.render(parser.parse(markdown)));
+            } catch (Throwable t) {
+                failure.set(t);
+            }
+        }, "table-separator-stack-overflow-test", STACK_SIZE_BYTES);
+        thread.start();
+        thread.join();
+
+        assertNull("rendering must not fail: " + failure.get(), failure.get());
+        return html.get();
+    }
+
+    private static String wideTable(String separatorCell) {
+        StringBuilder header = new StringBuilder("|");
+        StringBuilder separator = new StringBuilder("|");
+        StringBuilder body = new StringBuilder("|");
+        for (int i = 0; i < COLUMNS; i++) {
+            header.append("h|");
+            separator.append(separatorCell).append('|');
+            body.append("v|");
+        }
+        return header + "\n" + separator + "\n" + body + "\n";
+    }
+
+    private static int count(String text, String needle) {
+        int count = 0;
+        for (int i = text.indexOf(needle); i >= 0; i = text.indexOf(needle, i + needle.length())) count++;
+        return count;
+    }
+
+    @Test
+    public void wideTable_plainSeparator() throws InterruptedException {
+        String html = renderOnSmallStack(wideTable("---"), 3);
+        assertEquals(COLUMNS, count(html, "<th>"));
+        assertEquals(COLUMNS, count(html, "<td>"));
+    }
+
+    @Test
+    public void wideTable_leftAlignedSeparator() throws InterruptedException {
+        String html = renderOnSmallStack(wideTable(":-"), 1);
+        assertEquals(COLUMNS, count(html, "<th align=\"left\">"));
+    }
+
+    @Test
+    public void wideTable_rightAlignedSeparator() throws InterruptedException {
+        String html = renderOnSmallStack(wideTable("-:"), 1);
+        assertEquals(COLUMNS, count(html, "<th align=\"right\">"));
+    }
+
+    @Test
+    public void wideTable_centerAlignedSeparator() throws InterruptedException {
+        String html = renderOnSmallStack(wideTable(" :-: "), 1);
+        assertEquals(COLUMNS, count(html, "<th align=\"center\">"));
+    }
+
+    @Test
+    public void mixedAlignment_isKept() throws InterruptedException {
+        String html = renderOnSmallStack("" +
+                "a|b|c|d\n" +
+                ":--|:-:|--:|---\n" +
+                "1|2|3|4\n", 3);
+        assertTrue(html, html.contains("<th align=\"left\">a</th>"));
+        assertTrue(html, html.contains("<th align=\"center\">b</th>"));
+        assertTrue(html, html.contains("<th align=\"right\">c</th>"));
+        assertTrue(html, html.contains("<th>d</th>"));
+    }
+}