inspect: read only the end of each segment over HTTP

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `Chunk.prefetch` and `VortexHandle.prefetch` start the range requests of several columns together over HTTP, merged into few requests, and an HTTP handle keeps the segments it has read. ([#744](https://github.com/dfa1/vortex-java/issues/744))
 
 ### Fixed
+- An unexpected HTTP status now reports the server's own explanation and `Retry-After`, so a Hugging Face rate limit (`429`) says what to do instead of just the code. ([#743](https://github.com/dfa1/vortex-java/issues/743))
+- `vortex inspect` over HTTP fetches only the end of each segment, several at a time, instead of the whole file, and shows a progress bar. ([#743](https://github.com/dfa1/vortex-java/issues/743))
 - `vortex view` no longer freezes while it fetches data over HTTP: it shows a loading spinner, `q` quits at once, and it fetches only the columns on screen. ([#744](https://github.com/dfa1/vortex-java/issues/744))
 - A decimal min or max in a flat node's stats is decoded as a `BigDecimal` instead of text, so a filtered scan can prune on it and `columnStats()` reports it. ([#710](https://github.com/dfa1/vortex-java/issues/710))
 - A `vortex.uuid` column's file statistics carry its min and max, as Rust writes them from 0.92, and `WriteSummary` reports them as `UUID`s. ([#730](https://github.com/dfa1/vortex-java/pull/730))

cli/src/main/java/io/github/dfa1/vortex/cli/InspectCommand.java

@@ -1,5 +1,6 @@
 package io.github.dfa1.vortex.cli;
 
+import io.github.dfa1.vortex.core.error.VortexException;
 import io.github.dfa1.vortex.inspect.HtmlReport;
 import io.github.dfa1.vortex.inspect.InspectorTree;
 import io.github.dfa1.vortex.inspect.VortexInspector;
@@ -24,13 +25,19 @@ static int run(String[] args) {
             if (handle == null) {
                 return ExitStatus.FILE_NOT_FOUND;
             }
+            InspectorTree tree = InspectorTree.build(handle, (done, total) -> ProgressBar.render(done, total, "segments"));
+            ProgressBar.clear();
             System.out.print(html
-                    ? HtmlReport.render(InspectorTree.build(handle), lastSegment(target))
-                    : VortexInspector.inspect(handle));
+                    ? HtmlReport.render(tree, lastSegment(target))
+                    : VortexInspector.render(tree));
             return ExitStatus.OK;
         } catch (IOException | RuntimeException e) {
             System.err.println("error: " + CliHandles.describe(e));
-            e.printStackTrace(System.err);
+            // a VortexException is an expected failure (bad file, HTTP error) already fully described
+            // above; only an unexpected error needs its stack
+            if (!(e instanceof VortexException)) {
+                e.printStackTrace(System.err);
+            }
             return ExitStatus.ERROR;
         }
     }

cli/src/main/java/io/github/dfa1/vortex/cli/ProgressBar.java

@@ -17,10 +17,19 @@ private ProgressBar() {
     /// @param done  rows processed so far
     /// @param total total rows expected
     static void render(long done, long total) {
+        render(done, total, "rows");
+    }
+
+    /// Renders the same bar counting `unit` instead of rows.
+    ///
+    /// @param done  units processed so far
+    /// @param total units expected
+    /// @param unit  what is counted, e.g. `segments`
+    static void render(long done, long total, String unit) {
         int pct = total > 0 ? (int) (done * 100L / total) : 100;
         int filled = pct * WIDTH / 100;
         String bar = "=".repeat(filled) + (filled < WIDTH ? ">" : "") + " ".repeat(Math.max(0, WIDTH - 1 - filled));
-        System.err.printf("\r  [%s] %3d%%  %,d / %,d rows", bar, pct, done, total);
+        System.err.printf("\r  [%s] %3d%%  %,d / %,d %s", bar, pct, done, total, unit);
         System.err.flush();
     }
 

docs/reference.md

@@ -85,7 +85,9 @@ Interface common to both handle implementations below — everything scan-shaped
 that only reads (e.g. `ParquetExporter.exportParquet(VortexHandle, Path)`) doesn't care whether the
 source is a local file or a remote URL. Implements `Closeable`. `prefetch(Collection<SegmentSpec>)` is
 a read-ahead hint for the segments about to be read: a no-op on `VortexReader`, concurrent merged range
-requests on `VortexHttpReader`. Call it from the thread that opened the handle.
+requests on `VortexHttpReader`. Call it from the thread that opened the handle. `segmentTail(SegmentSpec)`
+reads only the end of a segment (its array node, which is all the inspector needs); over HTTP that is one
+small range request, with `prefetchTails(Collection<SegmentSpec>)` as its read-ahead hint.
 
 | Implementation    | Backing storage                              |
 |--------------------|-----------------------------------------------|

inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java

@@ -149,7 +149,7 @@ private static Node shallowNode(Layout layout, Optional<String> fieldName) {
     /// compression, or missing data.
     ///
     /// Callers should cache the result — every call triggers a fresh
-    /// [io.github.dfa1.vortex.reader.VortexHandle#rawSegment], which is a network round-trip on remote handles.
+    /// [io.github.dfa1.vortex.reader.VortexHandle#segmentTail], which is a network round-trip on remote handles.
     ///
     /// @param node   node to resolve
     /// @param handle open file handle
@@ -164,7 +164,7 @@ public static Peek peek(Node node, VortexHandle handle) {
         if (spec.compression().code != 0) {
             return Peek.EMPTY;
         }
-        MemorySegment seg = handle.rawSegment(spec);
+        MemorySegment seg = handle.segmentTail(spec);
         return peekFlatRoot(seg, handle.footer().arraySpecs());
     }
 
@@ -180,7 +180,13 @@ public static InspectorTree build(VortexHandle handle, Progress progress) {
         Layout layout = handle.layout();
         DType dtype = handle.dtype();
 
-        int total = countPeekableSegments(layout, footer);
+        List<SegmentSpec> peekable = new ArrayList<>();
+        collectPeekableSegments(layout, footer, peekable);
+        int total = peekable.size();
+        handle.prefetchTails(peekable);
+        List<SegmentSpec> zoneTables = new ArrayList<>();
+        collectZoneTableSegments(layout, footer, zoneTables);
+        handle.prefetch(zoneTables);
         int[] counter = {0};
 
         List<String> colNames = rootChildLabels(dtype);
@@ -332,7 +338,7 @@ private static Node buildNode(Layout layout, Optional<String> fieldName, VortexH
             int segIdx = layout.segments().getFirst();
             SegmentSpec spec = handle.footer().segmentSpecs().get(segIdx);
             if (spec.compression().code == 0) {
-                MemorySegment seg = handle.rawSegment(spec);
+                MemorySegment seg = handle.segmentTail(spec);
                 Peek peek = peekFlatRoot(seg, arraySpecs);
                 if (peek.encoding() != null) {
                     localUsed.addAll(peek.nestedEncodings());
@@ -354,18 +360,28 @@ private static Node buildNode(Layout layout, Optional<String> fieldName, VortexH
         return new Node(layout, fieldName, Set.copyOf(localUsed), stats, List.copyOf(children));
     }
 
-    private static int countPeekableSegments(Layout layout, Footer footer) {
-        int n = 0;
+    /// The segments of every zone-map table, which `columnZones` reads one by one after the peeks.
+    private static void collectZoneTableSegments(Layout layout, Footer footer, List<SegmentSpec> out) {
+        if (layout.isZoned() && layout.children().size() >= 2) {
+            for (int segment : layout.children().get(1).segments()) {
+                out.add(footer.segmentSpecs().get(segment));
+            }
+        }
+        for (Layout child : layout.children()) {
+            collectZoneTableSegments(child, footer, out);
+        }
+    }
+
+    private static void collectPeekableSegments(Layout layout, Footer footer, List<SegmentSpec> out) {
         if (layout.isFlat() && !layout.segments().isEmpty()) {
             SegmentSpec spec = footer.segmentSpecs().get(layout.segments().getFirst());
             if (spec.compression().code == 0) {
-                n++;
+                out.add(spec);
             }
         }
         for (Layout child : layout.children()) {
-            n += countPeekableSegments(child, footer);
+            collectPeekableSegments(child, footer, out);
         }
-        return n;
     }
 
     /// Callback used by [#build(VortexHandle, Progress)] to report how many

inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java

@@ -211,8 +211,8 @@ void buildShallow_skipsAllSlicesAndStillNamesColumns() {
         assertThat(sut.usedEncodings()).isEmpty();
         assertThat(sut.root().children().get(0).usedEncodings()).isEmpty();
         assertThat(sut.root().children().getFirst().stats()).isEqualTo(io.github.dfa1.vortex.core.model.ArrayStats.empty());
-        // rawSegment is reserved for lazy peek; shallow build must never call it
-        org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).rawSegment(
+        // segmentTail is reserved for lazy peek; shallow build must never call it
+        org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).segmentTail(
                 org.mockito.Mockito.any());
     }
 
@@ -230,7 +230,7 @@ void peek_nonFlatNode_returnsEmptyWithoutSlicing() {
 
         // Then
         assertThat(result).isSameAs(InspectorTree.Peek.EMPTY);
-        org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).rawSegment(
+        org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).segmentTail(
                 org.mockito.Mockito.any());
     }
 
@@ -251,7 +251,7 @@ void peek_compressedFlatSegment_returnsEmptyWithoutSlicing() {
 
         // Then
         assertThat(result).isSameAs(InspectorTree.Peek.EMPTY);
-        org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).rawSegment(
+        org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).segmentTail(
                 org.mockito.Mockito.any());
     }
 
@@ -287,7 +287,7 @@ void build_flatSegmentWithNestedEncoding_reportsRootAndNestedEncodings() {
             SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE);
             givenHandle(dtype, root, arraySpecs, List.of(spec));
             byte[] fb = nestedArrayFlatBuffer(0, 1);
-            given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena));
+            given(handle.segmentTail(spec)).willReturn(wrapAsSegment(fb, arena));
 
             // When
             InspectorTree sut = InspectorTree.build(handle);
@@ -308,7 +308,7 @@ void build_flatSegmentEncodingTreeAtDepthLimit_clearsGuard() {
             SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE);
             givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec));
             byte[] fb = deeplyNestedArrayFlatBuffer(62);
-            given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena));
+            given(handle.segmentTail(spec)).willReturn(wrapAsSegment(fb, arena));
 
             // When
             InspectorTree sut = InspectorTree.build(handle);
@@ -329,7 +329,7 @@ void build_flatSegmentEncodingTreeOneOverDepthLimit_throwsVortexException() {
             SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE);
             givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec));
             byte[] fb = deeplyNestedArrayFlatBuffer(63);
-            given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena));
+            given(handle.segmentTail(spec)).willReturn(wrapAsSegment(fb, arena));
 
             // When / Then
             assertThatThrownBy(() -> InspectorTree.build(handle))
@@ -350,7 +350,7 @@ void build_flatSegmentEncodingIndexOutOfBounds_throwsVortexException() {
             SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE);
             givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec));
             byte[] fb = nestedArrayFlatBuffer(0, 999);
-            given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena));
+            given(handle.segmentTail(spec)).willReturn(wrapAsSegment(fb, arena));
 
             // When / Then
             assertThatThrownBy(() -> InspectorTree.build(handle))
@@ -373,7 +373,7 @@ void build_flatSegmentEncodingTreeOverNodeCountLimit_throwsVortexException() {
             SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE);
             givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec));
             byte[] fb = wideArrayFlatBuffer(InspectorTree.MAX_ENCODING_TREE_NODES + 1);
-            given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena));
+            given(handle.segmentTail(spec)).willReturn(wrapAsSegment(fb, arena));
 
             // When / Then
             assertThatThrownBy(() -> InspectorTree.build(handle))

reader/src/main/java/io/github/dfa1/vortex/reader/VortexHandle.java

@@ -61,6 +61,24 @@ public interface VortexHandle extends Closeable {
     ///         `spec.offset() + spec.length()`
     MemorySegment rawSegment(SegmentSpec spec);
 
+    /// Returns the end of the given segment: at least its array-node FlatBuffer and the 4-byte
+    /// length that follows it, which is all the inspector reads to name an encoding and its stats.
+    /// May be the whole segment. On an HTTP handle it fetches a few KiB instead of the segment.
+    ///
+    /// @param spec the segment to read the end of
+    /// @return a read-only [MemorySegment] ending at the same byte as the segment
+    default MemorySegment segmentTail(SegmentSpec spec) {
+        return rawSegment(spec);
+    }
+
+    /// Hints that `specs` are about to be read with [#segmentTail(SegmentSpec)], so a remote
+    /// handle can fetch them concurrently. Does nothing on a memory-mapped handle. Call it from
+    /// the thread that reads the segments, the one that opened the handle.
+    ///
+    /// @param specs the segments whose ends are about to be read
+    default void prefetchTails(Collection<SegmentSpec> specs) {
+    }
+
     /// Hints that `specs` are about to be read with [#rawSegment(SegmentSpec)], so a remote handle
     /// can fetch them concurrently instead of one round trip each. Does nothing on a
     /// memory-mapped handle, where a read is already free. Call it from the thread that reads the