GH-3095: Follow the spec's LIST rules for the Avro schema parquet-cli derives

parquet-cli/src/main/java/org/apache/parquet/cli/util/Schemas.java

@@ -82,7 +82,15 @@ public static Schema fromParquet(Configuration conf, URI location) throws IOExce
     if (schemaString != null) {
       return new Schema.Parser().parse(schemaString);
     } else {
-      return new AvroSchemaConverter(conf)
+      // Without an Avro schema, the file was likely not written by parquet-avro, so apply the LIST
+      // backward-compatibility rules of the Parquet spec: a repeated group with a single field is a
+      // synthetic layer around the element unless it is named "array" or "<list name>_tuple".
+      Configuration converterConf = conf;
+      if (conf.get(AvroSchemaConverter.ADD_LIST_ELEMENT_RECORDS) == null) {
+        converterConf = new Configuration(conf);
+        converterConf.setBoolean(AvroSchemaConverter.ADD_LIST_ELEMENT_RECORDS, false);
+      }
+      return new AvroSchemaConverter(converterConf)
           .convert(footer.getFileMetaData().getSchema());
     }
   }

parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java

@@ -25,6 +25,8 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.IntStream;
@@ -105,6 +107,51 @@ void scanRejectsColumnMissingFromInputSchema() throws Exception {
     assertThat(result.output()).contains("Cannot find field", "missing_field");
   }
 
+  @Test
+  void catReadsStandardListsWithoutAvroSchema() throws Exception {
+    // Written without an Avro schema in the file metadata, like files from Spark or PyArrow
+    Path path = getTempFolder().toPath().resolve("lists.parquet");
+    MessageType schema = Types.buildMessage()
+        .required(PrimitiveTypeName.INT32)
+        .named("id")
+        .optionalList()
+        .optionalElement(PrimitiveTypeName.BINARY)
+        .as(LogicalTypeAnnotation.stringType())
+        .named("tags")
+        .named("record");
+    List<List<String>> tags = Arrays.asList(List.of("a", "b"), List.of(), null, Arrays.asList("c", null));
+    SimpleGroupFactory groups = new SimpleGroupFactory(schema);
+    try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toUri()))
+        .withType(schema)
+        .build()) {
+      for (int i = 0; i < tags.size(); i++) {
+        Group record = groups.newGroup().append("id", i);
+        if (tags.get(i) != null) {
+          Group list = record.addGroup("tags");
+          for (String tag : tags.get(i)) {
+            Group element = list.addGroup("list");
+            if (tag != null) {
+              element.append("element", tag);
+            }
+          }
+        }
+        writer.write(record);
+      }
+    }
+
+    CliResult result = cli("cat", path.toString());
+
+    assertThat(result.exitCode()).as(result.output()).isZero();
+    List<Map<String, Object>> expected = new ArrayList<>();
+    for (int i = 0; i < tags.size(); i++) {
+      Map<String, Object> record = new HashMap<>();
+      record.put("id", i);
+      record.put("tags", tags.get(i));
+      expected.add(record);
+    }
+    assertThat(readRecords(result.output())).containsExactlyElementsOf(expected);
+  }
+
   private Path writeRecords(String name, int firstId, int count) throws IOException {
     Path path = getTempFolder().toPath().resolve(name);
     MessageType schema = Types.buildMessage()

parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java

@@ -24,6 +24,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import org.apache.avro.Schema;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
 import org.junit.jupiter.api.Test;
 
 class SchemaCliTest extends CliTestBase {
@@ -64,6 +71,57 @@ void printsAvroSchemaByDefault() throws Exception {
         .isEqualTo("date");
   }
 
+  @Test
+  void printsStandardListsWithoutAvroSchemaAsArrays() throws Exception {
+    CliResult result = cli("schema", writeStandardListFile().toString());
+
+    assertThat(result.exitCode()).as(result.output()).isZero();
+    Schema tags = listSchema(result.output(), "tags");
+    assertThat(tags.getType()).isEqualTo(Schema.Type.ARRAY);
+    assertThat(tags.getElementType().toString()).isEqualTo("[\"null\",\"string\"]");
+  }
+
+  @Test
+  void keepsExplicitAddListElementRecordsSetting() throws Exception {
+    CliResult result = cli(
+        "-D",
+        "parquet.avro.add-list-element-records=true",
+        "schema",
+        writeStandardListFile().toString());
+
+    assertThat(result.exitCode()).as(result.output()).isZero();
+    Schema element = listSchema(result.output(), "tags").getElementType();
+    assertThat(element.getType()).isEqualTo(Schema.Type.RECORD);
+    assertThat(element.getName()).isEqualTo("list");
+    assertThat(element.getFields()).extracting(Schema.Field::name).containsExactly("element");
+  }
+
+  // Written without an Avro schema in the file metadata, like files from Spark or PyArrow
+  private Path writeStandardListFile() throws Exception {
+    Path input = getTempFolder().toPath().resolve("lists.parquet");
+    MessageType parquetSchema = Types.buildMessage()
+        .optionalList()
+        .optionalElement(PrimitiveTypeName.BINARY)
+        .as(LogicalTypeAnnotation.stringType())
+        .named("tags")
+        .named("record");
+    try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(input.toUri()))
+        .withType(parquetSchema)
+        .build()) {
+      // only the schema is needed
+    }
+    return input;
+  }
+
+  private static Schema listSchema(String avroSchema, String field) {
+    return new Schema.Parser()
+        .parse(avroSchema)
+        .getField(field)
+        .schema()
+        .getTypes()
+        .get(1);
+  }
+
   @Test
   void printsParquetSchemaForPathContainingSpaces() throws Exception {
     Path input = getTempFolder().toPath().resolve("input with spaces.parquet");