fix: For failure in reading dictionary encoded parquet strings (deeph…

…aven#5945)
kosak · Aug 15, 2024 · 9ca4332 · 9ca4332
1 parent c74e570
commit 9ca4332
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -267,9 +267,7 @@ public ColumnPageReader next(@NotNull final SeekableChannelContext channelContex
                 final long dataOffset = ch.position();
                 nextHeaderOffset = dataOffset + pageHeader.getCompressed_page_size();
                 final PageType pageType = pageHeader.type;
-                if (pageType == PageType.DICTIONARY_PAGE && headerOffset == columnChunk.meta_data.getData_page_offset()
-                        && columnChunk.meta_data.getDictionary_page_offset() == 0) {
-                    // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding
+                if (pageType == PageType.DICTIONARY_PAGE) {
                     // Skip the dictionary page and jump to the data page
                     return next(holder.get());
                 }

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java
@@ -86,6 +86,20 @@ public void readSampleParquetFilesFromPublicS3Part2() {
                 .head(10).select();
     }
 
+    @Test
+    public void readSampleParquetFilesFromPublicS3Part3() {
+        Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING);
+        final S3Instructions s3Instructions = S3Instructions.builder()
+                .regionName("us-east-1")
+                .readTimeout(Duration.ofSeconds(60))
+                .credentials(Credentials.anonymous())
+                .build();
+        final ParquetInstructions readInstructions = new ParquetInstructions.Builder()
+                .setSpecialInstructions(s3Instructions)
+                .build();
+        readTable("s3://redshift-downloads/redset/serverless/full.parquet", readInstructions).head(10).select();
+    }
+
     @Test
     public void readKeyValuePartitionedParquetFromPublicS3() {
         Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING);