[AMORO-3084]: iceberg version bump to 1.6.1

apache · Sep 5, 2024 · 5a18ccd · 5a18ccd
1 parent 04fef53
commit 5a18ccd
Show file tree

Hide file tree

Showing 147 changed files with 226 additions and 44,463 deletions.
diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
@@ -24,7 +24,6 @@
 import org.apache.amoro.table.MixedTable;
 import org.apache.amoro.utils.map.StructLikeCollections;
 import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.StructLike;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.data.GenericAppenderFactory;
 import org.apache.iceberg.data.IdentityPartitionConverters;
@@ -46,18 +45,6 @@ public IcebergRewriteExecutor(
     super(input, table, structLikeCollections);
   }
 
-  // TODO We can remove this override method after upgrading Iceberg version to 1.5+.
-  @Override
-  protected StructLike partition() {
-    StructLike partitionData = super.partition();
-    if (partitionData != null && partitionData.size() == 0) {
-      // Cast empty partition data to NULL to avoid creating empty partition directory.
-      return null;
-    } else {
-      return partitionData;
-    }
-  }
-
   @Override
   protected OptimizingDataReader dataReader() {
     return new GenericCombinedIcebergDataReader(

diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
@@ -65,6 +65,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 /** Basic implementation of {@link UnkeyedTable}, wrapping a {@link Table}. */
 public class BasicUnkeyedTable implements UnkeyedTable, HasTableOperations {
@@ -318,4 +319,9 @@ public StructLikeMap<Map<String, String>> partitionProperty() {
   public UpdatePartitionProperties updatePartitionProperties(Transaction transaction) {
     return new PartitionPropertiesUpdate(this, transaction);
   }
+
+  @Override
+  public UUID uuid() {
+    return UUID.fromString(this.operations().current().uuid());
+  }
 }
diff --git a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
@@ -25,24 +25,44 @@
 import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.types.Conversions;
 import org.apache.iceberg.types.Types;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
 
 @RunWith(Parameterized.class)
-public class TestIcebergFindFiles extends TableTestBase {
+public class TestIcebergFindFiles extends TestBase {
+
   @Parameterized.Parameters(name = "formatVersion = {0}")
-  public static Object[] parameters() {
-    return new Object[] {1, 2};
+  public static List<Object> parameters() {
+    return Arrays.asList(1, 2);
   }
 
+  @Rule public TemporaryFolder tempDir = new TemporaryFolder();
+
   public TestIcebergFindFiles(int formatVersion) {
-    super(formatVersion);
+    this.formatVersion = formatVersion;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    this.tableDir = tempDir.newFolder();
+    this.tableDir.delete();
+    super.setupTable();
+  }
+
+  @After
+  public void cleanUp() throws Exception {
+    super.cleanupTables();
   }
 
   @Test

diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-format/pom.xml b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-format/pom.xml
@@ -31,7 +31,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
-        <flink.version>1.17.1</flink.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>

diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-iceberg-bridge/pom.xml b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-iceberg-bridge/pom.xml
@@ -31,10 +31,9 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
-        <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
-        <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.17.1</flink.version>
+        <testcontainers.version>1.18.3</testcontainers.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>
@@ -58,7 +57,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <scope>provided</scope>
             <exclusions>
@@ -99,7 +98,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
-            <version>${flink.version}</version>
+            <version>3.2.0-1.18</version>
             <scope>provided</scope>
         </dependency>
 
@@ -184,7 +183,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>
@@ -275,9 +274,9 @@
 
         <!--   exclude jackson-databind   -->
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${jackson.vesion}</version>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-shaded-jackson</artifactId>
+            <version>2.15.3-18.0</version>
             <scope>provided</scope>
         </dependency>