Merge branch 'master' into AMORO-FixBug-in-PageHelper

apache · Dec 6, 2024 · ec18b0b · ec18b0b
2 parents 8fe4573 + 530b700
commit ec18b0b
Show file tree

Hide file tree

Showing 12 changed files with 548 additions and 365 deletions.
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
@@ -394,7 +394,7 @@ public void preHandleRequest(Context ctx) {
   public void handleException(Exception e, Context ctx) {
     if (e instanceof ForbiddenException) {
       // request doesn't start with /ams is  page request. we return index.html
-      if (!ctx.req.getRequestURI().startsWith("/ams")) {
+      if (!ctx.req.getRequestURI().startsWith("/api/ams")) {
         ctx.html(getIndexFileContent());
       } else {
         ctx.json(new ErrorResponse(HttpCode.FORBIDDEN, "Please login first", ""));

diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/utils/IcebergTableUtil.java b/amoro-ams/src/main/java/org/apache/amoro/server/utils/IcebergTableUtil.java
@@ -149,8 +149,9 @@ public static Set<DeleteFile> getDanglingDeleteFiles(Table internalTable) {
     if (internalTable.currentSnapshot() == null) {
       return Collections.emptySet();
     }
+    long snapshotId = internalTable.currentSnapshot().snapshotId();
     Set<String> deleteFilesPath = new HashSet<>();
-    TableScan tableScan = internalTable.newScan();
+    TableScan tableScan = internalTable.newScan().useSnapshot(snapshotId);
     try (CloseableIterable<FileScanTask> fileScanTasks = tableScan.planFiles()) {
       for (FileScanTask fileScanTask : fileScanTasks) {
         for (DeleteFile delete : fileScanTask.deletes()) {
@@ -165,7 +166,7 @@ public static Set<DeleteFile> getDanglingDeleteFiles(Table internalTable) {
     Set<DeleteFile> danglingDeleteFiles = new HashSet<>();
     TableEntriesScan entriesScan =
         TableEntriesScan.builder(internalTable)
-            .useSnapshot(internalTable.currentSnapshot().snapshotId())
+            .useSnapshot(snapshotId)
             .includeFileContent(FileContent.EQUALITY_DELETES, FileContent.POSITION_DELETES)
             .build();
     try (CloseableIterable<IcebergFileEntry> entries = entriesScan.entries()) {

diff --git a/...t-iceberg/src/main/java/org/apache/amoro/optimizing/plan/AbstractOptimizingEvaluator.java b/...t-iceberg/src/main/java/org/apache/amoro/optimizing/plan/AbstractOptimizingEvaluator.java
@@ -32,7 +32,6 @@
 import org.apache.amoro.table.KeyedTableSnapshot;
 import org.apache.amoro.table.MixedTable;
 import org.apache.amoro.table.TableSnapshot;
-import org.apache.amoro.utils.MixedTableUtil;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Snapshot;
 import org.apache.iceberg.SnapshotSummary;
@@ -121,9 +120,7 @@ private void initPartitionPlans(TableFileScanHelper tableFileScanHelper) {
     try (CloseableIterable<TableFileScanHelper.FileScanResult> results =
         tableFileScanHelper.scan()) {
       for (TableFileScanHelper.FileScanResult fileScanResult : results) {
-        PartitionSpec partitionSpec =
-            MixedTableUtil.getMixedTablePartitionSpecById(
-                mixedTable, fileScanResult.file().specId());
+        PartitionSpec partitionSpec = tableFileScanHelper.getSpec(fileScanResult.file().specId());
         StructLike partition = fileScanResult.file().partition();
         String partitionPath = partitionSpec.partitionToPath(partition);
         PartitionEvaluator evaluator =

diff --git a/...at-iceberg/src/main/java/org/apache/amoro/optimizing/scan/IcebergTableFileScanHelper.java b/...at-iceberg/src/main/java/org/apache/amoro/optimizing/scan/IcebergTableFileScanHelper.java
@@ -22,19 +22,24 @@
 import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
 import org.apache.amoro.utils.IcebergThreadPools;
 import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.expressions.Expression;
 import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.io.CloseableIterable;
 
+import java.util.Map;
+
 public class IcebergTableFileScanHelper implements TableFileScanHelper {
   private final Table table;
   private Expression partitionFilter = Expressions.alwaysTrue();
   private final long snapshotId;
+  private final Map<Integer, PartitionSpec> specs;
 
   public IcebergTableFileScanHelper(Table table, long snapshotId) {
     this.table = table;
     this.snapshotId = snapshotId;
+    this.specs = table.specs();
   }
 
   @Override
@@ -61,4 +66,9 @@ public TableFileScanHelper withPartitionFilter(Expression partitionFilter) {
     this.partitionFilter = partitionFilter;
     return this;
   }
+
+  @Override
+  public PartitionSpec getSpec(int specId) {
+    return specs.get(specId);
+  }
 }
diff --git a/...rmat-iceberg/src/main/java/org/apache/amoro/optimizing/scan/KeyedTableFileScanHelper.java b/...rmat-iceberg/src/main/java/org/apache/amoro/optimizing/scan/KeyedTableFileScanHelper.java
@@ -68,11 +68,13 @@ public class KeyedTableFileScanHelper implements TableFileScanHelper {
   private final long changeSnapshotId;
   private final long baseSnapshotId;
   private Expression partitionFilter = Expressions.alwaysTrue();
+  private final PartitionSpec spec;
 
   public KeyedTableFileScanHelper(KeyedTable keyedTable, KeyedTableSnapshot snapshot) {
     this.keyedTable = keyedTable;
     this.baseSnapshotId = snapshot.baseSnapshotId();
     this.changeSnapshotId = snapshot.changeSnapshotId();
+    this.spec = keyedTable.spec();
   }
 
   /**
@@ -441,4 +443,13 @@ public void setMinTransactionIdAfter(long minTransactionIdAfter) {
       this.minTransactionIdAfter = minTransactionIdAfter;
     }
   }
+
+  @Override
+  public PartitionSpec getSpec(int specId) {
+    if (specId != spec.specId()) {
+      throw new IllegalArgumentException(
+          "Partition spec id " + specId + " not found in table " + keyedTable.name());
+    }
+    return spec;
+  }
 }
diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/scan/TableFileScanHelper.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/scan/TableFileScanHelper.java
@@ -20,6 +20,7 @@
 
 import org.apache.iceberg.ContentFile;
 import org.apache.iceberg.DataFile;
+import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.expressions.Expression;
 import org.apache.iceberg.io.CloseableIterable;
 
@@ -47,4 +48,6 @@ public List<ContentFile<?>> deleteFiles() {
   CloseableIterable<FileScanResult> scan();
 
   TableFileScanHelper withPartitionFilter(Expression partitionFilter);
+
+  PartitionSpec getSpec(int specId);
 }
diff --git a/amoro-web/package.json b/amoro-web/package.json
@@ -18,13 +18,13 @@
     "dayjs": "^1.11.11",
     "echarts": "^5.5.0",
     "pinia": "^2.1.7",
+    "swagger-ui": "^5.17.14",
+    "swagger-ui-dist": "^5.17.14",
     "vue": "^3.4.27",
     "vue-clipboard3": "^2.0.0",
     "vue-i18n": "^9.13.1",
     "vue-router": "^4.3.3",
-    "vue-virtual-scroller": "2.0.0-beta.8",
-    "swagger-ui": "^5.17.14",
-    "swagger-ui-dist": "^5.17.14"
+    "vue-virtual-scroller": "2.0.0-beta.8"
   },
   "devDependencies": {
     "@antfu/eslint-config": "^2.21.1",
@@ -41,7 +41,7 @@
     "vite": "^5.2.13",
     "vite-plugin-fake-server": "^2.1.1",
     "vite-plugin-svg-icons": "^2.0.1",
-    "vite-plugin-vue-devtools": "^7.4.5",
+    "vite-plugin-vue-devtools": "^7.6.7",
     "vue-tsc": "^2.0.21"
   },
   "engines": {