diff --git a/.github/workflows/core-hadoop3-ci.yml b/.github/workflows/core-hadoop3-ci.yml
index aed6c0e462..cd5dbe0e2d 100644
--- a/.github/workflows/core-hadoop3-ci.yml
+++ b/.github/workflows/core-hadoop3-ci.yml
@@ -67,12 +67,56 @@ jobs:
if: matrix.spark == '3.3'
run: ./mvnw clean install -pl '!amoro-ams,!dist' -Djacoco.flink.skip=true -B -ntp -Pjava${{ matrix.jdk }} -Pspark-${{ matrix.spark }}
+ # Standard-directory tests already run in the full build above.
+ - name: Test Paimon AMS integration
+ if: matrix.spark == '3.5'
+ run: |
+ ./mvnw test -pl amoro-ams -am -B -ntp \
+ -Pjava${{ matrix.jdk }},support-paimon-format,spark-3.5,skip-dashboard-build \
+ -Dtest=TestPaimonProcessIntegration \
+ -Dsurefire.failIfNoSpecifiedTests=false
+ python3 - <<'PYTHON'
+ import xml.etree.ElementTree as ET
+ report = ET.parse('amoro-ams/target/surefire-reports/TEST-org.apache.amoro.server.process.TestPaimonProcessIntegration.xml').getroot()
+ assert int(report.get('tests', '0')) > 0, 'Paimon AMS integration tests did not run'
+ assert all(int(report.get(key, '0')) == 0 for key in ('failures', 'errors', 'skipped')), report.attrib
+ PYTHON
+
- name: Code coverage
uses: codecov/codecov-action@v3
with:
verbose: true
flags: core
+ paimon-1-1-1:
+ name: Paimon 1.1.1 format regression with JDK 17
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: maven
+ check-latest: false
+ - name: Test Paimon 1.1.1 and Scala 2.13
+ run: |
+ ./mvnw clean test -pl amoro-format-paimon -am -B -ntp \
+ -Pjava17,scala-2.13 \
+ -Dtest=TestPaimonSnapshotExpiration,TestPaimonTableRefresh \
+ -Dsurefire.failIfNoSpecifiedTests=false
+ python3 - <<'PYTHON'
+ import xml.etree.ElementTree as ET
+ for test_class in (
+ 'org.apache.amoro.formats.paimon.process.TestPaimonSnapshotExpiration',
+ 'org.apache.amoro.formats.paimon.TestPaimonTableRefresh',
+ ):
+ report = ET.parse(f'amoro-format-paimon/target/surefire-reports/TEST-{test_class}.xml').getroot()
+ assert int(report.get('tests', '0')) > 0, f'{test_class} did not run'
+ assert all(int(report.get(key, '0')) == 0 for key in ('failures', 'errors', 'skipped')), report.attrib
+ PYTHON
+
spark-optimizer-4-1:
name: Build Spark 4.1 Optimizer with JDK 17
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index a88a820d98..3792035891 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,6 +82,10 @@ resources
!javadoc/resources
!amoro-ams/src/main/resources/
!amoro-ams/src/main/resources/**
+!amoro-format-paimon/src/main/resources/
+!amoro-format-paimon/src/main/resources/META-INF/
+!amoro-format-paimon/src/main/resources/META-INF/services/
+!amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory
!dist/src/main/amoro-bin/bin/
!dist/src/main/amoro-bin/conf/
diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml
index 9ba57d0011..4a51b32e9b 100644
--- a/amoro-ams/pom.xml
+++ b/amoro-ams/pom.xml
@@ -518,6 +518,12 @@
${swagger-ui.version}
+
+ org.junit.platform
+ junit-platform-launcher
+ test
+
+
org.testcontainers
@@ -603,6 +609,41 @@
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+ ${maven-build-helper-plugin.version}
+
+
+ paimon-integration-tests
+
+ add-test-source
+
+ generate-test-sources
+
+
+ src/test/paimon
+
+
+
+
+
+
+ com.diffplug.spotless
+ spotless-maven-plugin
+
+
+
+ src/main/java/**/*.java
+ src/test/**/*.java
+
+
+
+
+
+
pl.project13.maven
@@ -706,6 +747,14 @@
amoro-format-paimon
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+
support-hudi-format
@@ -757,6 +806,14 @@
amoro-format-lance
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+
fail-on-no-git-dir
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
index c5829edbb4..37fa4a51fc 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
@@ -261,6 +261,7 @@ private void startOptimizingService() throws Exception {
catalogManager,
optimizerManager,
tableService,
+ processService,
bucketAssignStore,
haContainer);
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java
index afccf8a970..e5d4688714 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java
@@ -54,6 +54,7 @@
import org.apache.amoro.server.persistence.mapper.OptimizerMapper;
import org.apache.amoro.server.persistence.mapper.ResourceMapper;
import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
+import org.apache.amoro.server.process.ProcessService;
import org.apache.amoro.server.process.TableProcessMeta;
import org.apache.amoro.server.resource.Containers;
import org.apache.amoro.server.resource.OptimizerInstance;
@@ -137,6 +138,7 @@ public class DefaultOptimizingService extends StatedPersistentBase
private final CatalogManager catalogManager;
private final OptimizerManager optimizerManager;
private final TableService tableService;
+ private final ProcessService processService;
private final RuntimeHandlerChain tableHandlerChain;
private final ExecutorService planExecutor;
private final BucketAssignStore bucketAssignStore;
@@ -148,6 +150,7 @@ public DefaultOptimizingService(
CatalogManager catalogManager,
OptimizerManager optimizerManager,
TableService tableService,
+ ProcessService processService,
BucketAssignStore bucketAssignStore,
HighAvailabilityContainer haContainer) {
this.optimizerTouchTimeout =
@@ -170,6 +173,7 @@ public DefaultOptimizingService(
this.groupMaxKeepingAttempts =
serviceConfig.getInteger(AmoroManagementConf.OPTIMIZER_GROUP_MAX_KEEPING_ATTEMPTS);
this.tableService = tableService;
+ this.processService = processService;
this.catalogManager = catalogManager;
this.optimizerManager = optimizerManager;
this.bucketAssignStore = bucketAssignStore;
@@ -499,16 +503,18 @@ public boolean cancelProcess(long processId) {
return false;
}
long tableId = processMeta.getTableId();
- DefaultTableRuntime tableRuntime = (DefaultTableRuntime) tableService.getRuntime(tableId);
+ TableRuntime tableRuntime = tableService.getRuntime(tableId);
if (tableRuntime == null) {
return false;
}
- OptimizingProcess process = tableRuntime.getOptimizingProcess();
- if (process == null || process.getProcessId() != processId) {
- return false;
+ if (tableRuntime instanceof DefaultTableRuntime) {
+ OptimizingProcess process = ((DefaultTableRuntime) tableRuntime).getOptimizingProcess();
+ if (process != null && process.getProcessId() == processId) {
+ process.close(true);
+ return true;
+ }
}
- process.close(true);
- return true;
+ return processService.cancelProcess(tableRuntime.getTableIdentifier(), processId);
}
@Override
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java
index 2db674fd54..52e9f2a744 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java
@@ -687,6 +687,11 @@ public Pair, Integer> getOptimizingProcessesInfo(
if (identifier == null) {
return Pair.of(Collections.emptyList(), 0);
}
+ if (TableProcessQuery.handles(this, processCategory, type)) {
+ return new TableProcessQuery()
+ .list(
+ identifier, getProcessTypesByCategory(processCategory), type, status, limit, offset);
+ }
int total = 0;
// page helper is 1-based
int pageNumber = (offset / limit) + 1;
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java
index bbd68bf691..ba7f6fd92a 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java
@@ -137,6 +137,16 @@ public Pair, Integer> getOptimizingProcessesInfo(
int offset) {
AmoroTable> amoroTable = loadTable(tableIdentifier);
FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format());
+ if (TableProcessQuery.handles(formatTableDescriptor, processCategory, type)) {
+ return new TableProcessQuery()
+ .list(
+ amoroTable.id(),
+ formatTableDescriptor.getProcessTypesByCategory(processCategory),
+ type,
+ status,
+ limit,
+ offset);
+ }
return formatTableDescriptor.getOptimizingProcessesInfo(
amoroTable, type, processCategory, status, limit, offset);
}
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java
new file mode 100644
index 0000000000..61653cba1f
--- /dev/null
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard;
+
+import com.github.pagehelper.Page;
+import com.github.pagehelper.PageHelper;
+import com.github.pagehelper.PageInfo;
+import org.apache.amoro.ServerTableIdentifier;
+import org.apache.amoro.process.ProcessStatus;
+import org.apache.amoro.server.persistence.PersistentBase;
+import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
+import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
+import org.apache.amoro.server.process.TableProcessMeta;
+import org.apache.amoro.table.TableIdentifier;
+import org.apache.amoro.table.descriptor.FilesStatistics;
+import org.apache.amoro.table.descriptor.FormatTableDescriptor;
+import org.apache.amoro.table.descriptor.OptimizingProcessInfo;
+import org.apache.amoro.table.descriptor.ProcessCategory;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Format-independent queries of AMS cleanup records, without optimizing task metrics. */
+public class TableProcessQuery extends PersistentBase {
+ static boolean handles(FormatTableDescriptor descriptor, String category, String type) {
+ return ProcessCategory.CLEANUP.getName().equalsIgnoreCase(category)
+ && (StringUtils.isBlank(type)
+ || descriptor.getProcessTypesByCategory(category).contains(type));
+ }
+
+ Pair, Integer> list(
+ TableIdentifier table,
+ List actions,
+ String type,
+ ProcessStatus status,
+ int limit,
+ int offset) {
+ ServerTableIdentifier identifier =
+ getAs(
+ TableMetaMapper.class,
+ mapper ->
+ mapper.selectTableIdentifier(
+ table.getCatalog(), table.getDatabase(), table.getTableName()));
+ return list(identifier, actions, type, status, limit, offset);
+ }
+
+ public Pair, Integer> list(
+ ServerTableIdentifier table,
+ List actions,
+ String type,
+ ProcessStatus status,
+ int limit,
+ int offset) {
+ String filterType = StringUtils.isBlank(type) ? null : type;
+ if (table == null || (filterType == null && (actions == null || actions.isEmpty()))) {
+ return Pair.of(Collections.emptyList(), 0);
+ }
+ try (Page> ignored = PageHelper.startPage(offset / limit + 1, limit, true)) {
+ List records =
+ getAs(
+ TableProcessMapper.class,
+ mapper ->
+ mapper.listProcessMeta(
+ table.getId(), filterType, filterType == null ? actions : null, status));
+ PageInfo page = new PageInfo<>(records);
+ if (page.getSize() == 0) {
+ return Pair.of(Collections.emptyList(), 0);
+ }
+ return Pair.of(
+ records.stream().map(meta -> toInfo(table, meta)).collect(Collectors.toList()),
+ (int) page.getTotal());
+ }
+ }
+
+ private static OptimizingProcessInfo toInfo(ServerTableIdentifier table, TableProcessMeta meta) {
+ OptimizingProcessInfo result = new OptimizingProcessInfo();
+ result.setTableId(meta.getTableId());
+ result.setCatalogName(table.getCatalog());
+ result.setDbName(table.getDatabase());
+ result.setTableName(table.getTableName());
+ result.setProcessId(String.valueOf(meta.getProcessId()));
+ result.setStartTime(meta.getCreateTime());
+ result.setFinishTime(meta.getFinishTime());
+ result.setDuration(
+ (meta.getFinishTime() > 0 ? meta.getFinishTime() : System.currentTimeMillis())
+ - meta.getCreateTime());
+ result.setOptimizingType(meta.getProcessType());
+ result.setStatus(meta.getStatus());
+ result.setFailReason(meta.getFailMessage());
+ result.setInputFiles(new FilesStatistics());
+ result.setOutputFiles(new FilesStatistics());
+ result.setSummary(meta.getSummary());
+ return result;
+ }
+}
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java
index a2219f7370..4ef746cad9 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java
@@ -706,7 +706,10 @@ public void cancelOptimizingProcess(Context ctx) {
OptimizingClientPools.getClient(
AmsUtil.getAMSThriftAddress(serviceConfig, Constants.THRIFT_OPTIMIZING_SERVICE_NAME));
try {
- client.cancelProcess(processId);
+ Preconditions.checkState(
+ client.cancelProcess(processId),
+ "Process could not be canceled: it is no longer active or is not managed by this AMS, processId: %s",
+ processId);
} catch (TException e) {
throw new IllegalStateException("Failed to cancel optimizing process:" + e.getMessage());
}
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java
index 0dfce34f9e..e4c74997f0 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java
@@ -284,7 +284,8 @@ protected boolean validTransition(
|| getStatus() == ProcessStatus.PENDING)
&& newStatus == ProcessStatus.FAILED);
case CANCEL_REQUESTED:
- return (getStatus() == ProcessStatus.UNKNOWN
+ return (getStatus() == ProcessStatus.FAILED
+ || getStatus() == ProcessStatus.UNKNOWN
|| getStatus() == ProcessStatus.SUBMITTED
|| getStatus() == ProcessStatus.RUNNING
|| getStatus() == ProcessStatus.PENDING)
@@ -324,11 +325,6 @@ public boolean tryTransitState(
transitionLock.lock();
try {
int priority = processEvent.getPriority();
- if (priority >= ProcessEvent.KILL_REQUESTED.getPriority()) {
- highestPriority.updateAndGet(curr -> Math.max(curr, priority));
- terminalRequested = true;
- }
-
ProcessStatus currentStatus = getStatus();
if (isTerminal(currentStatus)) {
LOG.warn(
@@ -393,6 +389,10 @@ public boolean tryTransitState(
default:
return false;
}
+ if (priority >= ProcessEvent.KILL_REQUESTED.getPriority()) {
+ highestPriority.updateAndGet(curr -> Math.max(curr, priority));
+ terminalRequested = true;
+ }
} finally {
transitionLock.unlock();
}
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java
index bf44c64808..b21fb8f737 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java
@@ -126,6 +126,15 @@ public void dispose() {
activeTableProcess.clear();
}
+ /**
+ * Await worker termination after {@link #dispose()}, without initiating shutdown or cancellation.
+ * Tests use this to keep persistence available until completion callbacks have returned.
+ */
+ @VisibleForTesting
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return processExecutionPool.awaitTermination(timeout, unit);
+ }
+
/**
* Find the scheduler that supports the given table format for the specified action.
*
@@ -410,32 +419,45 @@ private void executeOrTraceProcess(TableProcessStore store, TableProcess process
}
/**
- * Cancel the given process and untrack it.
+ * Cancel a process owned by this service for the given table, using the existing lifecycle.
*
- * @param process table process
+ * @param table table identity
+ * @param processId process identifier
+ * @return true if the tracked process transitioned to canceled, false if absent or terminal
*/
- private void cancelProcess(TableProcessStore store, TableProcess process) {
-
- store.tryTransitState(
- ProcessStatus.CANCELED,
- ProcessEvent.CANCEL_REQUESTED,
- store.getExternalProcessIdentifier(),
- "Gracefully Canceled.",
- process.getProcessParameters(),
- process.getSummary());
+ public boolean cancelProcess(ServerTableIdentifier table, long processId) {
+ TableProcessHolder holder = getTableProcessInstances(table).get(processId);
+ return holder != null && cancelProcess(holder.getStore(), holder.getProcess());
+ }
+
+ private boolean cancelProcess(TableProcessStore store, TableProcess process) {
+
+ boolean canceled =
+ store.tryTransitState(
+ ProcessStatus.CANCELED,
+ ProcessEvent.CANCEL_REQUESTED,
+ store.getExternalProcessIdentifier(),
+ "Gracefully Canceled.",
+ process.getProcessParameters(),
+ process.getSummary());
+ if (!canceled) {
+ return false;
+ }
untrackTableProcessInstance(
process.getTableRuntime().getTableIdentifier(), store.getProcessId());
ExecuteEngine executeEngine = executeEngines.get(store.getExecutionEngine());
- if (executeEngine != null) {
- executeEngine.tryCancelTableProcess(process, store.getExternalProcessIdentifier());
+ String externalIdentifier = store.getExternalProcessIdentifier();
+ if (executeEngine != null && externalIdentifier != null && !externalIdentifier.isEmpty()) {
+ executeEngine.tryCancelTableProcess(process, externalIdentifier);
LOG.info(
"Cancel table process {} in engine {}, process id:{}",
process,
executeEngine.name(),
store.getProcessId());
}
+ return canceled;
}
/**
@@ -700,7 +722,10 @@ protected void handleTableRemoved(TableRuntime tableRuntime) {
.collect(Collectors.toList());
for (TableProcessHolder holder : processes) {
if (holder != null) {
- cancelProcess(holder.getStore(), holder.getProcess());
+ if (!cancelProcess(holder.getStore(), holder.getProcess())) {
+ untrackTableProcessInstance(
+ tableRuntime.getTableIdentifier(), holder.getStore().getProcessId());
+ }
}
}
}
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java
index b16517e3a0..e9f67f07f3 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java
@@ -70,119 +70,139 @@ public void run() {
}
try {
- if (store.getStatus() == ProcessStatus.UNKNOWN
- || store.getStatus() == ProcessStatus.PENDING
- || Strings.isNullOrEmpty(store.getExternalProcessIdentifier())) {
- externalProcessIdentifier = executeEngine.submitTableProcess(tableProcess);
- LOG.info(
- "Submit table process {} to engine {} success, external process identifier is {}",
- store.getProcessId(),
- executeEngine.engineType(),
- externalProcessIdentifier);
- } else {
- externalProcessIdentifier = store.getExternalProcessIdentifier();
- }
-
- validateIdentifier(externalProcessIdentifier);
-
- status = executeEngine.getStatus(externalProcessIdentifier);
-
- // If the engine returns UNKNOWN, the process was lost (e.g., AMS restart cleared
- // LocalExecutionEngine's in-memory process map). Treat it as FAILED so the process
- // is properly terminated and optionally retried, rather than being stuck in UNKNOWN.
- if (status == ProcessStatus.UNKNOWN) {
- LOG.warn(
- "Table process {} got UNKNOWN status from engine (identifier={}), "
- + "likely due to AMS restart or process loss, marking as FAILED",
- store.getProcessId(),
- externalProcessIdentifier);
- status = ProcessStatus.FAILED;
- message = "Process lost: engine returned UNKNOWN status";
- } else {
- store.tryTransitState(
- status,
- ProcessEvent.SUBMIT_REQUESTED,
- externalProcessIdentifier,
- "Complete Submitted.",
- tableProcess.getProcessParameters(),
- tableProcess.getSummary());
- }
+ try {
+ if (store.getStatus() == ProcessStatus.UNKNOWN
+ || store.getStatus() == ProcessStatus.PENDING
+ || Strings.isNullOrEmpty(store.getExternalProcessIdentifier())) {
+ externalProcessIdentifier = executeEngine.submitTableProcess(tableProcess);
+ LOG.info(
+ "Submit table process {} to engine {} success, external process identifier is {}",
+ store.getProcessId(),
+ executeEngine.engineType(),
+ externalProcessIdentifier);
+ } else {
+ externalProcessIdentifier = store.getExternalProcessIdentifier();
+ }
- while (isTableProcessExecuting(status)) {
+ validateIdentifier(externalProcessIdentifier);
if (isTableProcessCanceling(store.getStatus())) {
+ return;
+ }
+
+ status = executeEngine.getStatus(externalProcessIdentifier);
+
+ // If the engine returns UNKNOWN, the process was lost (e.g., AMS restart cleared
+ // LocalExecutionEngine's in-memory process map). Treat it as FAILED so the process
+ // is properly terminated and optionally retried, rather than being stuck in UNKNOWN.
+ if (status == ProcessStatus.UNKNOWN) {
+ LOG.warn(
+ "Table process {} got UNKNOWN status from engine (identifier={}), "
+ + "likely due to AMS restart or process loss, marking as FAILED",
+ store.getProcessId(),
+ externalProcessIdentifier);
+ status = ProcessStatus.FAILED;
+ message = "Process lost: engine returned UNKNOWN status";
+ } else {
+ store.tryTransitState(
+ status,
+ ProcessEvent.SUBMIT_REQUESTED,
+ externalProcessIdentifier,
+ "Complete Submitted.",
+ tableProcess.getProcessParameters(),
+ tableProcess.getSummary());
+ }
+
+ while (isTableProcessExecuting(status)) {
+ if (isTableProcessCanceling(store.getStatus())) {
+ LOG.info(
+ "Table process {} with identifier {} may have been in canceling, exit submit process.",
+ store.getProcessId(),
+ externalProcessIdentifier);
+ return;
+ }
+ try {
+ Thread.sleep(DEFAULT_POLL_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ throw e;
+ }
+ status = executeEngine.getStatus(externalProcessIdentifier);
+ }
+ } catch (Throwable t) {
+ if (t instanceof InterruptedException) {
LOG.info(
- "Table process {} with identifier {} may have been in canceling, exit submit process.",
+ "Table process {} with identifier {} may have been interrupted by process service disposing, exit submit process.",
store.getProcessId(),
externalProcessIdentifier);
return;
+ } else {
+ status = ProcessStatus.FAILED;
+ message = t.getMessage();
}
- try {
- Thread.sleep(DEFAULT_POLL_INTERVAL_MS);
- } catch (InterruptedException e) {
- throw e;
- }
- status = executeEngine.getStatus(externalProcessIdentifier);
}
- } catch (Throwable t) {
- if (t instanceof InterruptedException) {
- LOG.info(
- "Table process {} with identifier {} may have been interrupted by process service disposing, exit submit process.",
- store.getProcessId(),
- externalProcessIdentifier);
- return;
+
+ LOG.info("The process {} is finished with status {}", tableProcess, status);
+ if (status == ProcessStatus.KILLED) {
+ store.tryTransitState(
+ status,
+ ProcessEvent.KILL_REQUESTED,
+ store.getExternalProcessIdentifier(),
+ "Gracefully Killed.",
+ tableProcess.getProcessParameters(),
+ tableProcess.getSummary());
+ } else if (status == ProcessStatus.CANCELED) {
+ store.tryTransitState(
+ status,
+ ProcessEvent.CANCEL_REQUESTED,
+ store.getExternalProcessIdentifier(),
+ "Gracefully Cancelled.",
+ tableProcess.getProcessParameters(),
+ tableProcess.getSummary());
+ } else if (status == ProcessStatus.CLOSED) {
+ store.tryTransitState(
+ status,
+ ProcessEvent.KILL_REQUESTED,
+ store.getExternalProcessIdentifier(),
+ "Gracefully Closed.",
+ tableProcess.getProcessParameters(),
+ tableProcess.getSummary());
+ } else if (status == ProcessStatus.FAILED) {
+ store.tryTransitState(
+ status,
+ ProcessEvent.COMPLETE_FAILED,
+ store.getExternalProcessIdentifier(),
+ message,
+ tableProcess.getProcessParameters(),
+ tableProcess.getSummary());
+ } else if (status == ProcessStatus.SUCCESS) {
+ store.tryTransitState(
+ status,
+ ProcessEvent.COMPLETE_SUCCESS,
+ store.getExternalProcessIdentifier(),
+ "Complete Success",
+ tableProcess.getProcessParameters(),
+ tableProcess.getSummary());
} else {
- status = ProcessStatus.FAILED;
- message = t.getMessage();
+ LOG.warn("Un expected terminal status: {} for process: {}.", status, store.getProcessId());
}
- }
- LOG.info("The process {} is finished with status {}", tableProcess, status);
- if (status == ProcessStatus.KILLED) {
- store.tryTransitState(
- status,
- ProcessEvent.KILL_REQUESTED,
- store.getExternalProcessIdentifier(),
- "Gracefully Killed.",
- tableProcess.getProcessParameters(),
- tableProcess.getSummary());
- } else if (status == ProcessStatus.CANCELED) {
- store.tryTransitState(
- status,
- ProcessEvent.CANCEL_REQUESTED,
- store.getExternalProcessIdentifier(),
- "Gracefully Cancelled.",
- tableProcess.getProcessParameters(),
- tableProcess.getSummary());
- } else if (status == ProcessStatus.CLOSED) {
- store.tryTransitState(
- status,
- ProcessEvent.KILL_REQUESTED,
- store.getExternalProcessIdentifier(),
- "Gracefully Closed.",
- tableProcess.getProcessParameters(),
- tableProcess.getSummary());
- } else if (status == ProcessStatus.FAILED) {
- store.tryTransitState(
- status,
- ProcessEvent.COMPLETE_FAILED,
- store.getExternalProcessIdentifier(),
- message,
- tableProcess.getProcessParameters(),
- tableProcess.getSummary());
- } else if (status == ProcessStatus.SUCCESS) {
- store.tryTransitState(
- status,
- ProcessEvent.COMPLETE_SUCCESS,
- store.getExternalProcessIdentifier(),
- "Complete Success",
- tableProcess.getProcessParameters(),
- tableProcess.getSummary());
- } else {
- LOG.warn("Un expected terminal status: {} for process: {}.", status, store.getProcessId());
- }
-
- if (finishedCallback != null) {
- finishedCallback.run();
+ if (finishedCallback != null) {
+ finishedCallback.run();
+ }
+ } finally {
+ // Cancellation can win before submit returns or before its identifier is persisted.
+ // Always hand the identifier obtained by this attempt to the engine before exiting.
+ if (!Strings.isNullOrEmpty(externalProcessIdentifier)
+ && isTableProcessCanceling(store.getStatus())) {
+ try {
+ executeEngine.tryCancelTableProcess(tableProcess, externalProcessIdentifier);
+ } catch (Exception e) {
+ LOG.error(
+ "Failed to cancel submitted process {} with identifier {}",
+ store.getProcessId(),
+ externalProcessIdentifier,
+ e);
+ }
+ }
}
}
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java b/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java
index 9c2abe9698..9015cb2697 100644
--- a/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java
@@ -61,10 +61,16 @@ protected static void initTableService(Duration optimizerHeartbeatTimeout) {
Duration.ofMillis(10L));
TABLE_SERVICE =
new DefaultTableService(new Configurations(), CATALOG_MANAGER, runtimeFactory);
+ PROCESS_SERVICE = new ProcessService(TABLE_SERVICE);
OPTIMIZING_SERVICE =
new DefaultOptimizingService(
- configurations, CATALOG_MANAGER, OPTIMIZER_MANAGER, TABLE_SERVICE, null, null);
- PROCESS_SERVICE = new ProcessService(TABLE_SERVICE);
+ configurations,
+ CATALOG_MANAGER,
+ OPTIMIZER_MANAGER,
+ TABLE_SERVICE,
+ PROCESS_SERVICE,
+ null,
+ null);
TABLE_SERVICE.addHandlerChain(OPTIMIZING_SERVICE.getTableRuntimeHandler());
TABLE_SERVICE.addHandlerChain(PROCESS_SERVICE.getTableHandlerChain());
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
index e146a57eea..21dc77cfc9 100644
--- a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
@@ -49,6 +49,7 @@
import org.apache.amoro.process.ProcessStatus;
import org.apache.amoro.resource.ResourceGroup;
import org.apache.amoro.server.manager.MetricManager;
+import org.apache.amoro.server.optimizing.OptimizingProcess;
import org.apache.amoro.server.optimizing.OptimizingQueue;
import org.apache.amoro.server.optimizing.OptimizingStatus;
import org.apache.amoro.server.optimizing.TaskRuntime;
@@ -66,6 +67,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.MixedTable;
+import org.apache.amoro.table.TableProperties;
import org.apache.amoro.table.UnkeyedTable;
import org.apache.amoro.utils.SerializationUtil;
import org.apache.ibatis.session.SqlSession;
@@ -167,6 +169,39 @@ private void appendData(UnkeyedTable table, int id) {
appendFiles.commit();
}
+ @Test
+ public void testCancelProcessKeepsOptimizingLifecycle() {
+ OptimizingTask task = optimizingService().pollTask(token, THREAD_ID);
+ Assertions.assertNotNull(task);
+ DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId());
+ Assertions.assertFalse(runtime.isAllowPartialCommit());
+ OptimizingProcess process = runtime.getOptimizingProcess();
+ Assertions.assertTrue(optimizingService().cancelProcess(process.getProcessId()));
+ Assertions.assertEquals(ProcessStatus.CLOSED, process.getStatus());
+ Assertions.assertNull(runtime.getOptimizingProcess());
+ Assertions.assertEquals(OptimizingStatus.IDLE, runtime.getOptimizingStatus());
+ Assertions.assertFalse(optimizingService().cancelProcess(process.getProcessId()));
+ }
+
+ @Test
+ public void testCancelProcessKeepsPartialCommit() {
+ MixedTable table =
+ (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable();
+ table
+ .asUnkeyedTable()
+ .updateProperties()
+ .set(TableProperties.SELF_OPTIMIZING_ALLOW_PARTIAL_COMMIT, "true")
+ .commit();
+ DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId());
+ runtime.refresh(tableService().loadTable(serverTableIdentifier()));
+ Assertions.assertTrue(runtime.isAllowPartialCommit());
+ Assertions.assertNotNull(optimizingService().pollTask(token, THREAD_ID));
+ OptimizingProcess process = runtime.getOptimizingProcess();
+ Assertions.assertTrue(optimizingService().cancelProcess(process.getProcessId()));
+ Assertions.assertEquals(OptimizingStatus.COMMITTING, runtime.getOptimizingStatus());
+ Assertions.assertFalse(process.isClosed());
+ }
+
@Test
public void testPollWithoutAuth() {
// 1.poll task
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java b/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java
index 81f71da8d7..e1e841f6d1 100644
--- a/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java
@@ -722,7 +722,13 @@ public void testDisposeUnregistersDraMetrics() {
optimizerManager().createResourceGroup(group);
DefaultOptimizingService formerLeader =
new DefaultOptimizingService(
- new Configurations(), catalogManager(), optimizerManager(), tableService(), null, null);
+ new Configurations(),
+ catalogManager(),
+ optimizerManager(),
+ tableService(),
+ processServiceService(),
+ null,
+ null);
formerLeader.createResourceGroup(group);
Assertions.assertNotNull(
draMetric(DynamicAllocationMetrics.OPTIMIZER_GROUP_SCALE_UP_TOTAL, group.getName()));
@@ -750,7 +756,13 @@ public void testWatchAfterDisposeDoesNotRegisterMetrics() {
ResourceGroup group = buildSlowDraResourceGroup(TEST_GROUP_NAME + "-19", 0);
DefaultOptimizingService formerLeader =
new DefaultOptimizingService(
- new Configurations(), catalogManager(), optimizerManager(), tableService(), null, null);
+ new Configurations(),
+ catalogManager(),
+ optimizerManager(),
+ tableService(),
+ processServiceService(),
+ null,
+ null);
formerLeader.dispose();
formerLeader.updateResourceGroup(group);
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java
new file mode 100644
index 0000000000..572d3b28d9
--- /dev/null
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard;
+
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.ServerTableIdentifier;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.optimizing.MetricsSummary;
+import org.apache.amoro.process.ProcessStatus;
+import org.apache.amoro.server.catalog.CatalogManager;
+import org.apache.amoro.server.catalog.ServerCatalog;
+import org.apache.amoro.server.persistence.PersistentBase;
+import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
+import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
+import org.apache.amoro.server.table.DerbyPersistence;
+import org.apache.amoro.server.table.TableManager;
+import org.apache.amoro.table.TableIdentifier;
+import org.apache.amoro.table.descriptor.OptimizingProcessInfo;
+import org.apache.commons.lang3.tuple.Pair;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+class TestTableProcessQuery extends PersistentBase {
+ private final Persistence persistence = new Persistence();
+ private final ServerTableIdentifier table =
+ ServerTableIdentifier.of(1L, "catalog", "db", "tbl", TableFormat.PAIMON);
+ private final String action = PaimonActions.EXPIRE_SNAPSHOTS.getName();
+
+ @AfterEach
+ void cleanup() {
+ persistence.clear();
+ }
+
+ @Test
+ void cleanupQueryPreservesResultsAndLimitsEveryQueryToTargetTable() {
+ doAs(TableMetaMapper.class, mapper -> mapper.insertTable(table));
+ insert(table.getId(), 11, action, ProcessStatus.SUCCESS);
+ insert(table.getId(), 12, action, ProcessStatus.FAILED);
+ insert(table.getId(), 13, "MINOR", ProcessStatus.SUCCESS);
+ insert(table.getId() + 1, 14, action, ProcessStatus.SUCCESS);
+ TableProcessQuery query = new TableProcessQuery();
+ Pair, Integer> page =
+ query.list(table, List.of(action), " ", null, 1, 0);
+ Assertions.assertEquals(2, page.getRight());
+ OptimizingProcessInfo result = page.getLeft().get(0);
+ Assertions.assertEquals("12", result.getProcessId());
+ Assertions.assertEquals("catalog", result.getCatalogName());
+ Assertions.assertEquals("db", result.getDbName());
+ Assertions.assertEquals("tbl", result.getTableName());
+ Assertions.assertEquals(Map.of("outcome", "noop"), result.getSummary());
+ Assertions.assertEquals(0, result.getTotalTasks());
+ Assertions.assertNotNull(result.getInputFiles());
+ Assertions.assertNotNull(result.getOutputFiles());
+ Assertions.assertEquals(
+ "11",
+ query
+ .list(table, List.of(action), null, ProcessStatus.SUCCESS, 10, 0)
+ .getLeft()
+ .get(0)
+ .getProcessId());
+ Assertions.assertEquals(
+ 0, query.list(table, Collections.emptyList(), null, null, 10, 0).getRight());
+ Assertions.assertEquals(
+ 0, query.list(table, List.of(action), "unknown", null, 10, 0).getRight());
+ Assertions.assertEquals(0, query.list(table, List.of(action), null, null, 10, 50).getRight());
+ }
+
+ @Test
+ void serverRoutingRetainsIcebergTypePriorityAndOptimizationMetrics() {
+ doAs(TableMetaMapper.class, mapper -> mapper.insertTable(table));
+ insert(table.getId(), 11, action, ProcessStatus.SUCCESS);
+ Map metrics =
+ Map.of(
+ MetricsSummary.INPUT_DATA_FILES,
+ "4",
+ MetricsSummary.INPUT_DATA_SIZE,
+ "40",
+ MetricsSummary.OUTPUT_DATA_FILES,
+ "1",
+ MetricsSummary.OUTPUT_DATA_SIZE,
+ "10");
+ doAs(
+ TableProcessMapper.class,
+ mapper ->
+ mapper.insertProcess(
+ table.getId(),
+ 12L,
+ "",
+ ProcessStatus.SUCCESS,
+ "MINOR",
+ "default",
+ "local",
+ 0,
+ 1000L,
+ Map.of(),
+ metrics));
+ AmoroTable> iceberg = Mockito.mock(AmoroTable.class);
+ Mockito.when(iceberg.id()).thenReturn(TableIdentifier.of("catalog", "db", "tbl"));
+ Mockito.when(iceberg.format()).thenReturn(TableFormat.ICEBERG);
+ CatalogManager catalogs = Mockito.mock(CatalogManager.class);
+ ServerCatalog catalog = Mockito.mock(ServerCatalog.class);
+ Mockito.when(catalogs.getServerCatalog("catalog")).thenReturn(catalog);
+ Mockito.doReturn(iceberg).when(catalog).loadTable("db", "tbl");
+ ServerTableDescriptor descriptor =
+ new ServerTableDescriptor(catalogs, Mockito.mock(TableManager.class), new Configurations());
+ org.apache.amoro.api.TableIdentifier id = iceberg.id().buildTableIdentifier();
+ Assertions.assertEquals(
+ "11",
+ descriptor
+ .getOptimizingProcessesInfo(id, null, "CLEANUP", null, 10, 0)
+ .getLeft()
+ .get(0)
+ .getProcessId());
+ for (String category : new String[] {"CLEANUP", "OPTIMIZING", null}) {
+ OptimizingProcessInfo minor =
+ descriptor
+ .getOptimizingProcessesInfo(id, "MINOR", category, null, 10, 0)
+ .getLeft()
+ .get(0);
+ Assertions.assertEquals("12", minor.getProcessId());
+ Assertions.assertEquals(4, minor.getInputFiles().getFileCnt());
+ Assertions.assertEquals(10L, minor.getOutputFiles().getTotalSize());
+ }
+ Assertions.assertEquals(
+ 0, descriptor.getOptimizingProcessesInfo(id, "unknown", "CLEANUP", null, 10, 0).getRight());
+ Assertions.assertEquals(
+ 0, descriptor.getOptimizingProcessesInfo(id, null, null, null, 10, 0).getRight());
+ Assertions.assertEquals(
+ 0, descriptor.getOptimizingProcessesInfo(id, null, "unknown", null, 10, 0).getRight());
+ Mockito.when(iceberg.id()).thenReturn(TableIdentifier.of("catalog", "db", "unregistered"));
+ Assertions.assertEquals(
+ 0, descriptor.getOptimizingProcessesInfo(id, null, "CLEANUP", null, 10, 0).getRight());
+ Mockito.when(catalog.loadTable("db", "tbl"))
+ .thenThrow(new IllegalStateException("table unavailable"));
+ Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> descriptor.getOptimizingProcessesInfo(id, null, "CLEANUP", null, 10, 0));
+ }
+
+ private void insert(long tableId, long processId, String type, ProcessStatus status) {
+ doAs(
+ TableProcessMapper.class,
+ mapper ->
+ mapper.insertProcess(
+ tableId,
+ processId,
+ "",
+ status,
+ type,
+ "default",
+ "local",
+ 0,
+ 1000L,
+ Collections.emptyMap(),
+ Map.of("outcome", "noop")));
+ }
+
+ private static class Persistence extends DerbyPersistence {
+ void clear() {
+ super.after();
+ }
+ }
+}
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java
new file mode 100644
index 0000000000..f53f9c8102
--- /dev/null
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.dashboard.controller;
+
+import io.javalin.http.Context;
+import org.apache.amoro.Action;
+import org.apache.amoro.Constants;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.ServerTableIdentifier;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.api.OptimizingService;
+import org.apache.amoro.client.OptimizingClientPools;
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.process.LocalExecutionEngine;
+import org.apache.amoro.process.LocalProcess;
+import org.apache.amoro.process.ProcessStatus;
+import org.apache.amoro.process.TableProcess;
+import org.apache.amoro.server.DefaultOptimizingService;
+import org.apache.amoro.server.catalog.CatalogManager;
+import org.apache.amoro.server.persistence.PersistentBase;
+import org.apache.amoro.server.persistence.TableRuntimeMeta;
+import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
+import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
+import org.apache.amoro.server.process.ProcessService;
+import org.apache.amoro.server.table.DefaultTableRuntime;
+import org.apache.amoro.server.table.DefaultTableRuntimeStore;
+import org.apache.amoro.server.table.DerbyPersistence;
+import org.apache.amoro.server.table.TableManager;
+import org.apache.amoro.server.table.TableService;
+import org.apache.amoro.shade.thrift.org.apache.thrift.TMultiplexedProcessor;
+import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TMultiplexedProtocol;
+import org.apache.amoro.shade.thrift.org.apache.thrift.server.TThreadPoolServer;
+import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TServerSocket;
+import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TSocket;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.net.ServerSocket;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Stream;
+
+/**
+ * Exercise the dashboard controller through real Thrift and the production cancellation service.
+ */
+class TestProcessCancellation extends PersistentBase {
+ private final Persistence persistence = new Persistence();
+ private final AwaitableEngine engine = new AwaitableEngine();
+ private final TableService tables = Mockito.mock(TableService.class);
+ private final TableManager tableManager = Mockito.mock(TableManager.class);
+ private final Context ctx = Mockito.mock(Context.class);
+ private final CountDownLatch entered = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+ private final CountDownLatch finished = new CountDownLatch(1);
+ private final ExecutorService workers = Executors.newFixedThreadPool(2);
+ private ServerTableIdentifier identifier;
+ private ProcessService processes;
+ private DefaultOptimizingService endpoint;
+ private TableController controller;
+ private TThreadPoolServer server;
+ private Thread serverThread;
+ private TSocket connection;
+ private OptimizingService.Client client;
+ private MockedStatic clientLookup;
+ private long processId;
+ private boolean closed;
+ private final CountDownLatch completionEntered = new CountDownLatch(1);
+ private final CountDownLatch releaseCompletion = new CountDownLatch(1);
+ private volatile boolean pauseBeforePersistence;
+ private volatile boolean pauseCallback;
+
+ @BeforeEach
+ void setup() throws Exception {
+ engine.open(Collections.emptyMap());
+ ProcessService.ExecuteEngineManager engines =
+ Mockito.mock(ProcessService.ExecuteEngineManager.class);
+ Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine));
+ processes =
+ new ProcessService(tables, Collections.emptyList(), engines) {
+ @Override
+ public TableProcess untrackTableProcessInstance(ServerTableIdentifier table, long id) {
+ TableProcess process = super.untrackTableProcessInstance(table, id);
+ if (pauseCallback && process != null) {
+ awaitCompletionRelease();
+ }
+ return process;
+ }
+ };
+ processes.getTableHandlerChain().initialize(Collections.emptyList());
+ endpoint =
+ new DefaultOptimizingService(
+ new Configurations(), null, null, tables, processes, null, null);
+ TMultiplexedProcessor processor = new TMultiplexedProcessor();
+ processor.registerProcessor(
+ Constants.THRIFT_OPTIMIZING_SERVICE_NAME, new OptimizingService.Processor<>(endpoint));
+ ServerSocket socket = new ServerSocket(0);
+ server =
+ new TThreadPoolServer(
+ new TThreadPoolServer.Args(new TServerSocket(socket))
+ .processor(processor)
+ .executorService(workers));
+ serverThread = new Thread(server::serve, "cancel-regression-thrift");
+ serverThread.start();
+ await(server::isServing);
+ connection = new TSocket("localhost", socket.getLocalPort(), 5000);
+ connection.open();
+ client =
+ new OptimizingService.Client(
+ new TMultiplexedProtocol(
+ new TBinaryProtocol(connection), Constants.THRIFT_OPTIMIZING_SERVICE_NAME));
+ // Replace only client discovery; the controller uses a real transport and service
+ // implementation.
+ clientLookup = Mockito.mockStatic(OptimizingClientPools.class);
+ clientLookup
+ .when(() -> OptimizingClientPools.getClient(Mockito.anyString()))
+ .thenReturn(client);
+ CatalogManager catalogs = Mockito.mock(CatalogManager.class);
+ Mockito.when(catalogs.catalogExist("catalog")).thenReturn(true);
+ Mockito.when(tableManager.getServerTableIdentifier(Mockito.any()))
+ .thenAnswer(call -> identifier);
+ Mockito.when(tableManager.getTableProcessMeta(Mockito.anyLong()))
+ .thenAnswer(
+ call ->
+ getAs(
+ TableProcessMapper.class,
+ mapper -> mapper.getProcessMeta(call.getArgument(0))));
+ controller = new TableController(catalogs, tableManager, null, new Configurations());
+ Mockito.when(ctx.pathParam("catalog")).thenReturn("catalog");
+ Mockito.when(ctx.pathParam("db")).thenReturn("db");
+ Mockito.when(ctx.pathParam("table")).thenReturn("tbl");
+ }
+
+ @AfterEach
+ void close() throws Exception {
+ close(10, TimeUnit.SECONDS);
+ }
+
+ private void close(long timeout, TimeUnit unit) throws Exception {
+ if (closed) {
+ return;
+ }
+ release.countDown();
+ if (clientLookup != null) {
+ clientLookup.close();
+ clientLookup = null;
+ }
+ if (connection != null) {
+ connection.close();
+ }
+ if (server != null) {
+ server.stop();
+ }
+ workers.shutdownNow();
+ if (serverThread != null) {
+ serverThread.join(5000);
+ Assertions.assertFalse(serverThread.isAlive(), "Thrift server did not terminate");
+ }
+ Assertions.assertTrue(
+ workers.awaitTermination(10, TimeUnit.SECONDS), "Thrift workers did not terminate");
+ if (endpoint != null) {
+ endpoint.dispose();
+ }
+ if (processes != null) {
+ processes.dispose();
+ }
+ engine.close();
+ engine.awaitWorkers();
+ if (processes != null) {
+ Assertions.assertTrue(
+ processes.awaitTermination(timeout, unit),
+ "Process callbacks did not terminate; Derby must remain available");
+ }
+ // A timed-out or interrupted drain leaves persistence intact so teardown can be retried.
+ persistence.clear();
+ closed = true;
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void dashboardCancelsCleanupAndRejectsRepeatedCancellation(TableFormat format) throws Exception {
+ startCleanup(format);
+ controller.cancelOptimizingProcess(ctx);
+ Assertions.assertEquals(ProcessStatus.CANCELED, status());
+ Assertions.assertTrue(processes.getTableProcessInstances(identifier).isEmpty());
+ Assertions.assertThrows(
+ IllegalStateException.class, () -> controller.cancelOptimizingProcess(ctx));
+ Mockito.verify(ctx, Mockito.times(1)).json(Mockito.any());
+ Assertions.assertFalse(client.cancelProcess(Long.MAX_VALUE));
+ }
+
+ static Stream formats() {
+ return Stream.of(TableFormat.ICEBERG, TableFormat.PAIMON);
+ }
+
+ @Test
+ void dashboardRejectsCancellationForAnotherTable() throws Exception {
+ startCleanup(TableFormat.ICEBERG);
+ Mockito.when(tableManager.getServerTableIdentifier(Mockito.any()))
+ .thenReturn(
+ ServerTableIdentifier.of(
+ identifier.getId() + 1, "catalog", "db", "other", TableFormat.ICEBERG));
+ Assertions.assertThrows(
+ IllegalStateException.class, () -> controller.cancelOptimizingProcess(ctx));
+ Assertions.assertEquals(ProcessStatus.RUNNING, status());
+ Mockito.verify(ctx, Mockito.never()).json(Mockito.any());
+ Assertions.assertTrue(client.cancelProcess(processId));
+ }
+
+ @Test
+ void completedCleanupIsNotReportedAsCanceled() throws Exception {
+ startCleanup(TableFormat.ICEBERG);
+ release.countDown();
+ await(() -> status() == ProcessStatus.SUCCESS);
+ Assertions.assertThrows(
+ IllegalStateException.class, () -> controller.cancelOptimizingProcess(ctx));
+ Assertions.assertEquals(ProcessStatus.SUCCESS, status());
+ Mockito.verify(ctx, Mockito.never()).json(Mockito.any());
+ }
+
+ @ParameterizedTest
+ @MethodSource("cleanupPhases")
+ void cleanupTimeoutPreservesPersistenceUntilCompletionExits(
+ TableFormat format, boolean beforePersistence) throws Exception {
+ startCleanup(format);
+ pauseBeforePersistence = beforePersistence;
+ pauseCallback = !beforePersistence;
+ try {
+ // Model a failure before the test reaches its explicit cancellation. Hold the real tracer
+ // either before its terminal write or in its completion callback, after it was untracked.
+ release.countDown();
+ Assertions.assertTrue(completionEntered.await(10, TimeUnit.SECONDS));
+ AssertionError timeout =
+ Assertions.assertThrows(AssertionError.class, () -> close(0, TimeUnit.NANOSECONDS));
+ Assertions.assertTrue(timeout.getMessage().contains("Process callbacks did not terminate"));
+ Assertions.assertFalse(processes.awaitTermination(0, TimeUnit.NANOSECONDS));
+ Assertions.assertEquals(
+ beforePersistence ? ProcessStatus.RUNNING : ProcessStatus.SUCCESS, status());
+ } finally {
+ releaseCompletion.countDown();
+ // Retry the same teardown only after the held worker can finish; @AfterEach is idempotent.
+ close();
+ }
+ Assertions.assertTrue(processes.awaitTermination(0, TimeUnit.NANOSECONDS));
+ Assertions.assertNull(
+ getAs(TableProcessMapper.class, mapper -> mapper.getProcessMeta(processId)));
+ }
+
+ static Stream cleanupPhases() {
+ return formats()
+ .flatMap(format -> Stream.of(Arguments.of(format, true), Arguments.of(format, false)));
+ }
+
+ private void awaitCompletionRelease() {
+ completionEntered.countDown();
+ try {
+ Assertions.assertTrue(
+ releaseCompletion.await(40, TimeUnit.SECONDS), "Completion gate timed out");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private void startCleanup(TableFormat format) throws Exception {
+ identifier = ServerTableIdentifier.of(1L, "catalog", "db", "tbl", format);
+ doAs(TableMetaMapper.class, mapper -> mapper.insertTable(identifier));
+ TableRuntimeMeta meta = new TableRuntimeMeta();
+ meta.setTableId(identifier.getId());
+ meta.setGroupName("default");
+ meta.setStatusCode(0);
+ DefaultTableRuntimeStore store =
+ new DefaultTableRuntimeStore(
+ identifier, meta, DefaultTableRuntime.REQUIRED_STATES, Collections.emptyList());
+ DefaultTableRuntime runtime = new DefaultTableRuntime(store, () -> null);
+ store.setTableRuntime(runtime);
+ Mockito.when(tables.getRuntime(identifier.getId())).thenReturn(runtime);
+ processes.register(runtime, new BlockingCleanup(runtime));
+ Assertions.assertTrue(entered.await(10, TimeUnit.SECONDS));
+ processId = processes.getTableProcessInstances(identifier).keySet().iterator().next();
+ await(() -> status() == ProcessStatus.RUNNING);
+ Mockito.when(ctx.pathParam("processId")).thenReturn(String.valueOf(processId));
+ }
+
+ private ProcessStatus status() {
+ return getAs(TableProcessMapper.class, mapper -> mapper.getProcessMeta(processId)).getStatus();
+ }
+
+ private static void await(BooleanSupplier condition) throws Exception {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15);
+ while (!condition.getAsBoolean() && System.nanoTime() < deadline) {
+ Thread.sleep(20);
+ }
+ Assertions.assertTrue(condition.getAsBoolean(), "Timed out waiting for cancellation lifecycle");
+ }
+
+ private class BlockingCleanup extends TableProcess implements LocalProcess {
+ BlockingCleanup(DefaultTableRuntime runtime) {
+ super(runtime, engine);
+ }
+
+ @Override
+ public Action getAction() {
+ return PaimonActions.EXPIRE_SNAPSHOTS;
+ }
+
+ @Override
+ public Map getProcessParameters() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map getSummary() {
+ if (pauseBeforePersistence && finished.getCount() == 0) {
+ awaitCompletionRelease();
+ }
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public void run() {
+ entered.countDown();
+ try {
+ if (!release.await(20, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Test release timed out");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ } finally {
+ finished.countDown();
+ }
+ }
+ }
+
+ private static class AwaitableEngine extends LocalExecutionEngine {
+ private final List pools = new ArrayList<>();
+
+ @Override
+ protected ThreadPoolExecutor newFixedPool(String tag, int size) {
+ ThreadPoolExecutor pool = super.newFixedPool(tag, size);
+ pools.add(pool);
+ return pool;
+ }
+
+ void awaitWorkers() throws InterruptedException {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ for (ThreadPoolExecutor pool : pools) {
+ Assertions.assertTrue(
+ pool.awaitTermination(Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS),
+ "Local workers did not terminate; Derby must remain available");
+ }
+ }
+ }
+
+ private static class Persistence extends DerbyPersistence {
+ void clear() {
+ super.after();
+ }
+ }
+}
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java b/amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java
new file mode 100644
index 0000000000..1f4b33e4d0
--- /dev/null
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.process;
+
+import org.apache.amoro.Action;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.ServerTableIdentifier;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.process.ActionCoordinator;
+import org.apache.amoro.process.LocalExecutionEngine;
+import org.apache.amoro.process.LocalProcess;
+import org.apache.amoro.process.ProcessEvent;
+import org.apache.amoro.process.ProcessStatus;
+import org.apache.amoro.process.TableProcess;
+import org.apache.amoro.server.DefaultOptimizingService;
+import org.apache.amoro.server.persistence.PersistentBase;
+import org.apache.amoro.server.persistence.TableRuntimeMeta;
+import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
+import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
+import org.apache.amoro.server.table.DefaultTableRuntime;
+import org.apache.amoro.server.table.DefaultTableRuntimeStore;
+import org.apache.amoro.server.table.DerbyPersistence;
+import org.apache.amoro.server.table.TableService;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+/** Deterministic cancellation races using the real service, store and local execution engine. */
+class TestProcessCancellationConcurrency {
+ static Stream formats() {
+ return Stream.of(TableFormat.ICEBERG, TableFormat.PAIMON);
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void cancellationBeforeIdentifierPublicationCancelsQueuedWork(TableFormat format)
+ throws Exception {
+ try (Fixture f = new Fixture(format)) {
+ Work blocker = f.work(() -> await(f.releasePool, 30));
+ f.engine.submitDirect(blocker);
+ await(blocker.started);
+ Work target = f.work(() -> {});
+ f.service.register(f.runtime, target);
+ await(f.engine.submitted);
+ Assertions.assertEquals(0, target.runs.get());
+
+ Assertions.assertTrue(f.endpoint.cancelProcess(f.service.store.getProcessId()));
+ Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus());
+ f.engine.releaseSubmit.countDown();
+ await(f.engine.canceled);
+ Assertions.assertEquals(ProcessStatus.CANCELED, f.engine.getStatus(f.engine.id));
+
+ // A marker behind the canceled task proves the queue drained without invoking its body.
+ Work marker = f.work(() -> {});
+ f.engine.submitDirect(marker);
+ f.releasePool.countDown();
+ await(marker.finished);
+ Assertions.assertEquals(0, target.runs.get());
+ Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void cancellationBetweenFailureAndRetryBecomesCanceled(TableFormat format) throws Exception {
+ try (Fixture f = new Fixture(format)) {
+ f.service.pause = Pause.AFTER_FAILURE;
+ f.engine.releaseSubmit.countDown();
+ Work target =
+ f.work(
+ () -> {
+ throw new IllegalStateException("controlled failure");
+ });
+ f.service.register(f.runtime, target);
+ await(f.service.transitionReached);
+ Assertions.assertEquals(ProcessStatus.FAILED, f.service.store.getStatus());
+
+ Assertions.assertTrue(f.endpoint.cancelProcess(f.service.store.getProcessId()));
+ Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus());
+ f.service.releaseTransition.countDown();
+ await(f.service.callbackFinished);
+ Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus());
+ Assertions.assertEquals(0, f.service.store.getRetryNumber());
+ Assertions.assertEquals(1, f.engine.submissions.get());
+ Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void rejectedCancellationDoesNotPreventRetry(TableFormat format) throws Exception {
+ try (Fixture f = new Fixture(format)) {
+ f.service.pause = Pause.AFTER_FAILURE;
+ f.engine.releaseSubmit.countDown();
+ AtomicInteger attempts = new AtomicInteger();
+ Work target =
+ f.work(
+ () -> {
+ if (attempts.getAndIncrement() == 0) {
+ throw new IllegalStateException("first attempt fails");
+ }
+ });
+ f.service.register(f.runtime, target);
+ await(f.service.transitionReached);
+ Assertions.assertFalse(
+ f.service.store.tryTransitState(
+ ProcessStatus.RUNNING,
+ ProcessEvent.CANCEL_REQUESTED,
+ f.service.store.getExternalProcessIdentifier(),
+ "invalid target",
+ Collections.emptyMap(),
+ Collections.emptyMap()));
+ Assertions.assertEquals(ProcessStatus.FAILED, f.service.store.getStatus());
+ Assertions.assertFalse(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ f.service.releaseTransition.countDown();
+ await(f.service.successRecorded);
+ await(f.service.callbackFinished);
+ Assertions.assertEquals(ProcessStatus.SUCCESS, f.service.store.getStatus());
+ Assertions.assertEquals(1, f.service.store.getRetryNumber());
+ Assertions.assertEquals(2, f.engine.submissions.get());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void tableRemovalBetweenFailureAndRetryCancels(TableFormat format) throws Exception {
+ try (Fixture f = new Fixture(format)) {
+ f.service.pause = Pause.AFTER_FAILURE;
+ f.engine.releaseSubmit.countDown();
+ Work target =
+ f.work(
+ () -> {
+ throw new IllegalStateException("controlled failure");
+ });
+ f.service.register(f.runtime, target);
+ await(f.service.transitionReached);
+ f.service.getTableHandlerChain().fireTableRemoved(f.runtime);
+ Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus());
+ f.service.releaseTransition.countDown();
+ await(f.service.callbackFinished);
+ Assertions.assertEquals(0, f.service.store.getRetryNumber());
+ Assertions.assertEquals(1, f.engine.submissions.get());
+ Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void completionWinsCancellation(TableFormat format) throws Exception {
+ try (Fixture f = new Fixture(format)) {
+ f.service.pause = Pause.AFTER_SUCCESS;
+ f.engine.releaseSubmit.countDown();
+ f.service.register(f.runtime, f.work(() -> {}));
+ await(f.service.transitionReached);
+ Assertions.assertFalse(f.endpoint.cancelProcess(f.service.store.getProcessId()));
+ Assertions.assertEquals(ProcessStatus.SUCCESS, f.service.store.getStatus());
+ Assertions.assertFalse(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ f.service.releaseTransition.countDown();
+ await(f.service.callbackFinished);
+ Assertions.assertEquals(ProcessStatus.SUCCESS, f.service.store.getStatus());
+ Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("formats")
+ void cancellationWinsCompletion(TableFormat format) throws Exception {
+ try (Fixture f = new Fixture(format)) {
+ f.service.pause = Pause.BEFORE_SUCCESS;
+ f.engine.releaseSubmit.countDown();
+ f.service.register(f.runtime, f.work(() -> {}));
+ await(f.service.transitionReached);
+ Assertions.assertTrue(f.endpoint.cancelProcess(f.service.store.getProcessId()));
+ f.service.releaseTransition.countDown();
+ await(f.service.callbackFinished);
+ Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus());
+ Assertions.assertEquals(0, f.service.store.getRetryNumber());
+ Assertions.assertEquals(1, f.engine.submissions.get());
+ Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty());
+ }
+ }
+
+ private enum Pause {
+ NONE,
+ AFTER_FAILURE,
+ BEFORE_SUCCESS,
+ AFTER_SUCCESS
+ }
+
+ private static void await(CountDownLatch latch) {
+ await(latch, 10);
+ }
+
+ private static void await(CountDownLatch latch, int seconds) {
+ try {
+ Assertions.assertTrue(
+ latch.await(seconds, TimeUnit.SECONDS), "Timed out waiting for test gate");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static class Fixture extends PersistentBase implements AutoCloseable {
+ final Persistence persistence = new Persistence();
+ final CountDownLatch releasePool = new CountDownLatch(1);
+ final GatedEngine engine = new GatedEngine();
+ final ServerTableIdentifier identifier;
+ final DefaultTableRuntime runtime;
+ final ControlledService service;
+ final DefaultOptimizingService endpoint;
+
+ Fixture(TableFormat format) throws Exception {
+ identifier = ServerTableIdentifier.of(1L, "cancel-races", "db", "tbl", format);
+ doAs(TableMetaMapper.class, mapper -> mapper.insertTable(identifier));
+ TableRuntimeMeta meta = new TableRuntimeMeta();
+ meta.setTableId(identifier.getId());
+ meta.setGroupName("default");
+ meta.setStatusCode(0);
+ DefaultTableRuntimeStore state =
+ new DefaultTableRuntimeStore(
+ identifier, meta, DefaultTableRuntime.REQUIRED_STATES, Collections.emptyList());
+ runtime = new DefaultTableRuntime(state, () -> null);
+ state.setTableRuntime(runtime);
+ TableService tables = Mockito.mock(TableService.class);
+ Mockito.when(tables.getRuntime(identifier.getId())).thenReturn(runtime);
+ engine.open(Map.of("pool.default.thread-count", "1"));
+ ProcessService.ExecuteEngineManager engines =
+ Mockito.mock(ProcessService.ExecuteEngineManager.class);
+ Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine));
+ ActionCoordinator coordinator =
+ new MockActionCoordinator(engine) {
+ @Override
+ public Action action() {
+ return PaimonActions.EXPIRE_SNAPSHOTS;
+ }
+
+ @Override
+ public int parallelism() {
+ return 1;
+ }
+ };
+ service = new ControlledService(tables, List.of(coordinator), engines);
+ service.getTableHandlerChain().initialize(Collections.emptyList());
+ endpoint =
+ new DefaultOptimizingService(
+ new Configurations(), null, null, tables, service, null, null);
+ }
+
+ Work work(Runnable action) {
+ return new Work(runtime, engine, action);
+ }
+
+ @Override
+ public void close() {
+ engine.releaseSubmit.countDown();
+ service.releaseTransition.countDown();
+ if (service.store != null) {
+ endpoint.cancelProcess(service.store.getProcessId());
+ }
+ releasePool.countDown();
+ Work marker = work(() -> {});
+ engine.submitDirect(marker);
+ await(marker.finished);
+ service
+ .getActionCoordinators()
+ .values()
+ .forEach(schedulers -> schedulers.forEach(ActionCoordinatorScheduler::dispose));
+ endpoint.dispose();
+ service.dispose();
+ engine.close();
+ persistence.clear();
+ }
+ }
+
+ private static class ControlledService extends ProcessService {
+ volatile DefaultTableProcessStore store;
+ volatile Pause pause = Pause.NONE;
+ final AtomicBoolean paused = new AtomicBoolean();
+ final CountDownLatch transitionReached = new CountDownLatch(1);
+ final CountDownLatch releaseTransition = new CountDownLatch(1);
+ final CountDownLatch completionObserved = new CountDownLatch(1);
+ final CountDownLatch callbackFinished = new CountDownLatch(1);
+ final CountDownLatch successRecorded = new CountDownLatch(1);
+
+ ControlledService(
+ TableService tables, List coordinators, ExecuteEngineManager engines) {
+ super(tables, coordinators, engines);
+ }
+
+ @Override
+ public TableProcess untrackTableProcessInstance(ServerTableIdentifier table, long processId) {
+ TableProcess process = super.untrackTableProcessInstance(table, processId);
+ if (completionObserved.getCount() == 0) {
+ callbackFinished.countDown();
+ }
+ return process;
+ }
+
+ void pauseTransition() {
+ if (paused.compareAndSet(false, true)) {
+ transitionReached.countDown();
+ await(releaseTransition);
+ }
+ }
+
+ @Override
+ protected DefaultTableProcessStore persistTableProcess(TableProcess process) {
+ DefaultTableProcessStore initial = super.persistTableProcess(process);
+ TableProcessMeta meta =
+ getAs(TableProcessMapper.class, mapper -> mapper.getProcessMeta(initial.getProcessId()));
+ store =
+ new DefaultTableProcessStore(
+ initial.getProcessId(),
+ process.getTableRuntime(),
+ meta,
+ process.getAction(),
+ ActionCoordinatorScheduler.PROCESS_MAX_RETRY_NUMBER) {
+ @Override
+ public boolean tryTransitState(
+ ProcessStatus status,
+ ProcessEvent event,
+ String external,
+ String reason,
+ Map parameters,
+ Map summary) {
+ if (pause == Pause.BEFORE_SUCCESS && status == ProcessStatus.SUCCESS) {
+ pauseTransition();
+ }
+ boolean accepted =
+ super.tryTransitState(status, event, external, reason, parameters, summary);
+ if (accepted
+ && ((pause == Pause.AFTER_FAILURE && status == ProcessStatus.FAILED)
+ || (pause == Pause.AFTER_SUCCESS && status == ProcessStatus.SUCCESS))) {
+ pauseTransition();
+ }
+ if (event == ProcessEvent.COMPLETE_SUCCESS || event == ProcessEvent.COMPLETE_FAILED) {
+ completionObserved.countDown();
+ }
+ if (accepted && status == ProcessStatus.SUCCESS) {
+ successRecorded.countDown();
+ }
+ return accepted;
+ }
+ };
+ return store;
+ }
+ }
+
+ private static class GatedEngine extends LocalExecutionEngine {
+ final CountDownLatch submitted = new CountDownLatch(1);
+ final CountDownLatch releaseSubmit = new CountDownLatch(1);
+ final CountDownLatch canceled = new CountDownLatch(1);
+ volatile String id;
+ final AtomicInteger submissions = new AtomicInteger();
+
+ String submitDirect(TableProcess process) {
+ return super.submitTableProcess(process);
+ }
+
+ @Override
+ public String submitTableProcess(TableProcess process) {
+ submissions.incrementAndGet();
+ id = super.submitTableProcess(process);
+ submitted.countDown();
+ await(releaseSubmit);
+ return id;
+ }
+
+ @Override
+ public ProcessStatus tryCancelTableProcess(TableProcess process, String identifier) {
+ ProcessStatus status = super.tryCancelTableProcess(process, identifier);
+ if (status == ProcessStatus.CANCELED) {
+ canceled.countDown();
+ }
+ return status;
+ }
+ }
+
+ private static class Work extends TableProcess implements LocalProcess {
+ final AtomicInteger runs = new AtomicInteger();
+ final CountDownLatch started = new CountDownLatch(1);
+ final CountDownLatch finished = new CountDownLatch(1);
+ final Runnable action;
+
+ Work(TableRuntime runtime, LocalExecutionEngine engine, Runnable action) {
+ super(runtime, engine);
+ this.action = action;
+ }
+
+ @Override
+ public Action getAction() {
+ return PaimonActions.EXPIRE_SNAPSHOTS;
+ }
+
+ @Override
+ public Map getProcessParameters() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map getSummary() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public void run() {
+ runs.incrementAndGet();
+ started.countDown();
+ try {
+ action.run();
+ } finally {
+ finished.countDown();
+ }
+ }
+ }
+
+ private static class Persistence extends DerbyPersistence {
+ void clear() {
+ super.after();
+ }
+ }
+}
diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java
new file mode 100644
index 0000000000..da71e8490d
--- /dev/null
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.table;
+
+import org.apache.amoro.IcebergActions;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.process.LocalExecutionEngine;
+import org.apache.amoro.server.process.iceberg.IcebergProcessFactory;
+import org.apache.amoro.server.table.cleanup.TableRuntimeCleanupState;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class TestTableExpirationConfiguration {
+ @ParameterizedTest
+ @CsvSource({
+ "omitted,omitted,true",
+ "true,omitted,true",
+ "false,omitted,false",
+ "omitted,false,false",
+ "true,false,false",
+ "false,true,false"
+ })
+ void icebergExpirationKeepsExistingTableSwitchAndGcGate(
+ String tableExpire, String gc, boolean expected) {
+ Map properties = new HashMap<>();
+ if (!"omitted".equals(tableExpire)) {
+ properties.put("table-expire.enabled", tableExpire);
+ }
+ if (!"omitted".equals(gc)) {
+ properties.put("gc.enabled", gc);
+ }
+ TableRuntime runtime = Mockito.mock(TableRuntime.class);
+ Mockito.when(runtime.getTableConfiguration())
+ .thenReturn(TableConfigurations.parseTableConfig(properties));
+ Mockito.when(runtime.getState(DefaultTableRuntime.CLEANUP_STATE_KEY))
+ .thenReturn(new TableRuntimeCleanupState());
+ IcebergProcessFactory factory = new IcebergProcessFactory();
+ factory.open(Map.of("expire-snapshots.interval", "1h"));
+ factory.availableExecuteEngines(List.of(Mockito.mock(LocalExecutionEngine.class)));
+ Assertions.assertEquals(
+ expected, factory.trigger(runtime, IcebergActions.EXPIRE_SNAPSHOTS).isPresent());
+ }
+}
diff --git a/amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java
new file mode 100644
index 0000000000..23aba1cd30
--- /dev/null
+++ b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.process;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+
+/** Explicitly launched failure fixture; its name excludes it from Surefire test discovery. */
+class PaimonWarehouseLifecycle extends TestPaimonProcessIntegration {
+ static PaimonWarehouseLifecycle instance;
+ final CountDownLatch release = new CountDownLatch(1);
+ final AtomicReference worker = new AtomicReference<>();
+
+ PaimonWarehouseLifecycle() {
+ instance = this;
+ }
+
+ @Test
+ void schedulerTimeout() throws Exception {
+ startSchedulerWithBlockedRead(release, worker);
+ // Return normally: failure must come from the inherited, real @AfterEach cleanup.
+ }
+
+ @Test
+ void bodyFailure() {
+ Assertions.fail("simulated test failure");
+ }
+}
diff --git a/amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java
new file mode 100644
index 0000000000..991fa16b45
--- /dev/null
+++ b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java
@@ -0,0 +1,1036 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.process;
+
+import org.apache.amoro.Action;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.ServerTableIdentifier;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.formats.paimon.PaimonCatalog;
+import org.apache.amoro.formats.paimon.PaimonCatalogFactory;
+import org.apache.amoro.formats.paimon.process.PaimonProcessFactory;
+import org.apache.amoro.formats.paimon.process.PaimonSnapshotsExpiringProcess;
+import org.apache.amoro.process.ActionCoordinator;
+import org.apache.amoro.process.LocalExecutionEngine;
+import org.apache.amoro.process.ProcessFactory;
+import org.apache.amoro.process.ProcessStatus;
+import org.apache.amoro.process.TableProcess;
+import org.apache.amoro.server.DefaultOptimizingService;
+import org.apache.amoro.server.catalog.CatalogManager;
+import org.apache.amoro.server.catalog.ServerCatalog;
+import org.apache.amoro.server.dashboard.ServerTableDescriptor;
+import org.apache.amoro.server.manager.AbstractPluginManager;
+import org.apache.amoro.server.manager.PluginConfiguration;
+import org.apache.amoro.server.persistence.PersistentBase;
+import org.apache.amoro.server.persistence.TableRuntimeMeta;
+import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
+import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
+import org.apache.amoro.server.table.DefaultActionCoordinator;
+import org.apache.amoro.server.table.DefaultTableRuntimeFactory;
+import org.apache.amoro.server.table.DefaultTableRuntimeStore;
+import org.apache.amoro.server.table.DerbyPersistence;
+import org.apache.amoro.server.table.TableManager;
+import org.apache.amoro.server.table.TableService;
+import org.apache.amoro.shade.guava32.com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.amoro.table.TableRuntimeFactory;
+import org.apache.amoro.table.descriptor.OptimizingProcessInfo;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.CloseableIterator;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.platform.engine.discovery.DiscoverySelectors;
+import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
+import org.junit.platform.launcher.core.LauncherFactory;
+import org.junit.platform.launcher.listeners.SummaryGeneratingListener;
+import org.mockito.Mockito;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+class TestPaimonProcessIntegration extends PersistentBase {
+ private Path warehouse;
+ private final Persistence persistence = new Persistence();
+ private final ServerTableIdentifier identifier =
+ ServerTableIdentifier.of(1L, "catalog", "db", "tbl", TableFormat.PAIMON);
+ private final Clock clock = Mockito.mock(Clock.class);
+ private final ObservedFactory factory = new ObservedFactory(clock);
+ private final AwaitableEngine engine = new AwaitableEngine();
+ private Catalog nativeCatalog;
+ private PaimonCatalog catalog;
+ private TableRuntime runtime;
+ private AwaitableService service;
+ private ServerTableDescriptor descriptor;
+ private CountDownLatch loadEntered;
+ private CountDownLatch releaseLoad;
+ private boolean gateLocalLoad;
+ private TableService scheduledTables;
+ private CountDownLatch nativeReadEntered;
+ private CountDownLatch releaseNativeRead;
+ private boolean closed;
+ private Duration schedulerWaitTimeout = Duration.ofSeconds(40);
+ private ScheduledExecutorService continuousWriter;
+ private ScheduledFuture> continuousWrites;
+ private final CompletableFuture resourcesCleared = new CompletableFuture<>();
+
+ @BeforeEach
+ void setup() throws Exception {
+ // JUnit TempDir cleanup runs even after a failed teardown, while workers may still be alive.
+ warehouse = Files.createTempDirectory("amoro-paimon-");
+ Mockito.when(clock.millis()).thenAnswer(call -> System.currentTimeMillis());
+ nativeCatalog =
+ PaimonCatalogFactory.paimonCatalog(
+ Map.of("warehouse", warehouse.toString()), new Configuration());
+ nativeCatalog.createDatabase("db", false);
+ nativeCatalog.createTable(
+ Identifier.create("db", "tbl"),
+ Schema.newBuilder().column("id", DataTypes.INT()).option("write-only", "true").build(),
+ false);
+ catalog = new PaimonCatalog(nativeCatalog, "catalog");
+ doAs(TableMetaMapper.class, mapper -> mapper.insertTable(identifier));
+ engine.open(Map.of("pool.paimon-expire-snapshots.thread-count", "1"));
+ factory.open(Map.of("expire-snapshots.interval", "1h"));
+ factory.availableExecuteEngines(List.of(engine));
+ DefaultTableRuntimeFactory runtimeFactory = new DefaultTableRuntimeFactory();
+ runtimeFactory.initialize(List.of(factory));
+ runtimeFactory.withTableLoader(
+ id -> {
+ if (releaseLoad != null
+ && (!gateLocalLoad || Thread.currentThread().getName().startsWith("local-"))) {
+ loadEntered.countDown();
+ try {
+ if (!releaseLoad.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting for test load gate");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+ // The regression owns this gate and releases it in finally; it models slow native IO.
+ if (nativeReadEntered != null && Thread.currentThread().getName().startsWith("local-")) {
+ nativeReadEntered.countDown();
+ awaitGate(releaseNativeRead);
+ }
+ return catalog.loadTable(id.getDatabase(), id.getTableName());
+ });
+ TableRuntimeFactory.TableRuntimeCreator creator =
+ runtimeFactory.accept(identifier, Map.of()).orElseThrow();
+ TableRuntimeMeta meta = new TableRuntimeMeta();
+ meta.setTableId(identifier.getId());
+ meta.setGroupName("default");
+ meta.setStatusCode(0);
+ DefaultTableRuntimeStore store =
+ new DefaultTableRuntimeStore(
+ identifier, meta, creator.requiredStateKeys(), Collections.emptyList());
+ runtime = creator.create(store);
+ store.setTableRuntime(runtime);
+ ProcessService.ExecuteEngineManager engines =
+ Mockito.mock(ProcessService.ExecuteEngineManager.class);
+ Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine));
+ service =
+ new AwaitableService(
+ Mockito.mock(TableService.class), runtimeFactory.supportedCoordinators(), engines);
+ // Initialize lifecycle without scheduling automatic checks over test data.
+ service.getTableHandlerChain().initialize(Collections.emptyList());
+ CatalogManager manager = Mockito.mock(CatalogManager.class);
+ ServerCatalog serverCatalog = Mockito.mock(ServerCatalog.class);
+ Mockito.when(manager.getServerCatalog("catalog")).thenReturn(serverCatalog);
+ Mockito.when(serverCatalog.loadTable("db", "tbl"))
+ .thenAnswer(call -> catalog.loadTable("db", "tbl"));
+ descriptor =
+ new ServerTableDescriptor(manager, Mockito.mock(TableManager.class), new Configurations());
+ }
+
+ @AfterEach
+ synchronized void close() throws Exception {
+ if (closed) {
+ return;
+ }
+ if (scheduledTables != null) {
+ Mockito.when(scheduledTables.contains(identifier.getId())).thenReturn(false);
+ }
+ factory.stopObserving();
+ if (service != null) {
+ service
+ .getActionCoordinators()
+ .values()
+ .forEach(schedulers -> schedulers.forEach(ActionCoordinatorScheduler::dispose));
+ }
+ stopContinuousWriter();
+ if (releaseLoad != null) {
+ releaseLoad.countDown();
+ }
+ factory.awaitSchedulerWorkers(schedulerWaitTimeout);
+ if (service != null) {
+ // Keep the executor accepting bounded failure retries until its completion callback untracks.
+ await(() -> service.getTableProcessInstances(identifier).isEmpty());
+ service.dispose();
+ Assertions.assertTrue(
+ service.awaitTermination(40, TimeUnit.SECONDS),
+ "Process callbacks did not terminate; catalog and Derby must remain available");
+ }
+ engine.close();
+ // Cancellation completes the engine Future before an already-running body necessarily exits.
+ engine.awaitWorkers();
+ if (nativeCatalog != null) {
+ nativeCatalog.close();
+ }
+ persistence.clear();
+ if (warehouse != null) {
+ FileUtils.deleteDirectory(warehouse.toFile());
+ }
+ closed = true;
+ resourcesCleared.complete(null);
+ }
+
+ @Test
+ void realFrameworkPersistsPaimonResultsAndServesCleanupQueries() throws Exception {
+ service.register(
+ runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow());
+ await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.SUCCESS);
+ OptimizingProcessInfo info = rows().get(0);
+ Assertions.assertEquals("noop", info.getSummary().get("outcome"));
+ Assertions.assertEquals("0", info.getSummary().get("reported-expired-snapshots"));
+ Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0);
+ Assertions.assertEquals(
+ Map.of(PaimonActions.EXPIRE_SNAPSHOTS.getName(), "Expire Snapshots"),
+ descriptor.getTableProcessTypes(
+ identifier.getIdentifier().buildTableIdentifier(), "CLEANUP"));
+ Assertions.assertEquals(0, info.getTotalTasks());
+ Assertions.assertNotNull(info.getInputFiles());
+ await(() -> service.getTableProcessInstances(identifier).isEmpty());
+ }
+
+ @Test
+ void independentChangelogTableExpiresThroughProcessService() throws Exception {
+ nativeCatalog.alterTable(
+ Identifier.create("db", "tbl"),
+ List.of(
+ SchemaChange.setOption("snapshot.num-retained.min", "1"),
+ SchemaChange.setOption("snapshot.num-retained.max", "2"),
+ SchemaChange.setOption("snapshot.time-retained", "100 d"),
+ SchemaChange.setOption("changelog.num-retained.max", "5")),
+ false);
+ FileStoreTable table = (FileStoreTable) nativeCatalog.getTable(Identifier.create("db", "tbl"));
+ for (int id = 1; id <= 8; id++) {
+ append(table, id);
+ }
+ service.register(
+ runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow());
+ await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.SUCCESS);
+ OptimizingProcessInfo info = rows().get(0);
+ Assertions.assertEquals("executed", info.getSummary().get("outcome"));
+ Assertions.assertEquals("6", info.getSummary().get("reported-expired-snapshots"));
+ Assertions.assertEquals("5", info.getSummary().get("changelog.num-retained.max"));
+ Assertions.assertEquals(7L, table.snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals(6, table.changelogManager().safelyGetAllChangelogs().size());
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6, 7, 8), readIds(table));
+ Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0);
+ await(() -> service.getTableProcessInstances(identifier).isEmpty());
+ }
+
+ @Test
+ void failureSummaryIsPersistedAndRetriesAreBounded() throws Exception {
+ nativeCatalog.alterTable(
+ Identifier.create("db", "tbl"),
+ SchemaChange.setOption("snapshot.expire.limit", "0"),
+ false);
+ service.register(
+ runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow());
+ await(() -> !rows().isEmpty() && service.getTableProcessInstances(identifier).isEmpty());
+ Assertions.assertEquals(1, rows().size());
+ OptimizingProcessInfo info = rows().get(0);
+ Assertions.assertEquals(ProcessStatus.FAILED, info.getStatus());
+ Assertions.assertEquals("config", info.getSummary().get("error-stage"));
+ Assertions.assertNotNull(info.getSummary().get("error-message"));
+ Assertions.assertFalse(info.getSummary().containsKey("outcome"));
+ Assertions.assertEquals(
+ 3,
+ getAs(
+ TableProcessMapper.class,
+ mapper -> mapper.getProcessMeta(Long.parseLong(info.getProcessId())))
+ .getRetryNumber());
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ }
+
+ @Test
+ void recoveredLostLocalTaskRunsWithCurrentSettings() throws Exception {
+ doAs(
+ TableProcessMapper.class,
+ mapper ->
+ mapper.insertProcess(
+ identifier.getId(),
+ 42L,
+ "lost-local-process",
+ ProcessStatus.RUNNING,
+ PaimonActions.EXPIRE_SNAPSHOTS.getName(),
+ "default",
+ "local",
+ 0,
+ System.currentTimeMillis(),
+ Map.of(),
+ Map.of()));
+ nativeCatalog.alterTable(
+ Identifier.create("db", "tbl"),
+ SchemaChange.setOption("amoro.expire-snapshots.enabled", "false"),
+ false);
+ service.recoverProcesses(List.of(runtime));
+ await(() -> rows().get(0).getStatus() == ProcessStatus.SUCCESS);
+ Assertions.assertEquals("42", rows().get(0).getProcessId());
+ Assertions.assertEquals("skipped", rows().get(0).getSummary().get("outcome"));
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ await(() -> service.getTableProcessInstances(identifier).isEmpty());
+ }
+
+ @Test
+ void pluginDefaultsDoNotEnableMaintenanceAndExplicitConfigAdmitsRuntime() throws Exception {
+ for (List configs :
+ List.of(
+ Collections.emptyList(),
+ List.of(
+ new PluginConfiguration(
+ "paimon", false, 100, Map.of("expire-snapshots.interval", "1h"))),
+ List.of(PluginConfiguration.emptyConfig("paimon")),
+ List.of(
+ new PluginConfiguration(
+ "paimon",
+ true,
+ 100,
+ Map.of(
+ "expire-snapshots.enabled",
+ "false",
+ "expire-snapshots.interval",
+ "1h"))))) {
+ try (Plugins manager = new Plugins(configs)) {
+ manager.initialize();
+ DefaultTableRuntimeFactory runtimes = new DefaultTableRuntimeFactory();
+ runtimes.initialize(manager.installedPlugins());
+ Assertions.assertTrue(runtimes.accept(identifier, Map.of()).isEmpty());
+ }
+ }
+ PluginConfiguration enabled =
+ PluginConfiguration.fromJSONObject(
+ new ObjectMapper()
+ .readTree(
+ "{\"name\":\"paimon\",\"properties\":{\"expire-snapshots.interval\":\"1h\"}}"));
+ try (Plugins manager = new Plugins(List.of(enabled))) {
+ manager.initialize();
+ DefaultTableRuntimeFactory runtimes = new DefaultTableRuntimeFactory();
+ runtimes.initialize(manager.installedPlugins());
+ Assertions.assertTrue(runtimes.accept(identifier, Map.of()).isPresent());
+ Assertions.assertEquals(1, runtimes.supportedCoordinators().size());
+ }
+ }
+
+ @Test
+ void cancelServiceEntryCancelsPaimonCleanup() throws Exception {
+ TableProcess process = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow();
+ loadEntered = new CountDownLatch(1);
+ releaseLoad = new CountDownLatch(1);
+ TableService tables = Mockito.mock(TableService.class);
+ Mockito.when(tables.getRuntime(identifier.getId())).thenReturn(runtime);
+ DefaultOptimizingService endpoint =
+ new DefaultOptimizingService(new Configurations(), null, null, tables, service, null, null);
+ try {
+ service.register(runtime, process);
+ Assertions.assertTrue(loadEntered.await(10, TimeUnit.SECONDS));
+ await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.RUNNING);
+ long processId = Long.parseLong(rows().get(0).getProcessId());
+ Assertions.assertTrue(endpoint.cancelProcess(processId));
+ Assertions.assertEquals(ProcessStatus.CANCELED, rows().get(0).getStatus());
+ Assertions.assertTrue(service.getTableProcessInstances(identifier).isEmpty());
+ Assertions.assertFalse(endpoint.cancelProcess(processId));
+ Assertions.assertFalse(endpoint.cancelProcess(Long.MAX_VALUE));
+ } finally {
+ releaseLoad.countDown();
+ await(() -> process.getSummary().containsKey("outcome"));
+ endpoint.dispose();
+ }
+ }
+
+ @Test
+ void sameActionIsNotSubmittedTwiceAndTableRemovalCancelsTrackedProcess() throws Exception {
+ TableProcess process = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow();
+ loadEntered = new CountDownLatch(1);
+ releaseLoad = new CountDownLatch(1);
+ service.register(runtime, process);
+ Assertions.assertTrue(loadEntered.await(10, TimeUnit.SECONDS));
+ service.register(runtime, new PaimonSnapshotsExpiringProcess(runtime, engine));
+ await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.RUNNING);
+ service.getTableHandlerChain().fireTableRemoved(runtime);
+ Assertions.assertEquals(ProcessStatus.CANCELED, rows().get(0).getStatus());
+ Assertions.assertTrue(service.getTableProcessInstances(identifier).isEmpty());
+ releaseLoad.countDown();
+ // Cancellation is not a rollback or proof that the underlying invocation stopped.
+ await(() -> process.getSummary().containsKey("outcome"));
+ Assertions.assertEquals(ProcessStatus.CANCELED, rows().get(0).getStatus());
+ }
+
+ @Test
+ void periodicSchedulerWaitsForStartupAndSuccessInterval() throws Exception {
+ loadEntered = new CountDownLatch(1);
+ releaseLoad = new CountDownLatch(1);
+ gateLocalLoad = true;
+ factory.controlChecks = true;
+ long started = startPeriodicScheduling(1_000);
+
+ Check first = factory.nextCheck();
+ Assertions.assertTrue(first.started - started >= Duration.ofSeconds(11).toNanos());
+ Assertions.assertTrue(first.proceed().isPresent());
+ Assertions.assertTrue(loadEntered.await(10, TimeUnit.SECONDS));
+
+ // The real scheduler continues checking while the first local invocation is blocked.
+ Check second = factory.nextCheck();
+ Assertions.assertTrue(second.started - first.finished >= Duration.ofSeconds(1).toNanos());
+ Assertions.assertTrue(second.proceed().isPresent());
+ Check third = factory.nextCheck();
+ Assertions.assertEquals(1, rows().size(), "An in-flight action must not be registered twice");
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ releaseLoad.countDown();
+ await(() -> runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0);
+ long successTime = runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME);
+ Mockito.when(clock.millis()).thenReturn(successTime + 999);
+ Assertions.assertTrue(third.proceed().isEmpty(), "A check just after success must be skipped");
+
+ // Another real scheduler interval can pass while the injected time remains below the boundary.
+ Check fourth = factory.nextCheck();
+ Assertions.assertTrue(fourth.started - third.finished >= Duration.ofSeconds(1).toNanos());
+ Assertions.assertTrue(fourth.proceed().isEmpty());
+
+ // Completion tracing has its own polling interval. Let it finish before the next check.
+ await(() -> rows().get(0).getStatus() == ProcessStatus.SUCCESS);
+ await(() -> service.getTableProcessInstances(identifier).isEmpty());
+ Mockito.when(clock.millis()).thenReturn(successTime + 1_000);
+ Check fifth = factory.nextCheck();
+ Assertions.assertTrue(fifth.proceed().isPresent());
+ await(
+ () ->
+ rows().size() == 2
+ && rows().stream().allMatch(r -> r.getStatus() == ProcessStatus.SUCCESS));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"100,8", "300,12"})
+ void periodicExpirationCatchesUpWhileWritesContinue(long intervalMillis, int expireLimit)
+ throws Exception {
+ nativeCatalog.alterTable(
+ Identifier.create("db", "tbl"),
+ List.of(
+ SchemaChange.setOption("snapshot.num-retained.min", "2"),
+ SchemaChange.setOption("snapshot.num-retained.max", "2"),
+ SchemaChange.setOption("snapshot.time-retained", "100 d"),
+ SchemaChange.setOption("snapshot.expire.limit", String.valueOf(expireLimit))),
+ false);
+ FileStoreTable table = (FileStoreTable) nativeCatalog.getTable(Identifier.create("db", "tbl"));
+ append(table, 1);
+ table.createTag("first", 1);
+ for (int id = 2; id <= 24; id++) {
+ append(table, id);
+ }
+ Assertions.assertEquals(22, eligibleSnapshots(table));
+ startPeriodicScheduling(intervalMillis);
+ factory.nextCheck();
+
+ AtomicInteger committed = new AtomicInteger(24);
+ AtomicReference writeFailure = new AtomicReference<>();
+ continuousWriter = Executors.newSingleThreadScheduledExecutor();
+ continuousWrites =
+ continuousWriter.scheduleWithFixedDelay(
+ () -> {
+ try {
+ append(table, committed.get() + 1);
+ committed.incrementAndGet();
+ } catch (Throwable e) {
+ writeFailure.set(e);
+ throw new IllegalStateException("Continuous test writer failed", e);
+ }
+ },
+ 0,
+ 1,
+ TimeUnit.SECONDS);
+ try {
+ // At most one new snapshot per second. Allow for the framework's 5-second status poll,
+ // not just the much shorter coordinator interval, when choosing expiration capacity.
+ await(
+ () -> {
+ Assertions.assertNull(writeFailure.get(), "Writer must remain healthy");
+ return committed.get() >= 27 && eligibleSnapshots(table) == 0;
+ },
+ Duration.ofSeconds(90));
+ int firstCatchUp = committed.get();
+ // Catch up again after further writes: draining only the initial static backlog is not
+ // enough.
+ await(
+ () -> {
+ Assertions.assertNull(writeFailure.get(), "Writer must remain healthy");
+ return committed.get() >= firstCatchUp + 3 && eligibleSnapshots(table) == 0;
+ },
+ Duration.ofSeconds(90));
+ Assertions.assertFalse(
+ continuousWrites.isDone(), "Convergence must happen before the writer stops");
+ } finally {
+ stopContinuousWriter();
+ }
+ Assertions.assertNull(writeFailure.get());
+ Mockito.when(scheduledTables.contains(identifier.getId())).thenReturn(false);
+ await(() -> service.getTableProcessInstances(identifier).isEmpty());
+ Assertions.assertTrue(
+ table.snapshotManager().snapshotExists(table.snapshotManager().latestSnapshotId() - 1));
+ List completed = rows();
+ Assertions.assertTrue(
+ completed.size() >= 3, "Convergence must span multiple periodic processes");
+ for (OptimizingProcessInfo process : completed) {
+ Assertions.assertEquals(ProcessStatus.SUCCESS, process.getStatus());
+ int expired = Integer.parseInt(process.getSummary().get("reported-expired-snapshots"));
+ Assertions.assertTrue(expired >= 0 && expired <= expireLimit);
+ Assertions.assertEquals(
+ String.valueOf(expireLimit), process.getSummary().get("snapshot.expire.limit"));
+ }
+ Assertions.assertEquals(
+ IntStream.rangeClosed(1, committed.get()).boxed().collect(Collectors.toList()),
+ readIds(table));
+ Assertions.assertEquals(List.of(1), readIds(table.copy(Map.of("scan.tag-name", "first"))));
+ }
+
+ private void stopContinuousWriter() throws InterruptedException {
+ if (continuousWriter == null) {
+ return;
+ }
+ if (continuousWrites != null) {
+ continuousWrites.cancel(false);
+ }
+ continuousWriter.shutdown();
+ try {
+ if (!continuousWriter.awaitTermination(15, TimeUnit.SECONDS)) {
+ continuousWriter.shutdownNow();
+ Assertions.assertTrue(
+ continuousWriter.awaitTermination(40, TimeUnit.SECONDS),
+ "Continuous writer did not terminate; its table must remain available");
+ }
+ } catch (InterruptedException e) {
+ continuousWriter.shutdownNow();
+ Thread.currentThread().interrupt();
+ throw e;
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void cleanupAfterFailureWaitsForNativeWorkAndCallbacks(boolean nativeStillRunning)
+ throws Exception {
+ ExecutorService closer = Executors.newSingleThreadExecutor();
+ Future cleanup = null;
+ try {
+ if (nativeStillRunning) {
+ nativeReadEntered = new CountDownLatch(1);
+ releaseNativeRead = new CountDownLatch(1);
+ } else {
+ service.pauseCallback = true;
+ }
+ service.register(
+ runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow());
+ if (nativeStillRunning) {
+ Assertions.assertTrue(nativeReadEntered.await(10, TimeUnit.SECONDS));
+ await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.RUNNING);
+ Assertions.assertTrue(
+ service.cancelProcess(identifier, Long.parseLong(rows().get(0).getProcessId())));
+ } else {
+ Assertions.assertTrue(service.callbackEntered.await(10, TimeUnit.SECONDS));
+ }
+ // Both paths are already untracked; neither proves the native worker/callback has returned.
+ Assertions.assertTrue(service.getTableProcessInstances(identifier).isEmpty());
+ cleanup =
+ closer.submit(
+ (Callable)
+ () -> {
+ try {
+ throw new AssertionError("simulated test abort");
+ } finally {
+ close();
+ }
+ });
+ CompletableFuture draining = nativeStillRunning ? engine.draining : service.draining;
+ CompletableFuture.anyOf(draining, resourcesCleared).get(30, TimeUnit.SECONDS);
+ Assertions.assertFalse(resourcesCleared.isDone(), "Resources cleared before workers exited");
+ Assertions.assertTrue(draining.isDone());
+ Assertions.assertFalse(cleanup.isDone());
+ Assertions.assertEquals(1, rows().size(), "Derby must remain usable while work is pending");
+ } finally {
+ if (releaseNativeRead != null) {
+ releaseNativeRead.countDown();
+ }
+ service.releaseCallback.countDown();
+ try {
+ if (cleanup != null) {
+ Future result = cleanup;
+ ExecutionException failure =
+ Assertions.assertThrows(
+ ExecutionException.class, () -> result.get(40, TimeUnit.SECONDS));
+ Assertions.assertInstanceOf(AssertionError.class, failure.getCause());
+ Assertions.assertEquals("simulated test abort", failure.getCause().getMessage());
+ // Keep worker cleanup independent of the assertions above.
+ Assertions.assertTrue(service.awaitTermination(40, TimeUnit.SECONDS));
+ engine.awaitWorkers();
+ }
+ } finally {
+ closer.shutdownNow();
+ Assertions.assertTrue(closer.awaitTermination(40, TimeUnit.SECONDS));
+ }
+ }
+ Assertions.assertTrue(resourcesCleared.isDone());
+ }
+
+ @Test
+ void cleanupTimeoutPreservesResourcesUntilSchedulerExits() throws Exception {
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference schedulerThread = new AtomicReference<>();
+ catalog = Mockito.spy(catalog);
+ Mockito.doAnswer(
+ call -> {
+ if (Thread.currentThread().getName().startsWith("async-")) {
+ schedulerThread.set(Thread.currentThread());
+ entered.countDown();
+ // Model catalog IO that cannot respond to cancellation until the read completes.
+ boolean released = false;
+ while (!released) {
+ try {
+ release.await();
+ released = true;
+ } catch (InterruptedException ignored) {
+ // The regression releases the external operation in finally.
+ }
+ }
+ throw new IllegalStateException("Test catalog read released");
+ }
+ return call.callRealMethod();
+ })
+ .when(catalog)
+ .loadTable("db", "tbl");
+ try {
+ startPeriodicScheduling(100);
+ Assertions.assertTrue(entered.await(20, TimeUnit.SECONDS));
+ // Keep the production scheduler's real 30s + 30s stop path. Only the fixture's
+ // additional wait is zero, so the still-blocked worker deterministically times out.
+ schedulerWaitTimeout = Duration.ZERO;
+ AssertionError failure = Assertions.assertThrows(AssertionError.class, this::close);
+ Assertions.assertTrue(failure.getMessage().contains("Scheduler worker did not terminate"));
+ Assertions.assertFalse(resourcesCleared.isDone());
+ Assertions.assertTrue(schedulerThread.get().isAlive());
+ Assertions.assertNotNull(nativeCatalog.getTable(Identifier.create("db", "tbl")));
+ Assertions.assertEquals(
+ List.of(identifier),
+ getAs(TableMetaMapper.class, TableMetaMapper::selectAllTableIdentifiers));
+ } finally {
+ release.countDown();
+ if (schedulerThread.get() != null) {
+ schedulerThread.get().join(40_000);
+ Assertions.assertFalse(schedulerThread.get().isAlive());
+ }
+ schedulerWaitTimeout = Duration.ofSeconds(40);
+ close();
+ }
+ Assertions.assertTrue(resourcesCleared.isDone());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void junitTeardownOwnsWarehouseLifetime(boolean schedulerBlocked) throws Exception {
+ // The launched fixture needs exclusive ownership of the shared Derby tables.
+ close();
+ PaimonWarehouseLifecycle.instance = null;
+ SummaryGeneratingListener listener = new SummaryGeneratingListener();
+ try {
+ // Run a real Jupiter lifecycle. Expected inner failures are inspected here, not swallowed
+ // by a test extension before JUnit gets a chance to clean its method-scoped resources.
+ LauncherFactory.create()
+ .execute(
+ LauncherDiscoveryRequestBuilder.request()
+ .selectors(
+ DiscoverySelectors.selectMethod(
+ PaimonWarehouseLifecycle.class,
+ schedulerBlocked ? "schedulerTimeout" : "bodyFailure"))
+ .configurationParameter("junit.jupiter.execution.parallel.enabled", "false")
+ .build(),
+ listener);
+ Assertions.assertEquals(1, listener.getSummary().getTestsFoundCount());
+ Assertions.assertEquals(1, listener.getSummary().getTestsFailedCount());
+ Assertions.assertEquals(1, listener.getSummary().getFailures().size());
+ String message = listener.getSummary().getFailures().get(0).getException().getMessage();
+ Assertions.assertTrue(
+ message.contains(
+ schedulerBlocked ? "Scheduler worker did not terminate" : "simulated test failure"),
+ message);
+ PaimonWarehouseLifecycle inner = PaimonWarehouseLifecycle.instance;
+ Assertions.assertNotNull(inner);
+ TestPaimonProcessIntegration fixture = inner;
+ if (schedulerBlocked) {
+ Assertions.assertTrue(inner.worker.get().isAlive());
+ Assertions.assertFalse(fixture.resourcesCleared.isDone());
+ Assertions.assertTrue(
+ Files.exists(fixture.warehouse),
+ "JUnit removed warehouse while work was still running");
+ Assertions.assertNotNull(fixture.nativeCatalog.getTable(Identifier.create("db", "tbl")));
+ } else {
+ Assertions.assertTrue(fixture.resourcesCleared.isDone());
+ Assertions.assertFalse(Files.exists(fixture.warehouse));
+ }
+ } finally {
+ PaimonWarehouseLifecycle inner = PaimonWarehouseLifecycle.instance;
+ if (inner != null) {
+ inner.release.countDown();
+ if (inner.worker.get() != null) {
+ inner.worker.get().join(40_000);
+ Assertions.assertFalse(inner.worker.get().isAlive());
+ }
+ TestPaimonProcessIntegration fixture = inner;
+ fixture.schedulerWaitTimeout = Duration.ofSeconds(40);
+ fixture.close();
+ Assertions.assertFalse(Files.exists(fixture.warehouse));
+ }
+ PaimonWarehouseLifecycle.instance = null;
+ }
+ }
+
+ void startSchedulerWithBlockedRead(CountDownLatch release, AtomicReference worker)
+ throws Exception {
+ CountDownLatch entered = new CountDownLatch(1);
+ catalog = Mockito.spy(catalog);
+ Mockito.doAnswer(
+ call -> {
+ if (Thread.currentThread().getName().startsWith("async-")) {
+ worker.set(Thread.currentThread());
+ entered.countDown();
+ Assertions.assertTrue(
+ Uninterruptibles.awaitUninterruptibly(release, 3, TimeUnit.MINUTES),
+ "External read gate timed out");
+ throw new IllegalStateException("Test catalog read released");
+ }
+ return call.callRealMethod();
+ })
+ .when(catalog)
+ .loadTable("db", "tbl");
+ startPeriodicScheduling(100);
+ Assertions.assertTrue(entered.await(20, TimeUnit.SECONDS));
+ schedulerWaitTimeout = Duration.ZERO;
+ }
+
+ private static void awaitGate(CountDownLatch gate) {
+ try {
+ Assertions.assertTrue(gate.await(40, TimeUnit.SECONDS), "Test gate timed out");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static class AwaitableEngine extends LocalExecutionEngine {
+ private final List workers = new ArrayList<>();
+ private final CompletableFuture draining = new CompletableFuture<>();
+
+ @Override
+ protected ThreadPoolExecutor newFixedPool(String tag, int size) {
+ ThreadPoolExecutor pool = super.newFixedPool(tag, size);
+ workers.add(pool);
+ return pool;
+ }
+
+ void awaitWorkers() throws InterruptedException {
+ draining.complete(null);
+ long deadline = System.nanoTime() + Duration.ofSeconds(40).toNanos();
+ for (ThreadPoolExecutor worker : workers) {
+ Assertions.assertTrue(
+ worker.awaitTermination(
+ Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS),
+ "Local worker did not terminate; catalog and Derby must remain available");
+ }
+ }
+ }
+
+ private static class AwaitableService extends ProcessService {
+ private final CompletableFuture draining = new CompletableFuture<>();
+ private final CountDownLatch callbackEntered = new CountDownLatch(1);
+ private final CountDownLatch releaseCallback = new CountDownLatch(1);
+ private volatile boolean pauseCallback;
+
+ AwaitableService(
+ TableService tables, List coordinators, ExecuteEngineManager engines) {
+ super(tables, coordinators, engines);
+ }
+
+ @Override
+ public TableProcess untrackTableProcessInstance(ServerTableIdentifier table, long processId) {
+ TableProcess process = super.untrackTableProcessInstance(table, processId);
+ if (pauseCallback && process != null) {
+ callbackEntered.countDown();
+ awaitGate(releaseCallback);
+ }
+ return process;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ draining.complete(null);
+ return super.awaitTermination(timeout, unit);
+ }
+ }
+
+ private static long eligibleSnapshots(FileStoreTable table) {
+ // This fixture has no consumer and retains exactly the latest two snapshots. A tag protects
+ // its referenced data separately; it does not turn those two retained snapshots into backlog.
+ // Read the oldest boundary first so concurrent commits/expiration can only overestimate lag.
+ long earliest = table.snapshotManager().earliestSnapshotId();
+ return Math.max(0, table.snapshotManager().latestSnapshotId() - earliest - 1);
+ }
+
+ private static void append(FileStoreTable table, int id) throws Exception {
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite();
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(GenericRow.of(id));
+ commit.commit(write.prepareCommit());
+ }
+ }
+
+ private static List readIds(org.apache.paimon.table.Table table) throws Exception {
+ ReadBuilder builder = table.newReadBuilder();
+ List ids = new ArrayList<>();
+ try (CloseableIterator rows =
+ builder.newRead().createReader(builder.newScan().plan()).toCloseableIterator()) {
+ while (rows.hasNext()) {
+ ids.add(rows.next().getInt(0));
+ }
+ }
+ Collections.sort(ids);
+ return ids;
+ }
+
+ private long startPeriodicScheduling(long intervalMillis) throws InterruptedException {
+ service
+ .getActionCoordinators()
+ .values()
+ .forEach(schedulers -> schedulers.forEach(ActionCoordinatorScheduler::dispose));
+ service.dispose();
+ Assertions.assertTrue(service.awaitTermination(40, TimeUnit.SECONDS));
+ factory.open(Map.of("expire-snapshots.interval", intervalMillis + "ms"));
+ scheduledTables = Mockito.mock(TableService.class);
+ Mockito.when(scheduledTables.contains(identifier.getId())).thenReturn(true);
+ ProcessService.ExecuteEngineManager engines =
+ Mockito.mock(ProcessService.ExecuteEngineManager.class);
+ Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine));
+ service =
+ new AwaitableService(
+ scheduledTables,
+ List.of(
+ new DefaultActionCoordinator(
+ TableFormat.PAIMON, PaimonActions.EXPIRE_SNAPSHOTS, factory)),
+ engines);
+ factory.observing = true;
+ // Use the production startup delay and recurring scheduler, not direct trigger/register calls.
+ long started = System.nanoTime();
+ service.getTableHandlerChain().initialize(List.of(runtime));
+ return started;
+ }
+
+ private static class Check {
+ private final long started = System.nanoTime();
+ private long finished;
+ private final CountDownLatch permit = new CountDownLatch(1);
+ private final CompletableFuture> result = new CompletableFuture<>();
+
+ Optional proceed() throws Exception {
+ permit.countDown();
+ return result.get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ /** Observe the factory SPI without replacing scheduling, admission or expiration behavior. */
+ private static class ObservedFactory extends PaimonProcessFactory {
+ private final BlockingQueue checks = new LinkedBlockingQueue<>();
+ private volatile boolean observing;
+ private volatile boolean controlChecks;
+ private volatile boolean stopped;
+ private volatile Check current;
+ private final Set schedulerWorkers = new LinkedHashSet<>();
+
+ ObservedFactory(Clock clock) {
+ super(clock);
+ }
+
+ @Override
+ public Optional trigger(TableRuntime runtime, Action action) {
+ if (!observing) {
+ return super.trigger(runtime, action);
+ }
+ synchronized (this) {
+ // Close admission atomically with recording workers: a late check cannot access Catalog.
+ if (stopped) {
+ return Optional.empty();
+ }
+ schedulerWorkers.add(Thread.currentThread());
+ }
+ Check check = new Check();
+ current = check;
+ checks.add(check);
+ try {
+ if (controlChecks && !check.permit.await(30, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting for scheduled check gate");
+ }
+ Optional process =
+ stopped ? Optional.empty() : super.trigger(runtime, action);
+ check.finished = System.nanoTime();
+ check.result.complete(process);
+ return process;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ check.result.completeExceptionally(e);
+ throw new IllegalStateException(e);
+ } catch (RuntimeException e) {
+ check.result.completeExceptionally(e);
+ throw e;
+ }
+ }
+
+ Check nextCheck() throws Exception {
+ Check check = checks.poll(30, TimeUnit.SECONDS);
+ Assertions.assertNotNull(check, "The production scheduler did not run a check");
+ return check;
+ }
+
+ void awaitSchedulerWorkers(Duration timeout) throws InterruptedException {
+ List workers;
+ synchronized (this) {
+ workers = new ArrayList<>(schedulerWorkers);
+ }
+ long deadline = System.nanoTime() + timeout.toNanos();
+ for (Thread worker : workers) {
+ long remaining = deadline - System.nanoTime();
+ if (remaining > 0) {
+ TimeUnit.NANOSECONDS.timedJoin(worker, remaining);
+ }
+ // dispose() can return after logging a timeout. Only actual thread exit is sufficient;
+ // it also covers process registration and scheduler cleanup after trigger() returns.
+ Assertions.assertFalse(
+ worker.isAlive(),
+ "Scheduler worker did not terminate; catalog and Derby must remain available");
+ }
+ }
+
+ synchronized void stopObserving() {
+ // Releasing a pending check during teardown must not register another native invocation.
+ stopped = true;
+ controlChecks = false;
+ if (current != null) {
+ current.permit.countDown();
+ }
+ }
+ }
+
+ private List rows() {
+ return descriptor
+ .getOptimizingProcessesInfo(
+ identifier.getIdentifier().buildTableIdentifier(), null, "CLEANUP", null, 10, 0)
+ .getLeft();
+ }
+
+ private static void await(BooleanSupplier condition) throws Exception {
+ await(condition, Duration.ofSeconds(40));
+ }
+
+ private static void await(BooleanSupplier condition, Duration timeout) throws Exception {
+ long end = System.nanoTime() + timeout.toNanos();
+ while (System.nanoTime() < end) {
+ if (condition.getAsBoolean()) {
+ return;
+ }
+ Thread.sleep(20);
+ }
+ Assertions.fail("Timed out waiting for process lifecycle");
+ }
+
+ private static class Plugins extends AbstractPluginManager
+ implements AutoCloseable {
+ private final List configs;
+
+ Plugins(List configs) {
+ super("process-factories");
+ this.configs = configs;
+ }
+
+ @Override
+ protected List loadPluginConfigurations() {
+ return configs;
+ }
+ }
+
+ private static class Persistence extends DerbyPersistence {
+ void clear() {
+ super.after();
+ }
+ }
+}
diff --git a/amoro-common/src/main/java/org/apache/amoro/PaimonActions.java b/amoro-common/src/main/java/org/apache/amoro/PaimonActions.java
new file mode 100644
index 0000000000..fb8a40ee7f
--- /dev/null
+++ b/amoro-common/src/main/java/org/apache/amoro/PaimonActions.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro;
+
+/** Maintenance actions supported by Paimon tables. */
+public final class PaimonActions {
+ public static final Action EXPIRE_SNAPSHOTS = Action.register("expire-snapshots");
+
+ private PaimonActions() {}
+}
diff --git a/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java b/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java
index 5999de9f01..951a46bb4a 100755
--- a/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java
+++ b/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java
@@ -163,7 +163,8 @@ private ThreadPoolExecutor getPool(String tag) {
return pools.get(DEFAULT_POOL);
}
- private ThreadPoolExecutor newFixedPool(String tag, int size) {
+ /** Create a worker pool; subclasses can retain it to await termination in tests. */
+ protected ThreadPoolExecutor newFixedPool(String tag, int size) {
return new ThreadPoolExecutor(
size,
size,
diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java
index 2af18a9417..9da07674b5 100644
--- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java
+++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java
@@ -19,7 +19,6 @@
package org.apache.amoro.formats.paimon;
import org.apache.amoro.AlreadyExistsException;
-import org.apache.amoro.AmoroTable;
import org.apache.amoro.DatabaseNotEmptyException;
import org.apache.amoro.FormatCatalog;
import org.apache.amoro.NoSuchDatabaseException;
@@ -87,16 +86,22 @@ public void dropDatabase(String database) {
}
@Override
- public AmoroTable> loadTable(String database, String table) {
+ public PaimonTable loadTable(String database, String table) {
try {
return new PaimonTable(
TableIdentifier.of(name, database, table),
- catalog.getTable(Identifier.create(database, table)));
+ catalog.getTable(Identifier.create(database, table)),
+ () -> reloadTable(database, table));
} catch (Catalog.TableNotExistException e) {
throw new NoSuchTableException(e);
}
}
+ private PaimonTable reloadTable(String database, String table) {
+ catalog.invalidateTable(Identifier.create(database, table));
+ return loadTable(database, table);
+ }
+
@Override
public List listTables(String database) {
try {
diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java
index 27a5f57068..c4c6e6ea3e 100644
--- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java
+++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java
@@ -27,16 +27,31 @@
import org.apache.paimon.table.Table;
import java.util.Map;
+import java.util.function.Supplier;
public class PaimonTable implements AmoroTable {
private final TableIdentifier tableIdentifier;
private final Table table;
+ private final Supplier refresh;
public PaimonTable(TableIdentifier tableIdentifier, Table table) {
+ this(tableIdentifier, table, null);
+ }
+
+ PaimonTable(TableIdentifier tableIdentifier, Table table, Supplier refresh) {
this.tableIdentifier = tableIdentifier;
this.table = table;
+ this.refresh = refresh;
+ }
+
+ /** Reload this table through its owning catalog, invalidating any cached metadata. */
+ public PaimonTable refresh() {
+ if (refresh == null) {
+ throw new IllegalStateException("Paimon table has no catalog refresh callback");
+ }
+ return refresh.get();
}
@Override
diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java
index 7263dc8a5a..a25874b358 100644
--- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java
+++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java
@@ -21,6 +21,7 @@
import static org.apache.paimon.operation.FileStoreScan.Plan.groupByPartFiles;
import org.apache.amoro.AmoroTable;
+import org.apache.amoro.PaimonActions;
import org.apache.amoro.TableFormat;
import org.apache.amoro.api.CommitMetaProducer;
import org.apache.amoro.process.ProcessStatus;
@@ -479,8 +480,20 @@ public Map getTableOptimizingTypes(AmoroTable> amoroTable) {
return types;
}
+ @Override
+ public Map getTableProcessTypes(AmoroTable> table, String processCategory) {
+ if (ProcessCategory.CLEANUP.getName().equalsIgnoreCase(processCategory)) {
+ return Collections.singletonMap(PaimonActions.EXPIRE_SNAPSHOTS.getName(), "Expire Snapshots");
+ }
+ return FormatTableDescriptor.super.getTableProcessTypes(table, processCategory);
+ }
+
@Override
public List getProcessTypesByCategory(String processCategory) {
+ if (ProcessCategory.CLEANUP.getName().equalsIgnoreCase(processCategory)) {
+ return Collections.singletonList(PaimonActions.EXPIRE_SNAPSHOTS.getName());
+ }
+
if (ProcessCategory.OPTIMIZING.getName().equalsIgnoreCase(processCategory)) {
return OPTIMIZING_TYPES;
}
diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java
new file mode 100644
index 0000000000..c018f7a94d
--- /dev/null
+++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.paimon.process;
+
+import org.apache.amoro.Action;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.config.ConfigOption;
+import org.apache.amoro.config.ConfigOptions;
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.process.ExecuteEngine;
+import org.apache.amoro.process.LocalExecutionEngine;
+import org.apache.amoro.process.ProcessFactory;
+import org.apache.amoro.process.ProcessTriggerStrategy;
+import org.apache.amoro.process.RecoverProcessFailedException;
+import org.apache.amoro.process.TableProcess;
+import org.apache.amoro.process.TableProcessStore;
+import org.apache.amoro.table.StateKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/** Paimon maintenance, scheduled and recovered by the common Process framework. */
+public class PaimonProcessFactory implements ProcessFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(PaimonProcessFactory.class);
+ public static final ConfigOption EXPIRE_ENABLED =
+ ConfigOptions.key("expire-snapshots.enabled").booleanType().defaultValue(true);
+ public static final ConfigOption EXPIRE_INTERVAL =
+ ConfigOptions.key("expire-snapshots.interval").durationType().noDefaultValue();
+ public static final StateKey LAST_SUCCESS_TIME =
+ StateKey.stateKey("paimon.snapshot-expiration.last-success-time").longType().defaultValue(0L);
+
+ private final Clock clock;
+
+ /** SPI entry point using the system clock. */
+ public PaimonProcessFactory() {
+ this(Clock.systemUTC());
+ }
+
+ /** Allow subclasses to supply a clock without changing scheduling or retention policy. */
+ protected PaimonProcessFactory(Clock clock) {
+ this.clock = Objects.requireNonNull(clock, "clock");
+ }
+
+ private ExecuteEngine localEngine;
+ private final Map actions = new HashMap<>();
+
+ @Override
+ public String name() {
+ return "paimon";
+ }
+
+ @Override
+ public void open(Map properties) {
+ actions.clear();
+ Configurations configs =
+ Configurations.fromMap(properties == null ? Collections.emptyMap() : properties);
+ if (!configs.getBoolean(EXPIRE_ENABLED)) {
+ return;
+ }
+ if (properties == null || !properties.containsKey(EXPIRE_INTERVAL.key())) {
+ LOG.info("Paimon snapshot expiration is inactive: no explicit {}", EXPIRE_INTERVAL.key());
+ return;
+ }
+ Duration interval = configs.getDuration(EXPIRE_INTERVAL);
+ if (interval.toMillis() < 1) {
+ throw new IllegalArgumentException("Paimon expire-snapshots.interval must be at least 1 ms");
+ }
+ actions.put(PaimonActions.EXPIRE_SNAPSHOTS, ProcessTriggerStrategy.triggerAtFixRate(interval));
+ }
+
+ @Override
+ public void close() {
+ actions.clear();
+ }
+
+ @Override
+ public List> requiredStates() {
+ return Collections.singletonList(LAST_SUCCESS_TIME);
+ }
+
+ @Override
+ public Map> supportedActions() {
+ return actions.isEmpty()
+ ? Collections.emptyMap()
+ : Collections.singletonMap(TableFormat.PAIMON, Set.copyOf(actions.keySet()));
+ }
+
+ @Override
+ public ProcessTriggerStrategy triggerStrategy(TableFormat format, Action action) {
+ return format == TableFormat.PAIMON
+ ? actions.getOrDefault(action, ProcessTriggerStrategy.METADATA_TRIGGER)
+ : ProcessTriggerStrategy.METADATA_TRIGGER;
+ }
+
+ @Override
+ public void availableExecuteEngines(Collection engines) {
+ localEngine = null;
+ for (ExecuteEngine engine : engines) {
+ if (engine instanceof LocalExecutionEngine) {
+ localEngine = engine;
+ break;
+ }
+ }
+ }
+
+ @Override
+ public Optional trigger(TableRuntime runtime, Action action) {
+ if (runtime.getFormat() != TableFormat.PAIMON || !actions.containsKey(action)) {
+ return Optional.empty();
+ }
+
+ if (PaimonActions.EXPIRE_SNAPSHOTS.equals(action)) {
+ return triggerExpireSnapshot(runtime);
+ }
+
+ return Optional.empty();
+ }
+
+ private Optional triggerExpireSnapshot(TableRuntime runtime) {
+ if (localEngine == null) {
+ return Optional.empty();
+ }
+ ProcessTriggerStrategy strategy = actions.get(PaimonActions.EXPIRE_SNAPSHOTS);
+ if (clock.millis() - runtime.getState(LAST_SUCCESS_TIME)
+ < strategy.getTriggerInterval().toMillis()) {
+ return Optional.empty();
+ }
+ String reason = PaimonSnapshotExpiration.skipReason(PaimonSnapshotExpiration.refresh(runtime));
+ if (reason != null) {
+ if ("governance-disabled".equals(reason)) {
+ LOG.debug(
+ "Skip Paimon snapshot expiration for {}: {}", runtime.getTableIdentifier(), reason);
+ } else {
+ LOG.info(
+ "Skip Paimon snapshot expiration for {}: {}", runtime.getTableIdentifier(), reason);
+ }
+ return Optional.empty();
+ }
+ return Optional.of(new PaimonSnapshotsExpiringProcess(runtime, localEngine));
+ }
+
+ @Override
+ public TableProcess recover(TableRuntime runtime, TableProcessStore store)
+ throws RecoverProcessFailedException {
+ if (localEngine == null
+ || runtime.getFormat() != TableFormat.PAIMON
+ || !PaimonActions.EXPIRE_SNAPSHOTS.equals(store.getAction())) {
+ throw new RecoverProcessFailedException(
+ "Cannot recover Paimon snapshot expiration: unsupported action/format or no local engine");
+ }
+ return new PaimonSnapshotsExpiringProcess(runtime, localEngine);
+ }
+}
diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java
new file mode 100644
index 0000000000..b1cf799728
--- /dev/null
+++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.paimon.process;
+
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.formats.paimon.PaimonTable;
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.options.ExpireConfig;
+import org.apache.paimon.table.AppendOnlyFileStoreTable;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.PrimaryKeyFileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.utils.BranchManager;
+
+import java.time.Duration;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/** Table eligibility and native retention configuration for snapshot expiration. */
+final class PaimonSnapshotExpiration {
+ private static final String ENABLED = "amoro.expire-snapshots.enabled";
+ private static final boolean ENABLED_DEFAULT = true;
+
+ private PaimonSnapshotExpiration() {}
+
+ static PaimonTable refresh(TableRuntime runtime) {
+ AmoroTable> table = runtime.loadTable();
+ if (!(table instanceof PaimonTable)) {
+ throw new IllegalArgumentException("Snapshot expiration requires a Paimon table");
+ }
+ return ((PaimonTable) table).refresh();
+ }
+
+ /** Null means that this table is eligible; other values are safe diagnostic reason codes. */
+ static String skipReason(PaimonTable table) {
+ Map properties = table.properties();
+ if (!Boolean.parseBoolean(properties.getOrDefault(ENABLED, String.valueOf(ENABLED_DEFAULT)))) {
+ return "governance-disabled";
+ }
+ Table nativeTable = table.originalTable();
+ if (!(nativeTable instanceof PrimaryKeyFileStoreTable)
+ && !(nativeTable instanceof AppendOnlyFileStoreTable)) {
+ return "unsupported-table-type";
+ }
+ FileStoreTable fileStoreTable = (FileStoreTable) nativeTable;
+ if (!BranchManager.isMainBranch(fileStoreTable.snapshotManager().branch())) {
+ return "non-default-branch";
+ }
+ CoreOptions options = fileStoreTable.store().options();
+ if (!options.writeOnly()) {
+ return "write-only-required";
+ }
+ return null;
+ }
+
+ static ExpireConfig configuration(FileStoreTable table) {
+ ExpireConfig config = table.store().options().expireConfig();
+ validate(
+ config.getSnapshotRetainMin(),
+ config.getSnapshotRetainMax(),
+ config.getSnapshotTimeRetain(),
+ config.getSnapshotMaxDeletes());
+ validate(
+ config.getChangelogRetainMin(),
+ config.getChangelogRetainMax(),
+ config.getChangelogTimeRetain(),
+ config.getChangelogMaxDeletes());
+ return config;
+ }
+
+ private static void validate(int min, int max, Duration time, int limit) {
+ if (min < 1 || max < min || time == null || time.isNegative() || limit <= 0) {
+ throw new IllegalArgumentException("Invalid Paimon retention configuration");
+ }
+ // The native expiration implementation operates in milliseconds.
+ time.toMillis();
+ }
+
+ static Map summary(ExpireConfig config) {
+ Map result = new LinkedHashMap<>();
+ result.put("snapshot.num-retained.min", String.valueOf(config.getSnapshotRetainMin()));
+ result.put("snapshot.num-retained.max", String.valueOf(config.getSnapshotRetainMax()));
+ result.put("snapshot.time-retained", config.getSnapshotTimeRetain().toString());
+ result.put("snapshot.expire.limit", String.valueOf(config.getSnapshotMaxDeletes()));
+ result.put("changelog.num-retained.min", String.valueOf(config.getChangelogRetainMin()));
+ result.put("changelog.num-retained.max", String.valueOf(config.getChangelogRetainMax()));
+ result.put("changelog.time-retained", config.getChangelogTimeRetain().toString());
+ result.put("effective-changelog-max-deletes", String.valueOf(config.getChangelogMaxDeletes()));
+ return result;
+ }
+}
diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java
new file mode 100644
index 0000000000..4a6222cd1d
--- /dev/null
+++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.paimon.process;
+
+import org.apache.amoro.Action;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.formats.paimon.PaimonTable;
+import org.apache.amoro.process.ExecuteEngine;
+import org.apache.amoro.process.LocalProcess;
+import org.apache.amoro.process.TableProcess;
+import org.apache.paimon.options.ExpireConfig;
+import org.apache.paimon.table.FileStoreTable;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/** One native snapshot expiration invocation using the current table's retention settings. */
+public class PaimonSnapshotsExpiringProcess extends TableProcess implements LocalProcess {
+ public static final String TAG = "paimon-expire-snapshots";
+ private volatile Map summary = Collections.emptyMap();
+
+ public PaimonSnapshotsExpiringProcess(TableRuntime runtime, ExecuteEngine engine) {
+ super(runtime, engine);
+ }
+
+ @Override
+ public String tag() {
+ return TAG;
+ }
+
+ @Override
+ public void run() {
+ Map result = new LinkedHashMap<>();
+ publish(result);
+ String stage = "load";
+ try {
+ PaimonTable table = PaimonSnapshotExpiration.refresh(tableRuntime);
+ stage = "config";
+ String reason = PaimonSnapshotExpiration.skipReason(table);
+ if (reason != null) {
+ result.put("outcome", "skipped");
+ result.put("skip-reason", reason);
+ publish(result);
+ return;
+ }
+ FileStoreTable nativeTable = (FileStoreTable) table.originalTable();
+ ExpireConfig config = PaimonSnapshotExpiration.configuration(nativeTable);
+ result.putAll(PaimonSnapshotExpiration.summary(config));
+ publish(result);
+ stage = "expire";
+ int expired = nativeTable.newExpireSnapshots().config(config).expire();
+ result.put("outcome", expired == 0 ? "noop" : "executed");
+ result.put("reported-expired-snapshots", String.valueOf(expired));
+ result.put("result-scope", "last-completed-invocation");
+ publish(result);
+ stage = "state-update";
+ tableRuntime.updateState(
+ PaimonProcessFactory.LAST_SUCCESS_TIME, ignored -> System.currentTimeMillis());
+ } catch (Exception e) {
+ result.put("error-stage", stage);
+ // Exception messages can contain catalog credentials or signed object-store URLs.
+ result.put(
+ "error-message",
+ "Snapshot expiration failed during "
+ + stage
+ + " ("
+ + e.getClass().getSimpleName()
+ + "). See AMS logs for details.");
+ publish(result);
+ LOG.error(
+ "Paimon snapshot expiration failed during {} for {}", stage, getTableIdentifier(), e);
+ throw new RuntimeException("Paimon snapshot expiration failed during " + stage, e);
+ }
+ }
+
+ private void publish(Map result) {
+ summary = Collections.unmodifiableMap(new LinkedHashMap<>(result));
+ }
+
+ @Override
+ public Action getAction() {
+ return PaimonActions.EXPIRE_SNAPSHOTS;
+ }
+
+ @Override
+ public Map getProcessParameters() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map getSummary() {
+ return summary;
+ }
+}
diff --git a/amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory b/amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory
new file mode 100644
index 0000000000..9f49765bd9
--- /dev/null
+++ b/amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# *
+# http://www.apache.org/licenses/LICENSE-2.0
+# *
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.amoro.formats.paimon.process.PaimonProcessFactory
diff --git a/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java
new file mode 100644
index 0000000000..6c90b206dd
--- /dev/null
+++ b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.paimon;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.paimon.catalog.CachingCatalog;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.types.DataTypes;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.util.Collections;
+
+class TestPaimonTableRefresh {
+ @TempDir Path warehouse;
+
+ @Test
+ void refreshInvalidatesCachedTableWithoutClosingCatalog() throws Exception {
+ try (Catalog original =
+ PaimonCatalogFactory.paimonCatalog(
+ Collections.singletonMap("warehouse", warehouse.toString()), new Configuration());
+ Catalog cached = new CachingCatalog(original, new Options())) {
+ original.createDatabase("db", false);
+ Identifier id = Identifier.create("db", "tbl");
+ original.createTable(id, Schema.newBuilder().column("id", DataTypes.INT()).build(), false);
+ PaimonCatalog catalog = new PaimonCatalog(cached, "catalog");
+ PaimonTable before = (PaimonTable) catalog.loadTable("db", "tbl");
+ original.alterTable(
+ id, SchemaChange.setOption("amoro.expire-snapshots.enabled", "true"), false);
+ assertNull(before.properties().get("amoro.expire-snapshots.enabled"));
+ assertSame(before.originalTable(), catalog.loadTable("db", "tbl").originalTable());
+ PaimonTable fresh = before.refresh();
+ assertEquals("true", fresh.properties().get("amoro.expire-snapshots.enabled"));
+ original.alterTable(id, SchemaChange.setOption("snapshot.num-retained.min", "2"), false);
+ assertEquals("2", fresh.refresh().properties().get("snapshot.num-retained.min"));
+ assertTrue(catalog.tableExists("db", "tbl"));
+ assertThrows(
+ IllegalStateException.class,
+ () -> new PaimonTable(before.id(), before.originalTable()).refresh());
+ }
+ }
+}
diff --git a/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java
new file mode 100644
index 0000000000..378bcfb6bc
--- /dev/null
+++ b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java
@@ -0,0 +1,707 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.paimon.process;
+
+import org.apache.amoro.Action;
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.PaimonActions;
+import org.apache.amoro.ServerTableIdentifier;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.TableRuntime;
+import org.apache.amoro.config.TableConfiguration;
+import org.apache.amoro.formats.paimon.PaimonCatalog;
+import org.apache.amoro.formats.paimon.PaimonCatalogFactory;
+import org.apache.amoro.metrics.MetricRegistry;
+import org.apache.amoro.process.LocalExecutionEngine;
+import org.apache.amoro.process.LocalProcess;
+import org.apache.amoro.process.ProcessFactory;
+import org.apache.amoro.process.RecoverProcessFailedException;
+import org.apache.amoro.process.TableProcess;
+import org.apache.amoro.process.TableProcessStore;
+import org.apache.amoro.table.StateKey;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.paimon.catalog.CachingCatalog;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.consumer.Consumer;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.StreamTableScan;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.CloseableIterator;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Mockito;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.function.Function;
+import java.util.stream.StreamSupport;
+
+class TestPaimonSnapshotExpiration {
+ @TempDir Path warehouse;
+ Catalog original;
+ Catalog cached;
+ PaimonCatalog catalog;
+ final Identifier id = Identifier.create("db", "tbl");
+ final LocalExecutionEngine engine = new LocalExecutionEngine();
+
+ @BeforeEach
+ void setup() throws Exception {
+ original =
+ PaimonCatalogFactory.paimonCatalog(
+ Collections.singletonMap("warehouse", warehouse.toString()), new Configuration());
+ original.createDatabase("db", false);
+ cached = new CachingCatalog(original, new Options());
+ catalog = new PaimonCatalog(cached, "catalog");
+ }
+
+ @AfterEach
+ void close() throws Exception {
+ cached.close();
+ engine.close();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void expiresUsingTableRetentionAndLimitAndPreservesCurrentData(boolean primaryKey)
+ throws Exception {
+ createTable(
+ primaryKey,
+ Map.of(
+ "snapshot.num-retained.min",
+ "1",
+ "snapshot.num-retained.max",
+ "2",
+ "snapshot.time-retained",
+ "100 d",
+ "snapshot.expire.limit",
+ "1"));
+ append(1, 4);
+ Runtime runtime = new Runtime();
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ process.run();
+ Assertions.assertEquals(2L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals("executed", process.getSummary().get("outcome"));
+ Assertions.assertEquals("1", process.getSummary().get("reported-expired-snapshots"));
+ Assertions.assertEquals("2", process.getSummary().get("snapshot.num-retained.max"));
+ Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0);
+ Assertions.assertEquals(List.of(1, 2, 3, 4), readIds(table()));
+ process.run();
+ Assertions.assertEquals(3L, table().snapshotManager().earliestSnapshotId());
+ process.run();
+ Assertions.assertEquals("noop", process.getSummary().get("outcome"));
+ Assertions.assertEquals("0", process.getSummary().get("reported-expired-snapshots"));
+ }
+
+ @Test
+ void nativeDefaultsAndEmptyTableAreReportedWithoutOverrides() throws Exception {
+ createTable(false, Collections.emptyMap());
+ PaimonSnapshotsExpiringProcess process =
+ new PaimonSnapshotsExpiringProcess(new Runtime(), engine);
+ process.run();
+ Assertions.assertEquals("noop", process.getSummary().get("outcome"));
+ Assertions.assertEquals("10", process.getSummary().get("snapshot.num-retained.min"));
+ Assertions.assertEquals("2147483647", process.getSummary().get("snapshot.num-retained.max"));
+ Assertions.assertEquals("PT1H", process.getSummary().get("snapshot.time-retained"));
+ Assertions.assertEquals("50", process.getSummary().get("snapshot.expire.limit"));
+ Assertions.assertEquals("10", process.getSummary().get("changelog.num-retained.min"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "false,changelog.num-retained.min,2",
+ "true,changelog.num-retained.min,2",
+ "false,changelog.num-retained.max,5",
+ "true,changelog.num-retained.max,5",
+ "false,changelog.time-retained,200 d",
+ "true,changelog.time-retained,200 d"
+ })
+ void independentChangelogRetentionPreservesHistory(boolean primaryKey, String key, String value)
+ throws Exception {
+ Map options =
+ new HashMap<>(
+ Map.of(
+ "snapshot.num-retained.min", "1",
+ "snapshot.num-retained.max", "2",
+ "snapshot.time-retained", "100 d"));
+ if (primaryKey) {
+ options.put("changelog-producer", "input");
+ }
+ options.put(key, value);
+ createTable(primaryKey, options);
+ append(1, 8);
+ Assertions.assertTrue(table().store().options().changelogLifecycleDecoupled());
+ Runtime runtime = new Runtime();
+ TableProcess process = factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow();
+ ((LocalProcess) process).run();
+ Assertions.assertEquals("executed", process.getSummary().get("outcome"));
+ Assertions.assertEquals("6", process.getSummary().get("reported-expired-snapshots"));
+ Assertions.assertEquals(7L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0);
+ Assertions.assertEquals(List.of(1L, 2L, 3L, 4L, 5L, 6L), archivedChangelogIds());
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6, 7, 8), readIds(table()));
+
+ // Resume a consumer from an expired snapshot, reading actual retained changelog files.
+ ReadBuilder reader = table().newReadBuilder();
+ StreamTableScan scan = reader.newStreamScan();
+ scan.restore(1L);
+ List history = new ArrayList<>();
+ for (int i = 0; i < 8; i++) {
+ try (CloseableIterator rows =
+ reader.newRead().createReader(scan.plan()).toCloseableIterator()) {
+ while (rows.hasNext()) {
+ history.add(rows.next().getInt(0));
+ }
+ }
+ }
+ Collections.sort(history);
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6, 7, 8), history);
+ ((LocalProcess) process).run();
+ Assertions.assertEquals("noop", process.getSummary().get("outcome"));
+ // A separate changelog expiry would delete some of these archives; snapshot expiry must not.
+ Assertions.assertEquals(List.of(1L, 2L, 3L, 4L, 5L, 6L), archivedChangelogIds());
+ }
+
+ @Test
+ void queuedAndRecoveredProcessesAcceptIndependentChangelogRetention() throws Exception {
+ createTable(false, Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "2"));
+ append(1, 4);
+ Runtime runtime = new Runtime();
+ PaimonProcessFactory factory = factory();
+ TableProcess queued = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow();
+ original.alterTable(id, SchemaChange.setOption("changelog.num-retained.max", "5"), false);
+ ((LocalProcess) queued).run();
+ Assertions.assertEquals("executed", queued.getSummary().get("outcome"));
+ Assertions.assertEquals("5", queued.getSummary().get("changelog.num-retained.max"));
+ Assertions.assertEquals(List.of(1L, 2L), archivedChangelogIds());
+ append(5, 6);
+ TableProcessStore store = Mockito.mock(TableProcessStore.class);
+ Mockito.when(store.getAction()).thenReturn(PaimonActions.EXPIRE_SNAPSHOTS);
+ TableProcess recovered = factory.recover(runtime, store);
+ original.alterTable(id, SchemaChange.setOption("changelog.num-retained.max", "6"), false);
+ ((LocalProcess) recovered).run();
+ Assertions.assertEquals("executed", recovered.getSummary().get("outcome"));
+ Assertions.assertEquals("6", recovered.getSummary().get("changelog.num-retained.max"));
+ Assertions.assertEquals(5L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals(List.of(1L, 2L, 3L, 4L), archivedChangelogIds());
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6), readIds(table()));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "amoro.expire-snapshots.enabled,false,governance-disabled",
+ "write-only,false,write-only-required"
+ })
+ void queuedProcessRechecksEligibility(String key, String value, String reason) throws Exception {
+ createTable(false, Collections.emptyMap());
+ Runtime runtime = new Runtime();
+ PaimonProcessFactory factory = factory();
+ TableProcess process = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow();
+ original.alterTable(id, SchemaChange.setOption(key, value), false);
+ ((LocalProcess) process).run();
+ Assertions.assertEquals("skipped", process.getSummary().get("outcome"));
+ Assertions.assertEquals(reason, process.getSummary().get("skip-reason"));
+ Assertions.assertFalse(process.getSummary().containsKey("reported-expired-snapshots"));
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ }
+
+ @Test
+ void queuedAndRecoveredProcessesReadLatestPolicy() throws Exception {
+ createTable(
+ true,
+ Map.of(
+ "snapshot.num-retained.min",
+ "1",
+ "snapshot.num-retained.max",
+ "3",
+ "snapshot.time-retained",
+ "100 d",
+ "snapshot.expire.limit",
+ "1"));
+ append(1, 6);
+ Runtime runtime = new Runtime();
+ PaimonProcessFactory factory = factory();
+ TableProcess queued = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow();
+ original.alterTable(
+ id,
+ List.of(
+ SchemaChange.setOption("snapshot.num-retained.max", "2"),
+ SchemaChange.setOption("snapshot.expire.limit", "10")),
+ false);
+ ((LocalProcess) queued).run();
+ Assertions.assertEquals(5L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals("10", queued.getSummary().get("snapshot.expire.limit"));
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ Assertions.assertTrue(queued.getProcessParameters().isEmpty());
+ TableProcessStore store = Mockito.mock(TableProcessStore.class);
+ Mockito.when(store.getAction()).thenReturn(PaimonActions.EXPIRE_SNAPSHOTS);
+ original.alterTable(id, SchemaChange.setOption("snapshot.num-retained.max", "1"), false);
+ TableProcess recovered = factory.recover(runtime, store);
+ ((LocalProcess) recovered).run();
+ Assertions.assertEquals(6L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6), readIds(table()));
+ }
+
+ @Test
+ void nativeResultSurvivesStateFailureAndRetryReplacesSummary() throws Exception {
+ createTable(false, Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "1"));
+ append(1, 3);
+ Runtime runtime = new Runtime();
+ runtime.failStateUpdate = true;
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ Assertions.assertThrows(RuntimeException.class, process::run);
+ Map failed = process.getSummary();
+ Assertions.assertEquals("executed", failed.get("outcome"));
+ Assertions.assertEquals("2", failed.get("reported-expired-snapshots"));
+ Assertions.assertEquals("state-update", failed.get("error-stage"));
+ Assertions.assertEquals(3L, table().snapshotManager().earliestSnapshotId());
+ original.alterTable(
+ id, SchemaChange.setOption("amoro.expire-snapshots.enabled", "false"), false);
+ process.run();
+ Assertions.assertEquals("skipped", process.getSummary().get("outcome"));
+ Assertions.assertFalse(process.getSummary().containsKey("reported-expired-snapshots"));
+ Assertions.assertFalse(process.getSummary().containsKey("error-message"));
+ Assertions.assertEquals("executed", failed.get("outcome"));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class, () -> failed.put("outcome", "bad"));
+ }
+
+ @Test
+ void loadingFailureIsNotSkippedAndSummaryDoesNotLeakCredentials() throws Exception {
+ createTable(false, Collections.emptyMap());
+ Runtime runtime = new Runtime();
+ runtime.loader =
+ () -> {
+ throw new IllegalStateException("password=secret https://user:secret@host");
+ };
+ Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS));
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ Assertions.assertThrows(RuntimeException.class, process::run);
+ Assertions.assertEquals("load", process.getSummary().get("error-stage"));
+ Assertions.assertFalse(process.getSummary().containsKey("outcome"));
+ Assertions.assertFalse(process.getSummary().get("error-message").contains("secret"));
+ runtime.loader = () -> catalog.loadTable("db", "tbl");
+ process.run();
+ Assertions.assertEquals("noop", process.getSummary().get("outcome"));
+ Assertions.assertFalse(process.getSummary().containsKey("error-message"));
+ }
+
+ @Test
+ void invalidLimitFailsWithoutDeletingSnapshots() throws Exception {
+ createTable(
+ false,
+ Map.of(
+ "snapshot.num-retained.min",
+ "1",
+ "snapshot.num-retained.max",
+ "1",
+ "snapshot.expire.limit",
+ "0"));
+ append(1, 3);
+ PaimonSnapshotsExpiringProcess process =
+ new PaimonSnapshotsExpiringProcess(new Runtime(), engine);
+ Assertions.assertThrows(RuntimeException.class, process::run);
+ Assertions.assertEquals("config", process.getSummary().get("error-stage"));
+ Assertions.assertFalse(process.getSummary().containsKey("outcome"));
+ Assertions.assertEquals(1L, table().snapshotManager().earliestSnapshotId());
+ }
+
+ @Test
+ void nativeTagAndConsumerProtectionRemainEffective() throws Exception {
+ createTable(true, Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "1"));
+ append(1, 1);
+ table().createTag("first", 1);
+ append(2, 5);
+ table().consumerManager().resetConsumer("reader", new Consumer(3));
+ PaimonSnapshotsExpiringProcess process =
+ new PaimonSnapshotsExpiringProcess(new Runtime(), engine);
+ process.run();
+ Assertions.assertTrue(table().snapshotManager().snapshotExists(3));
+ Assertions.assertEquals(List.of(1), readIds(table().copy(Map.of("scan.tag-name", "first"))));
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5), readIds(table()));
+ table().consumerManager().deleteConsumer("reader");
+ process.run();
+ Assertions.assertEquals(5L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals(List.of(1), readIds(table().copy(Map.of("scan.tag-name", "first"))));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"omitted", "true", "false"})
+ void tableExpirationSwitchDefaultsToEnabled(String enabled) throws Exception {
+ createTable(
+ false,
+ "omitted".equals(enabled)
+ ? Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "1")
+ : Map.of(
+ "snapshot.num-retained.min", "1",
+ "snapshot.num-retained.max", "1",
+ "amoro.expire-snapshots.enabled", enabled));
+ append(1, 3);
+ Runtime runtime = new Runtime();
+ boolean expected = !"false".equals(enabled);
+ Assertions.assertEquals(
+ expected, factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ process.run();
+ Assertions.assertEquals(expected ? 3L : 1L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals(expected ? "executed" : "skipped", process.getSummary().get("outcome"));
+ if (!expected) {
+ Assertions.assertEquals("governance-disabled", process.getSummary().get("skip-reason"));
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ original.alterTable(id, SchemaChange.removeOption("amoro.expire-snapshots.enabled"), false);
+ Assertions.assertTrue(factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ process.run();
+ Assertions.assertEquals(3L, table().snapshotManager().earliestSnapshotId());
+ }
+ }
+
+ @Test
+ void pluginExpirationSwitchControlsActionRegistration() throws Exception {
+ createTable(false, Collections.emptyMap());
+ Runtime runtime = new Runtime();
+ PaimonProcessFactory factory = factory();
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ factory.open(Map.of("expire-snapshots.enabled", "false", "expire-snapshots.interval", "1h"));
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ // A disabled action does not require or parse its interval, as in the Iceberg factory.
+ factory.open(
+ Map.of("expire-snapshots.enabled", "false", "expire-snapshots.interval", "invalid"));
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ factory.open(Map.of("expire-snapshots.enabled", "false"));
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ factory.open(Map.of("expire-snapshots.enabled", "true"));
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ factory.open(Map.of("expire-snapshots.enabled", "true", "expire-snapshots.interval", "1h"));
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ }
+
+ @Test
+ void factoryDispatchOnlyAcceptsRegisteredPaimonActions() {
+ PaimonProcessFactory factory = factory();
+ TableRuntime runtime = Mockito.mock(TableRuntime.class);
+ Action unsupported = Action.register("test-paimon-unsupported");
+ Mockito.when(runtime.getFormat()).thenReturn(TableFormat.PAIMON);
+ Assertions.assertTrue(factory.trigger(runtime, unsupported).isEmpty());
+ Mockito.when(runtime.getFormat()).thenReturn(TableFormat.ICEBERG);
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ factory.close();
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ Mockito.when(runtime.getFormat()).thenReturn(TableFormat.PAIMON);
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ // Rejected dispatches must not read table state or load and refresh the table.
+ Mockito.verify(runtime, Mockito.atLeastOnce()).getFormat();
+ Mockito.verifyNoMoreInteractions(runtime);
+ }
+
+ @Test
+ void factoryRequiresExplicitIntervalAndLocalEngine() throws Exception {
+ PaimonProcessFactory factory = new PaimonProcessFactory();
+ factory.open(Collections.emptyMap());
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ factory.open(Map.of("expire-snapshots.interval", "1h"));
+ Assertions.assertEquals(
+ java.time.Duration.ofHours(1),
+ factory
+ .triggerStrategy(TableFormat.PAIMON, PaimonActions.EXPIRE_SNAPSHOTS)
+ .getTriggerInterval());
+ createTable(false, Collections.emptyMap());
+ Runtime runtime = new Runtime();
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ TableProcessStore store = Mockito.mock(TableProcessStore.class);
+ Mockito.when(store.getAction()).thenReturn(PaimonActions.EXPIRE_SNAPSHOTS);
+ Assertions.assertThrows(
+ RecoverProcessFailedException.class, () -> factory.recover(runtime, store));
+ factory.availableExecuteEngines(List.of(engine));
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ factory.open(Collections.emptyMap());
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ Assertions.assertTrue(
+ StreamSupport.stream(ServiceLoader.load(ProcessFactory.class).spliterator(), false)
+ .anyMatch(f -> f instanceof PaimonProcessFactory));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "0ms", "-1s", "1ns", "invalid", "9223372036854775807d"})
+ void invalidIntervalsDoNotRegisterActions(String interval) {
+ PaimonProcessFactory factory = new PaimonProcessFactory();
+ Assertions.assertThrows(
+ RuntimeException.class, () -> factory.open(Map.of("expire-snapshots.interval", interval)));
+ Assertions.assertTrue(factory.supportedActions().isEmpty());
+ }
+
+ @Test
+ void defaultBranchAndOrdinaryTableScopeAreCheckedBeforeExecution() throws Exception {
+ createTable(true, Collections.emptyMap());
+ append(1, 1);
+ table().createBranch("development");
+ Runtime runtime = new Runtime();
+ runtime.loader = () -> catalog.loadTable("db", "tbl$branch_development");
+ Assertions.assertTrue(factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ process.run();
+ Assertions.assertEquals("non-default-branch", process.getSummary().get("skip-reason"));
+ runtime.loader = () -> catalog.loadTable("db", "tbl$snapshots");
+ process.run();
+ Assertions.assertEquals("skipped", process.getSummary().get("outcome"));
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void localEngineUsesDedicatedPoolOrExistingDefaultFallback(boolean configured) throws Exception {
+ createTable(false, Collections.emptyMap());
+ engine.open(configured ? Map.of("pool.paimon-expire-snapshots.thread-count", "1") : Map.of());
+ Runtime runtime = new Runtime();
+ java.util.concurrent.atomic.AtomicReference thread =
+ new java.util.concurrent.atomic.AtomicReference<>();
+ runtime.loader =
+ () -> {
+ thread.set(Thread.currentThread().getName());
+ return catalog.loadTable("db", "tbl");
+ };
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ String executionId = engine.submitTableProcess(process);
+ long deadline = System.nanoTime() + java.time.Duration.ofSeconds(10).toNanos();
+ while (engine.getStatus(executionId) == org.apache.amoro.process.ProcessStatus.RUNNING
+ && System.nanoTime() < deadline) {
+ Thread.sleep(10);
+ }
+ Assertions.assertEquals(
+ org.apache.amoro.process.ProcessStatus.SUCCESS, engine.getStatus(executionId));
+ Assertions.assertTrue(
+ thread.get().startsWith(configured ? "local-paimon-expire-snapshots-" : "local-default-"));
+ }
+
+ @Test
+ void nativeFailureProducesFailureSummaryWithoutCallingChangelogExpiration() throws Exception {
+ createTable(true, Collections.emptyMap());
+ org.apache.paimon.table.PrimaryKeyFileStoreTable nativeTable =
+ Mockito.spy((org.apache.paimon.table.PrimaryKeyFileStoreTable) table());
+ org.apache.paimon.table.ExpireSnapshots expiration =
+ Mockito.mock(org.apache.paimon.table.ExpireSnapshots.class);
+ Mockito.doReturn(expiration).when(nativeTable).newExpireSnapshots();
+ Mockito.when(expiration.config(Mockito.any())).thenReturn(expiration);
+ Mockito.when(expiration.expire()).thenThrow(new IllegalStateException("secret-token=hidden"));
+ Catalog failingCatalog = Mockito.mock(Catalog.class);
+ Mockito.when(failingCatalog.getTable(id)).thenReturn(nativeTable);
+ PaimonCatalog wrapper = new PaimonCatalog(failingCatalog, "catalog");
+ Runtime runtime = new Runtime();
+ runtime.loader = () -> wrapper.loadTable("db", "tbl");
+ PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine);
+ Assertions.assertThrows(RuntimeException.class, process::run);
+ Assertions.assertEquals("expire", process.getSummary().get("error-stage"));
+ Assertions.assertFalse(process.getSummary().containsKey("reported-expired-snapshots"));
+ Assertions.assertFalse(process.getSummary().get("error-message").contains("hidden"));
+ Mockito.verify(nativeTable, Mockito.never()).newExpireChangelog();
+ Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ }
+
+ @Test
+ void nativeTimeAndMinimumRetentionAreRespectedAndChangelogSettingsPreserved() throws Exception {
+ createTable(
+ false,
+ Map.of(
+ "snapshot.num-retained.min",
+ "2",
+ "snapshot.time-retained",
+ "100 d",
+ "changelog.num-retained.min",
+ "1",
+ "changelog.time-retained",
+ "1 d",
+ "snapshot.expire.limit",
+ "7"));
+ append(1, 5);
+ PaimonSnapshotsExpiringProcess process =
+ new PaimonSnapshotsExpiringProcess(new Runtime(), engine);
+ process.run();
+ Assertions.assertEquals(1L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals("noop", process.getSummary().get("outcome"));
+ Assertions.assertEquals("1", process.getSummary().get("changelog.num-retained.min"));
+ Assertions.assertEquals("PT24H", process.getSummary().get("changelog.time-retained"));
+ Assertions.assertEquals("7", process.getSummary().get("effective-changelog-max-deletes"));
+ original.alterTable(
+ id,
+ List.of(
+ SchemaChange.setOption("snapshot.time-retained", "0 ms"),
+ SchemaChange.setOption("changelog.time-retained", "0 ms")),
+ false);
+ process.run();
+ Assertions.assertEquals(4L, table().snapshotManager().earliestSnapshotId());
+ Assertions.assertEquals(List.of(1, 2, 3, 4, 5), readIds(table()));
+ }
+
+ @Test
+ void factoryUsesControlledTimeAtSuccessIntervalBoundary() throws Exception {
+ createTable(false, Collections.emptyMap());
+ Runtime runtime = new Runtime();
+ runtime.updateState(PaimonProcessFactory.LAST_SUCCESS_TIME, ignored -> 10_000L);
+ java.time.Clock clock = Mockito.mock(java.time.Clock.class);
+ PaimonProcessFactory factory = new PaimonProcessFactory(clock);
+ factory.open(Map.of("expire-snapshots.interval", "1s"));
+ factory.availableExecuteEngines(List.of(engine));
+
+ Mockito.when(clock.millis()).thenReturn(10_999L);
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty());
+ Mockito.when(clock.millis()).thenReturn(11_000L);
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ Mockito.when(clock.millis()).thenReturn(11_001L);
+ Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent());
+ Assertions.assertEquals(10_000L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME));
+ }
+
+ private PaimonProcessFactory factory() {
+ PaimonProcessFactory factory = new PaimonProcessFactory();
+ factory.open(Map.of("expire-snapshots.interval", "1h"));
+ factory.availableExecuteEngines(List.of(engine));
+ return factory;
+ }
+
+ void createTable(boolean primaryKey, Map options) throws Exception {
+ Schema.Builder builder =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .option("write-only", "true")
+ .option("bucket", primaryKey ? "1" : "-1");
+ if (primaryKey) {
+ builder.primaryKey("id");
+ }
+ options.forEach(builder::option);
+ original.createTable(id, builder.build(), false);
+ }
+
+ FileStoreTable table() throws Exception {
+ return (FileStoreTable) original.getTable(id);
+ }
+
+ private List archivedChangelogIds() throws Exception {
+ List ids = new ArrayList<>();
+ table().changelogManager().safelyGetAllChangelogs().forEach(log -> ids.add(log.id()));
+ Collections.sort(ids);
+ return ids;
+ }
+
+ void append(int start, int end) throws Exception {
+ for (int i = start; i <= end; i++) {
+ BatchWriteBuilder builder = table().newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite();
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(GenericRow.of(i));
+ commit.commit(write.prepareCommit());
+ }
+ }
+ }
+
+ static List readIds(org.apache.paimon.table.Table table) throws Exception {
+ ReadBuilder builder = table.newReadBuilder();
+ List ids = new ArrayList<>();
+ try (CloseableIterator rows =
+ builder.newRead().createReader(builder.newScan().plan()).toCloseableIterator()) {
+ while (rows.hasNext()) {
+ ids.add(rows.next().getInt(0));
+ }
+ }
+ Collections.sort(ids);
+ return ids;
+ }
+
+ class Runtime implements TableRuntime {
+ private final Map, Object> states = new HashMap<>();
+ java.util.function.Supplier> loader = () -> catalog.loadTable("db", "tbl");
+ boolean failStateUpdate;
+
+ @Override
+ public AmoroTable> loadTable() {
+ return loader.get();
+ }
+
+ @Override
+ public ServerTableIdentifier getTableIdentifier() {
+ return ServerTableIdentifier.of(1L, "catalog", "db", "tbl", TableFormat.PAIMON);
+ }
+
+ @Override
+ public T getState(StateKey key) {
+ return (T) states.getOrDefault(key, key.getDefaultValue());
+ }
+
+ @Override
+ public void updateState(StateKey key, Function updater) {
+ if (failStateUpdate) {
+ throw new IllegalStateException("injected state failure");
+ }
+ states.put(key, updater.apply(getState(key)));
+ }
+
+ @Override
+ public List extends TableProcessStore> getProcessStates() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List extends TableProcessStore> getProcessStates(Action action) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String getGroupName() {
+ return "default";
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Map getTableConfig() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void registerMetric(MetricRegistry registry) {}
+
+ @Override
+ public void unregisterMetric() {}
+ }
+}
diff --git a/amoro-web/src/views/tables/components/Optimizing.vue b/amoro-web/src/views/tables/components/Optimizing.vue
index dac192b47d..d41cc1547b 100644
--- a/amoro-web/src/views/tables/components/Optimizing.vue
+++ b/amoro-web/src/views/tables/components/Optimizing.vue
@@ -163,7 +163,7 @@ async function refreshOptimizingProcesses() {
}
}
-async function cancel() {
+async function cancel(targetProcessId: string | number = processId.value) {
Modal.confirm({
title: t(props.cancelModalTitleKey),
onOk: async () => {
@@ -171,7 +171,7 @@ async function cancel() {
loading.value = true
await cancelOptimizingProcess({
...sourceData,
- processId: processId.value?.toString(),
+ processId: targetProcessId.toString(),
})
cancelDisabled.value = true
refresh()
@@ -305,7 +305,8 @@ onMounted(() => {
-
+ {{ record.processId }}
+
{{ record.processId }}
@@ -331,6 +332,13 @@ onMounted(() => {
+
+ {{ t('cancelProcess') }}
+
{{ key }} :
@@ -353,9 +361,10 @@ onMounted(() => {
-
{{ t("cancelProcess") }}
diff --git a/dist/src/main/amoro-bin/conf/plugins/execute-engines.yaml b/dist/src/main/amoro-bin/conf/plugins/execute-engines.yaml
index 14ebcc4e90..4522fd9ede 100755
--- a/dist/src/main/amoro-bin/conf/plugins/execute-engines.yaml
+++ b/dist/src/main/amoro-bin/conf/plugins/execute-engines.yaml
@@ -26,4 +26,5 @@ execute-engines:
pool.clean-dangling-delete-files.thread-count: 10
pool.expire-data.thread-count: 10
pool.auto-create-tags.thread-count: 3
- pool.sync-hive-tables.thread-count: 10
\ No newline at end of file
+ pool.sync-hive-tables.thread-count: 10
+ pool.paimon-expire-snapshots.thread-count: 1
diff --git a/dist/src/main/amoro-bin/conf/plugins/process-factories.yaml b/dist/src/main/amoro-bin/conf/plugins/process-factories.yaml
index 1085d364fe..2f5494c87a 100755
--- a/dist/src/main/amoro-bin/conf/plugins/process-factories.yaml
+++ b/dist/src/main/amoro-bin/conf/plugins/process-factories.yaml
@@ -32,3 +32,12 @@ process-factories:
auto-create-tags.interval: "1min"
sync-hive-tables.enabled: "false"
sync-hive-tables.interval: "10min"
+
+ # Requires the Paimon format module and write-only tables.
+ # The table property amoro.expire-snapshots.enabled defaults to true.
+ - name: paimon
+ enabled: false
+ priority: 100
+ properties:
+ expire-snapshots.enabled: "true"
+ expire-snapshots.interval: "1h"
diff --git a/docs/formats/paimon.md b/docs/formats/paimon.md
index 6c24a9eccd..877be2a50e 100644
--- a/docs/formats/paimon.md
+++ b/docs/formats/paimon.md
@@ -40,3 +40,103 @@ If you want to use S3 or OSS, please download the
[OSS](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-oss/0.5.0-incubating/paimon-oss-0.5.0-incubating.jar)
package and put it in the 'lib' directory of the Amoro installation package.
{{< /hint >}}
+
+
+## Snapshot expiration
+
+AMS can periodically expire snapshots for ordinary Paimon primary-key and append-only tables
+on the default branch. Install the Paimon format module (build AMS with
+`-Psupport-paimon-format`) and enable the `paimon` entry in
+`conf/plugins/process-factories.yaml`:
+
+```yaml
+ - name: paimon
+ enabled: true
+ priority: 100
+ properties:
+ expire-snapshots.enabled: "true"
+ expire-snapshots.interval: "1h"
+```
+
+The distributed configuration disables this plugin. The plugin property
+`expire-snapshots.enabled` defaults to `true`; setting it to `false` prevents registration of the
+snapshot-expiration action. An explicit positive interval of at least
+one millisecond is required; missing configuration does not register any Paimon maintenance
+actions, even when AMS automatically discovers plugins. Restart AMS after changing plugin
+configuration. The common plugin loader defaults an omitted `enabled` field to `true`.
+
+Add `pool.paimon-expire-snapshots.thread-count: 1` to the existing `local` execution engine's
+properties in `conf/plugins/execute-engines.yaml` (included in the distributed configuration).
+This selects a dedicated pool; without it, the local engine falls back to its default pool.
+Paimon may use additional internal threads for file deletion.
+
+The Amoro table property `amoro.expire-snapshots.enabled` defaults to `true`.
+After enabling the plugin and configuring its interval, eligible tables are scheduled without
+an explicit table switch. For example, the following Paimon Flink SQL makes the settings explicit:
+
+```sql
+ALTER TABLE my_table SET (
+ 'write-only' = 'true',
+ 'amoro.expire-snapshots.enabled' = 'true'
+);
+```
+
+Set `amoro.expire-snapshots.enabled=false` on an individual table to exclude it; removing the property
+restores the default `true`. Both the plugin action switch and the table switch must be enabled,
+and `write-only=true` is still required. This native option skips automatic compaction as well
+as snapshot expiration on the write path; this feature does not provide replacement compaction.
+Enabling the plugin assigns snapshot expiration for eligible tables to AMS; coordinate or stop
+external snapshot-expiration jobs before enabling it.
+AMS does not detect other maintenance jobs or change table properties automatically.
+System tables and non-default branches are skipped. Tables with independent changelog retention
+remain eligible for snapshot expiration.
+This release does not schedule orphan-file deletion, independent changelog expiration, compaction,
+partition expiration or tag lifecycle management.
+
+Each execution, retry and recovery reloads the table through its existing catalog, invalidating
+cached metadata. Paimon reads retention settings through `CoreOptions.expireConfig()` and applies
+them using `newExpireSnapshots().config(...).expire()`; AMS supplies no retention overrides.
+The relevant native settings are `snapshot.num-retained.min`, `snapshot.num-retained.max`,
+`snapshot.time-retained` and `snapshot.expire.limit`. With Paimon 1.2.0, defaults are respectively
+10, 2147483647, 1 hour and 50. Changelog settings retain their native values and fallback rules. In Paimon 1.1.1 and 1.2.0,
+the effective changelog deletion limit also comes from `snapshot.expire.limit`; there is no
+separate `changelog.expire.limit` table option in these versions.
+With independent changelog retention, native snapshot expiration preserves the files needed by
+changelog readers and archives expired snapshot metadata as changelog metadata. AMS does not call
+`newExpireChangelog()`: archived changelogs are not expired by this action, even on later runs.
+Because `write-only=true` also disables writer-side automatic changelog expiration, arrange
+separate native changelog maintenance when needed; setting changelog retention alone does not
+make AMS enforce that retention limit.
+The maximum snapshot count can expire snapshots within the time window; time retention alone
+is not an absolute guarantee. Paimon remains responsible for tag and consumer protection.
+Invalid retention configurations fail before expiration.
+
+Choose the interval and native expiration limit according to snapshot generation rate, storage
+load and retention needs. The interval controls scheduling checks, not guaranteed cleanup
+completion frequency. Checks also respect the last successful execution time, so asynchronous
+execution can make the effective period approach twice the configured interval; queuing and
+failures can extend it further. One process attempt makes one native call, without an unbounded
+cleanup loop. The limit applies to each invocation, not the cumulative work across retries.
+
+The table's **Cleanup** tab lists AMS tasks and their latest-attempt summaries:
+
+- `outcome=executed` or `noop` means the native method returned a positive count or zero.
+ `reported-expired-snapshots` is Paimon's reported snapshot result, not a verified deleted-file
+ count or reclaimed byte count. Normal return does not prove every file deletion succeeded.
+- `outcome=skipped` means eligibility changed before execution; no expiration was invoked and
+ the last successful cleanup time is unchanged.
+- `error-stage` and `error-message` identify a failure. If expiration returns but recording the
+ last-success time fails, the task can fail while retaining the completed invocation's result.
+ Retrying replaces the previous attempt's summary; this is not a complete attempt history.
+
+Expand a task to see its summary and, with write permission, cancel a running task. Cancellation
+is routed through AMS ProcessService for both Paimon and Iceberg cleanup tasks. A request that
+cannot cancel a tracked active task returns an error rather than reporting success. Iceberg
+self-optimizing tasks retain their existing cancellation and partial-commit behavior. A failed
+attempt that still has retries remaining can be canceled before its retry starts. If cancellation
+wins before the engine identifier is recorded, the executor forwards cancellation once that
+identifier becomes available; a task still queued at that point will not run. Cancellation
+uses the same framework semantics as Iceberg: it does not undo completed deletion or guarantee
+that underlying file operations immediately stop. Failures use the existing bounded retries
+(up to three additional attempts per process); later scheduling checks can create a new process.
+Recovery repeats the operation with current settings and does not guarantee exactly-once deletion.