From 9d44c32408cf01432d0f0997a83bec7cf3b5893f Mon Sep 17 00:00:00 2001 From: csurong Date: Tue, 17 Nov 2026 14:09:45 +0800 Subject: [PATCH] Support Paimon snapshot expiration --- .github/workflows/core-hadoop3-ci.yml | 44 + .gitignore | 4 + amoro-ams/pom.xml | 57 + .../amoro/server/AmoroServiceContainer.java | 1 + .../server/DefaultOptimizingService.java | 18 +- .../MixedAndIcebergTableDescriptor.java | 5 + .../dashboard/ServerTableDescriptor.java | 10 + .../server/dashboard/TableProcessQuery.java | 114 ++ .../dashboard/controller/TableController.java | 5 +- .../process/DefaultTableProcessStore.java | 12 +- .../amoro/server/process/ProcessService.java | 53 +- .../executor/TableProcessExecutor.java | 224 ++-- .../amoro/server/AMSServiceTestBase.java | 10 +- .../server/TestDefaultOptimizingService.java | 35 + .../server/TestOptimizerScaleKeeper.java | 16 +- .../dashboard/TestTableProcessQuery.java | 187 +++ .../controller/TestProcessCancellation.java | 392 +++++++ .../TestProcessCancellationConcurrency.java | 451 +++++++ .../TestTableExpirationConfiguration.java | 65 ++ .../process/PaimonWarehouseLifecycle.java | 47 + .../process/TestPaimonProcessIntegration.java | 1036 +++++++++++++++++ .../java/org/apache/amoro/PaimonActions.java | 26 + .../amoro/process/LocalExecutionEngine.java | 3 +- .../amoro/formats/paimon/PaimonCatalog.java | 11 +- .../amoro/formats/paimon/PaimonTable.java | 15 + .../formats/paimon/PaimonTableDescriptor.java | 13 + .../paimon/process/PaimonProcessFactory.java | 181 +++ .../process/PaimonSnapshotExpiration.java | 108 ++ .../PaimonSnapshotsExpiringProcess.java | 112 ++ .../org.apache.amoro.process.ProcessFactory | 19 + .../paimon/TestPaimonTableRefresh.java | 69 ++ .../process/TestPaimonSnapshotExpiration.java | 707 +++++++++++ .../views/tables/components/Optimizing.vue | 19 +- .../conf/plugins/execute-engines.yaml | 3 +- .../conf/plugins/process-factories.yaml | 9 + docs/formats/paimon.md | 100 ++ 36 files changed, 4038 insertions(+), 143 deletions(-) create mode 100644 amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java create mode 100644 amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java create mode 100644 amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java create mode 100644 amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java create mode 100644 amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java create mode 100644 amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java create mode 100644 amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java create mode 100644 amoro-common/src/main/java/org/apache/amoro/PaimonActions.java create mode 100644 amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java create mode 100644 amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java create mode 100644 amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java create mode 100644 amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory create mode 100644 amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java create mode 100644 amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java diff --git a/.github/workflows/core-hadoop3-ci.yml b/.github/workflows/core-hadoop3-ci.yml index aed6c0e462..cd5dbe0e2d 100644 --- a/.github/workflows/core-hadoop3-ci.yml +++ b/.github/workflows/core-hadoop3-ci.yml @@ -67,12 +67,56 @@ jobs: if: matrix.spark == '3.3' run: ./mvnw clean install -pl '!amoro-ams,!dist' -Djacoco.flink.skip=true -B -ntp -Pjava${{ matrix.jdk }} -Pspark-${{ matrix.spark }} + # Standard-directory tests already run in the full build above. + - name: Test Paimon AMS integration + if: matrix.spark == '3.5' + run: | + ./mvnw test -pl amoro-ams -am -B -ntp \ + -Pjava${{ matrix.jdk }},support-paimon-format,spark-3.5,skip-dashboard-build \ + -Dtest=TestPaimonProcessIntegration \ + -Dsurefire.failIfNoSpecifiedTests=false + python3 - <<'PYTHON' + import xml.etree.ElementTree as ET + report = ET.parse('amoro-ams/target/surefire-reports/TEST-org.apache.amoro.server.process.TestPaimonProcessIntegration.xml').getroot() + assert int(report.get('tests', '0')) > 0, 'Paimon AMS integration tests did not run' + assert all(int(report.get(key, '0')) == 0 for key in ('failures', 'errors', 'skipped')), report.attrib + PYTHON + - name: Code coverage uses: codecov/codecov-action@v3 with: verbose: true flags: core + paimon-1-1-1: + name: Paimon 1.1.1 format regression with JDK 17 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + check-latest: false + - name: Test Paimon 1.1.1 and Scala 2.13 + run: | + ./mvnw clean test -pl amoro-format-paimon -am -B -ntp \ + -Pjava17,scala-2.13 \ + -Dtest=TestPaimonSnapshotExpiration,TestPaimonTableRefresh \ + -Dsurefire.failIfNoSpecifiedTests=false + python3 - <<'PYTHON' + import xml.etree.ElementTree as ET + for test_class in ( + 'org.apache.amoro.formats.paimon.process.TestPaimonSnapshotExpiration', + 'org.apache.amoro.formats.paimon.TestPaimonTableRefresh', + ): + report = ET.parse(f'amoro-format-paimon/target/surefire-reports/TEST-{test_class}.xml').getroot() + assert int(report.get('tests', '0')) > 0, f'{test_class} did not run' + assert all(int(report.get(key, '0')) == 0 for key in ('failures', 'errors', 'skipped')), report.attrib + PYTHON + spark-optimizer-4-1: name: Build Spark 4.1 Optimizer with JDK 17 runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index a88a820d98..3792035891 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,10 @@ resources !javadoc/resources !amoro-ams/src/main/resources/ !amoro-ams/src/main/resources/** +!amoro-format-paimon/src/main/resources/ +!amoro-format-paimon/src/main/resources/META-INF/ +!amoro-format-paimon/src/main/resources/META-INF/services/ +!amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory !dist/src/main/amoro-bin/bin/ !dist/src/main/amoro-bin/conf/ diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml index 9ba57d0011..4a51b32e9b 100644 --- a/amoro-ams/pom.xml +++ b/amoro-ams/pom.xml @@ -518,6 +518,12 @@ ${swagger-ui.version} + + org.junit.platform + junit-platform-launcher + test + + org.testcontainers @@ -603,6 +609,41 @@ + + + + org.codehaus.mojo + build-helper-maven-plugin + ${maven-build-helper-plugin.version} + + + paimon-integration-tests + + add-test-source + + generate-test-sources + + + src/test/paimon + + + + + + + com.diffplug.spotless + spotless-maven-plugin + + + + src/main/java/**/*.java + src/test/**/*.java + + + + + + pl.project13.maven @@ -706,6 +747,14 @@ amoro-format-paimon + + + + org.codehaus.mojo + build-helper-maven-plugin + + + support-hudi-format @@ -757,6 +806,14 @@ amoro-format-lance + + + + org.codehaus.mojo + build-helper-maven-plugin + + + fail-on-no-git-dir diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java index c5829edbb4..37fa4a51fc 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java @@ -261,6 +261,7 @@ private void startOptimizingService() throws Exception { catalogManager, optimizerManager, tableService, + processService, bucketAssignStore, haContainer); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java index afccf8a970..e5d4688714 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java @@ -54,6 +54,7 @@ import org.apache.amoro.server.persistence.mapper.OptimizerMapper; import org.apache.amoro.server.persistence.mapper.ResourceMapper; import org.apache.amoro.server.persistence.mapper.TableProcessMapper; +import org.apache.amoro.server.process.ProcessService; import org.apache.amoro.server.process.TableProcessMeta; import org.apache.amoro.server.resource.Containers; import org.apache.amoro.server.resource.OptimizerInstance; @@ -137,6 +138,7 @@ public class DefaultOptimizingService extends StatedPersistentBase private final CatalogManager catalogManager; private final OptimizerManager optimizerManager; private final TableService tableService; + private final ProcessService processService; private final RuntimeHandlerChain tableHandlerChain; private final ExecutorService planExecutor; private final BucketAssignStore bucketAssignStore; @@ -148,6 +150,7 @@ public DefaultOptimizingService( CatalogManager catalogManager, OptimizerManager optimizerManager, TableService tableService, + ProcessService processService, BucketAssignStore bucketAssignStore, HighAvailabilityContainer haContainer) { this.optimizerTouchTimeout = @@ -170,6 +173,7 @@ public DefaultOptimizingService( this.groupMaxKeepingAttempts = serviceConfig.getInteger(AmoroManagementConf.OPTIMIZER_GROUP_MAX_KEEPING_ATTEMPTS); this.tableService = tableService; + this.processService = processService; this.catalogManager = catalogManager; this.optimizerManager = optimizerManager; this.bucketAssignStore = bucketAssignStore; @@ -499,16 +503,18 @@ public boolean cancelProcess(long processId) { return false; } long tableId = processMeta.getTableId(); - DefaultTableRuntime tableRuntime = (DefaultTableRuntime) tableService.getRuntime(tableId); + TableRuntime tableRuntime = tableService.getRuntime(tableId); if (tableRuntime == null) { return false; } - OptimizingProcess process = tableRuntime.getOptimizingProcess(); - if (process == null || process.getProcessId() != processId) { - return false; + if (tableRuntime instanceof DefaultTableRuntime) { + OptimizingProcess process = ((DefaultTableRuntime) tableRuntime).getOptimizingProcess(); + if (process != null && process.getProcessId() == processId) { + process.close(true); + return true; + } } - process.close(true); - return true; + return processService.cancelProcess(tableRuntime.getTableIdentifier(), processId); } @Override diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java index 2db674fd54..52e9f2a744 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java @@ -687,6 +687,11 @@ public Pair, Integer> getOptimizingProcessesInfo( if (identifier == null) { return Pair.of(Collections.emptyList(), 0); } + if (TableProcessQuery.handles(this, processCategory, type)) { + return new TableProcessQuery() + .list( + identifier, getProcessTypesByCategory(processCategory), type, status, limit, offset); + } int total = 0; // page helper is 1-based int pageNumber = (offset / limit) + 1; diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java index bbd68bf691..ba7f6fd92a 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java @@ -137,6 +137,16 @@ public Pair, Integer> getOptimizingProcessesInfo( int offset) { AmoroTable amoroTable = loadTable(tableIdentifier); FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format()); + if (TableProcessQuery.handles(formatTableDescriptor, processCategory, type)) { + return new TableProcessQuery() + .list( + amoroTable.id(), + formatTableDescriptor.getProcessTypesByCategory(processCategory), + type, + status, + limit, + offset); + } return formatTableDescriptor.getOptimizingProcessesInfo( amoroTable, type, processCategory, status, limit, offset); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java new file mode 100644 index 0000000000..61653cba1f --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/TableProcessQuery.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.dashboard; + +import com.github.pagehelper.Page; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import org.apache.amoro.ServerTableIdentifier; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.server.persistence.PersistentBase; +import org.apache.amoro.server.persistence.mapper.TableMetaMapper; +import org.apache.amoro.server.persistence.mapper.TableProcessMapper; +import org.apache.amoro.server.process.TableProcessMeta; +import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.table.descriptor.FilesStatistics; +import org.apache.amoro.table.descriptor.FormatTableDescriptor; +import org.apache.amoro.table.descriptor.OptimizingProcessInfo; +import org.apache.amoro.table.descriptor.ProcessCategory; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** Format-independent queries of AMS cleanup records, without optimizing task metrics. */ +public class TableProcessQuery extends PersistentBase { + static boolean handles(FormatTableDescriptor descriptor, String category, String type) { + return ProcessCategory.CLEANUP.getName().equalsIgnoreCase(category) + && (StringUtils.isBlank(type) + || descriptor.getProcessTypesByCategory(category).contains(type)); + } + + Pair, Integer> list( + TableIdentifier table, + List actions, + String type, + ProcessStatus status, + int limit, + int offset) { + ServerTableIdentifier identifier = + getAs( + TableMetaMapper.class, + mapper -> + mapper.selectTableIdentifier( + table.getCatalog(), table.getDatabase(), table.getTableName())); + return list(identifier, actions, type, status, limit, offset); + } + + public Pair, Integer> list( + ServerTableIdentifier table, + List actions, + String type, + ProcessStatus status, + int limit, + int offset) { + String filterType = StringUtils.isBlank(type) ? null : type; + if (table == null || (filterType == null && (actions == null || actions.isEmpty()))) { + return Pair.of(Collections.emptyList(), 0); + } + try (Page ignored = PageHelper.startPage(offset / limit + 1, limit, true)) { + List records = + getAs( + TableProcessMapper.class, + mapper -> + mapper.listProcessMeta( + table.getId(), filterType, filterType == null ? actions : null, status)); + PageInfo page = new PageInfo<>(records); + if (page.getSize() == 0) { + return Pair.of(Collections.emptyList(), 0); + } + return Pair.of( + records.stream().map(meta -> toInfo(table, meta)).collect(Collectors.toList()), + (int) page.getTotal()); + } + } + + private static OptimizingProcessInfo toInfo(ServerTableIdentifier table, TableProcessMeta meta) { + OptimizingProcessInfo result = new OptimizingProcessInfo(); + result.setTableId(meta.getTableId()); + result.setCatalogName(table.getCatalog()); + result.setDbName(table.getDatabase()); + result.setTableName(table.getTableName()); + result.setProcessId(String.valueOf(meta.getProcessId())); + result.setStartTime(meta.getCreateTime()); + result.setFinishTime(meta.getFinishTime()); + result.setDuration( + (meta.getFinishTime() > 0 ? meta.getFinishTime() : System.currentTimeMillis()) + - meta.getCreateTime()); + result.setOptimizingType(meta.getProcessType()); + result.setStatus(meta.getStatus()); + result.setFailReason(meta.getFailMessage()); + result.setInputFiles(new FilesStatistics()); + result.setOutputFiles(new FilesStatistics()); + result.setSummary(meta.getSummary()); + return result; + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index a2219f7370..4ef746cad9 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -706,7 +706,10 @@ public void cancelOptimizingProcess(Context ctx) { OptimizingClientPools.getClient( AmsUtil.getAMSThriftAddress(serviceConfig, Constants.THRIFT_OPTIMIZING_SERVICE_NAME)); try { - client.cancelProcess(processId); + Preconditions.checkState( + client.cancelProcess(processId), + "Process could not be canceled: it is no longer active or is not managed by this AMS, processId: %s", + processId); } catch (TException e) { throw new IllegalStateException("Failed to cancel optimizing process:" + e.getMessage()); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java index 0dfce34f9e..e4c74997f0 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/DefaultTableProcessStore.java @@ -284,7 +284,8 @@ protected boolean validTransition( || getStatus() == ProcessStatus.PENDING) && newStatus == ProcessStatus.FAILED); case CANCEL_REQUESTED: - return (getStatus() == ProcessStatus.UNKNOWN + return (getStatus() == ProcessStatus.FAILED + || getStatus() == ProcessStatus.UNKNOWN || getStatus() == ProcessStatus.SUBMITTED || getStatus() == ProcessStatus.RUNNING || getStatus() == ProcessStatus.PENDING) @@ -324,11 +325,6 @@ public boolean tryTransitState( transitionLock.lock(); try { int priority = processEvent.getPriority(); - if (priority >= ProcessEvent.KILL_REQUESTED.getPriority()) { - highestPriority.updateAndGet(curr -> Math.max(curr, priority)); - terminalRequested = true; - } - ProcessStatus currentStatus = getStatus(); if (isTerminal(currentStatus)) { LOG.warn( @@ -393,6 +389,10 @@ public boolean tryTransitState( default: return false; } + if (priority >= ProcessEvent.KILL_REQUESTED.getPriority()) { + highestPriority.updateAndGet(curr -> Math.max(curr, priority)); + terminalRequested = true; + } } finally { transitionLock.unlock(); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java index bf44c64808..b21fb8f737 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java @@ -126,6 +126,15 @@ public void dispose() { activeTableProcess.clear(); } + /** + * Await worker termination after {@link #dispose()}, without initiating shutdown or cancellation. + * Tests use this to keep persistence available until completion callbacks have returned. + */ + @VisibleForTesting + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return processExecutionPool.awaitTermination(timeout, unit); + } + /** * Find the scheduler that supports the given table format for the specified action. * @@ -410,32 +419,45 @@ private void executeOrTraceProcess(TableProcessStore store, TableProcess process } /** - * Cancel the given process and untrack it. + * Cancel a process owned by this service for the given table, using the existing lifecycle. * - * @param process table process + * @param table table identity + * @param processId process identifier + * @return true if the tracked process transitioned to canceled, false if absent or terminal */ - private void cancelProcess(TableProcessStore store, TableProcess process) { - - store.tryTransitState( - ProcessStatus.CANCELED, - ProcessEvent.CANCEL_REQUESTED, - store.getExternalProcessIdentifier(), - "Gracefully Canceled.", - process.getProcessParameters(), - process.getSummary()); + public boolean cancelProcess(ServerTableIdentifier table, long processId) { + TableProcessHolder holder = getTableProcessInstances(table).get(processId); + return holder != null && cancelProcess(holder.getStore(), holder.getProcess()); + } + + private boolean cancelProcess(TableProcessStore store, TableProcess process) { + + boolean canceled = + store.tryTransitState( + ProcessStatus.CANCELED, + ProcessEvent.CANCEL_REQUESTED, + store.getExternalProcessIdentifier(), + "Gracefully Canceled.", + process.getProcessParameters(), + process.getSummary()); + if (!canceled) { + return false; + } untrackTableProcessInstance( process.getTableRuntime().getTableIdentifier(), store.getProcessId()); ExecuteEngine executeEngine = executeEngines.get(store.getExecutionEngine()); - if (executeEngine != null) { - executeEngine.tryCancelTableProcess(process, store.getExternalProcessIdentifier()); + String externalIdentifier = store.getExternalProcessIdentifier(); + if (executeEngine != null && externalIdentifier != null && !externalIdentifier.isEmpty()) { + executeEngine.tryCancelTableProcess(process, externalIdentifier); LOG.info( "Cancel table process {} in engine {}, process id:{}", process, executeEngine.name(), store.getProcessId()); } + return canceled; } /** @@ -700,7 +722,10 @@ protected void handleTableRemoved(TableRuntime tableRuntime) { .collect(Collectors.toList()); for (TableProcessHolder holder : processes) { if (holder != null) { - cancelProcess(holder.getStore(), holder.getProcess()); + if (!cancelProcess(holder.getStore(), holder.getProcess())) { + untrackTableProcessInstance( + tableRuntime.getTableIdentifier(), holder.getStore().getProcessId()); + } } } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java index b16517e3a0..e9f67f07f3 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/TableProcessExecutor.java @@ -70,119 +70,139 @@ public void run() { } try { - if (store.getStatus() == ProcessStatus.UNKNOWN - || store.getStatus() == ProcessStatus.PENDING - || Strings.isNullOrEmpty(store.getExternalProcessIdentifier())) { - externalProcessIdentifier = executeEngine.submitTableProcess(tableProcess); - LOG.info( - "Submit table process {} to engine {} success, external process identifier is {}", - store.getProcessId(), - executeEngine.engineType(), - externalProcessIdentifier); - } else { - externalProcessIdentifier = store.getExternalProcessIdentifier(); - } - - validateIdentifier(externalProcessIdentifier); - - status = executeEngine.getStatus(externalProcessIdentifier); - - // If the engine returns UNKNOWN, the process was lost (e.g., AMS restart cleared - // LocalExecutionEngine's in-memory process map). Treat it as FAILED so the process - // is properly terminated and optionally retried, rather than being stuck in UNKNOWN. - if (status == ProcessStatus.UNKNOWN) { - LOG.warn( - "Table process {} got UNKNOWN status from engine (identifier={}), " - + "likely due to AMS restart or process loss, marking as FAILED", - store.getProcessId(), - externalProcessIdentifier); - status = ProcessStatus.FAILED; - message = "Process lost: engine returned UNKNOWN status"; - } else { - store.tryTransitState( - status, - ProcessEvent.SUBMIT_REQUESTED, - externalProcessIdentifier, - "Complete Submitted.", - tableProcess.getProcessParameters(), - tableProcess.getSummary()); - } + try { + if (store.getStatus() == ProcessStatus.UNKNOWN + || store.getStatus() == ProcessStatus.PENDING + || Strings.isNullOrEmpty(store.getExternalProcessIdentifier())) { + externalProcessIdentifier = executeEngine.submitTableProcess(tableProcess); + LOG.info( + "Submit table process {} to engine {} success, external process identifier is {}", + store.getProcessId(), + executeEngine.engineType(), + externalProcessIdentifier); + } else { + externalProcessIdentifier = store.getExternalProcessIdentifier(); + } - while (isTableProcessExecuting(status)) { + validateIdentifier(externalProcessIdentifier); if (isTableProcessCanceling(store.getStatus())) { + return; + } + + status = executeEngine.getStatus(externalProcessIdentifier); + + // If the engine returns UNKNOWN, the process was lost (e.g., AMS restart cleared + // LocalExecutionEngine's in-memory process map). Treat it as FAILED so the process + // is properly terminated and optionally retried, rather than being stuck in UNKNOWN. + if (status == ProcessStatus.UNKNOWN) { + LOG.warn( + "Table process {} got UNKNOWN status from engine (identifier={}), " + + "likely due to AMS restart or process loss, marking as FAILED", + store.getProcessId(), + externalProcessIdentifier); + status = ProcessStatus.FAILED; + message = "Process lost: engine returned UNKNOWN status"; + } else { + store.tryTransitState( + status, + ProcessEvent.SUBMIT_REQUESTED, + externalProcessIdentifier, + "Complete Submitted.", + tableProcess.getProcessParameters(), + tableProcess.getSummary()); + } + + while (isTableProcessExecuting(status)) { + if (isTableProcessCanceling(store.getStatus())) { + LOG.info( + "Table process {} with identifier {} may have been in canceling, exit submit process.", + store.getProcessId(), + externalProcessIdentifier); + return; + } + try { + Thread.sleep(DEFAULT_POLL_INTERVAL_MS); + } catch (InterruptedException e) { + throw e; + } + status = executeEngine.getStatus(externalProcessIdentifier); + } + } catch (Throwable t) { + if (t instanceof InterruptedException) { LOG.info( - "Table process {} with identifier {} may have been in canceling, exit submit process.", + "Table process {} with identifier {} may have been interrupted by process service disposing, exit submit process.", store.getProcessId(), externalProcessIdentifier); return; + } else { + status = ProcessStatus.FAILED; + message = t.getMessage(); } - try { - Thread.sleep(DEFAULT_POLL_INTERVAL_MS); - } catch (InterruptedException e) { - throw e; - } - status = executeEngine.getStatus(externalProcessIdentifier); } - } catch (Throwable t) { - if (t instanceof InterruptedException) { - LOG.info( - "Table process {} with identifier {} may have been interrupted by process service disposing, exit submit process.", - store.getProcessId(), - externalProcessIdentifier); - return; + + LOG.info("The process {} is finished with status {}", tableProcess, status); + if (status == ProcessStatus.KILLED) { + store.tryTransitState( + status, + ProcessEvent.KILL_REQUESTED, + store.getExternalProcessIdentifier(), + "Gracefully Killed.", + tableProcess.getProcessParameters(), + tableProcess.getSummary()); + } else if (status == ProcessStatus.CANCELED) { + store.tryTransitState( + status, + ProcessEvent.CANCEL_REQUESTED, + store.getExternalProcessIdentifier(), + "Gracefully Cancelled.", + tableProcess.getProcessParameters(), + tableProcess.getSummary()); + } else if (status == ProcessStatus.CLOSED) { + store.tryTransitState( + status, + ProcessEvent.KILL_REQUESTED, + store.getExternalProcessIdentifier(), + "Gracefully Closed.", + tableProcess.getProcessParameters(), + tableProcess.getSummary()); + } else if (status == ProcessStatus.FAILED) { + store.tryTransitState( + status, + ProcessEvent.COMPLETE_FAILED, + store.getExternalProcessIdentifier(), + message, + tableProcess.getProcessParameters(), + tableProcess.getSummary()); + } else if (status == ProcessStatus.SUCCESS) { + store.tryTransitState( + status, + ProcessEvent.COMPLETE_SUCCESS, + store.getExternalProcessIdentifier(), + "Complete Success", + tableProcess.getProcessParameters(), + tableProcess.getSummary()); } else { - status = ProcessStatus.FAILED; - message = t.getMessage(); + LOG.warn("Un expected terminal status: {} for process: {}.", status, store.getProcessId()); } - } - LOG.info("The process {} is finished with status {}", tableProcess, status); - if (status == ProcessStatus.KILLED) { - store.tryTransitState( - status, - ProcessEvent.KILL_REQUESTED, - store.getExternalProcessIdentifier(), - "Gracefully Killed.", - tableProcess.getProcessParameters(), - tableProcess.getSummary()); - } else if (status == ProcessStatus.CANCELED) { - store.tryTransitState( - status, - ProcessEvent.CANCEL_REQUESTED, - store.getExternalProcessIdentifier(), - "Gracefully Cancelled.", - tableProcess.getProcessParameters(), - tableProcess.getSummary()); - } else if (status == ProcessStatus.CLOSED) { - store.tryTransitState( - status, - ProcessEvent.KILL_REQUESTED, - store.getExternalProcessIdentifier(), - "Gracefully Closed.", - tableProcess.getProcessParameters(), - tableProcess.getSummary()); - } else if (status == ProcessStatus.FAILED) { - store.tryTransitState( - status, - ProcessEvent.COMPLETE_FAILED, - store.getExternalProcessIdentifier(), - message, - tableProcess.getProcessParameters(), - tableProcess.getSummary()); - } else if (status == ProcessStatus.SUCCESS) { - store.tryTransitState( - status, - ProcessEvent.COMPLETE_SUCCESS, - store.getExternalProcessIdentifier(), - "Complete Success", - tableProcess.getProcessParameters(), - tableProcess.getSummary()); - } else { - LOG.warn("Un expected terminal status: {} for process: {}.", status, store.getProcessId()); - } - - if (finishedCallback != null) { - finishedCallback.run(); + if (finishedCallback != null) { + finishedCallback.run(); + } + } finally { + // Cancellation can win before submit returns or before its identifier is persisted. + // Always hand the identifier obtained by this attempt to the engine before exiting. + if (!Strings.isNullOrEmpty(externalProcessIdentifier) + && isTableProcessCanceling(store.getStatus())) { + try { + executeEngine.tryCancelTableProcess(tableProcess, externalProcessIdentifier); + } catch (Exception e) { + LOG.error( + "Failed to cancel submitted process {} with identifier {}", + store.getProcessId(), + externalProcessIdentifier, + e); + } + } } } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java b/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java index 9c2abe9698..9015cb2697 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java @@ -61,10 +61,16 @@ protected static void initTableService(Duration optimizerHeartbeatTimeout) { Duration.ofMillis(10L)); TABLE_SERVICE = new DefaultTableService(new Configurations(), CATALOG_MANAGER, runtimeFactory); + PROCESS_SERVICE = new ProcessService(TABLE_SERVICE); OPTIMIZING_SERVICE = new DefaultOptimizingService( - configurations, CATALOG_MANAGER, OPTIMIZER_MANAGER, TABLE_SERVICE, null, null); - PROCESS_SERVICE = new ProcessService(TABLE_SERVICE); + configurations, + CATALOG_MANAGER, + OPTIMIZER_MANAGER, + TABLE_SERVICE, + PROCESS_SERVICE, + null, + null); TABLE_SERVICE.addHandlerChain(OPTIMIZING_SERVICE.getTableRuntimeHandler()); TABLE_SERVICE.addHandlerChain(PROCESS_SERVICE.getTableHandlerChain()); diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java index e146a57eea..21dc77cfc9 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java @@ -49,6 +49,7 @@ import org.apache.amoro.process.ProcessStatus; import org.apache.amoro.resource.ResourceGroup; import org.apache.amoro.server.manager.MetricManager; +import org.apache.amoro.server.optimizing.OptimizingProcess; import org.apache.amoro.server.optimizing.OptimizingQueue; import org.apache.amoro.server.optimizing.OptimizingStatus; import org.apache.amoro.server.optimizing.TaskRuntime; @@ -66,6 +67,7 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableProperties; import org.apache.amoro.table.UnkeyedTable; import org.apache.amoro.utils.SerializationUtil; import org.apache.ibatis.session.SqlSession; @@ -167,6 +169,39 @@ private void appendData(UnkeyedTable table, int id) { appendFiles.commit(); } + @Test + public void testCancelProcessKeepsOptimizingLifecycle() { + OptimizingTask task = optimizingService().pollTask(token, THREAD_ID); + Assertions.assertNotNull(task); + DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); + Assertions.assertFalse(runtime.isAllowPartialCommit()); + OptimizingProcess process = runtime.getOptimizingProcess(); + Assertions.assertTrue(optimizingService().cancelProcess(process.getProcessId())); + Assertions.assertEquals(ProcessStatus.CLOSED, process.getStatus()); + Assertions.assertNull(runtime.getOptimizingProcess()); + Assertions.assertEquals(OptimizingStatus.IDLE, runtime.getOptimizingStatus()); + Assertions.assertFalse(optimizingService().cancelProcess(process.getProcessId())); + } + + @Test + public void testCancelProcessKeepsPartialCommit() { + MixedTable table = + (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable(); + table + .asUnkeyedTable() + .updateProperties() + .set(TableProperties.SELF_OPTIMIZING_ALLOW_PARTIAL_COMMIT, "true") + .commit(); + DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); + runtime.refresh(tableService().loadTable(serverTableIdentifier())); + Assertions.assertTrue(runtime.isAllowPartialCommit()); + Assertions.assertNotNull(optimizingService().pollTask(token, THREAD_ID)); + OptimizingProcess process = runtime.getOptimizingProcess(); + Assertions.assertTrue(optimizingService().cancelProcess(process.getProcessId())); + Assertions.assertEquals(OptimizingStatus.COMMITTING, runtime.getOptimizingStatus()); + Assertions.assertFalse(process.isClosed()); + } + @Test public void testPollWithoutAuth() { // 1.poll task diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java b/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java index 81f71da8d7..e1e841f6d1 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java @@ -722,7 +722,13 @@ public void testDisposeUnregistersDraMetrics() { optimizerManager().createResourceGroup(group); DefaultOptimizingService formerLeader = new DefaultOptimizingService( - new Configurations(), catalogManager(), optimizerManager(), tableService(), null, null); + new Configurations(), + catalogManager(), + optimizerManager(), + tableService(), + processServiceService(), + null, + null); formerLeader.createResourceGroup(group); Assertions.assertNotNull( draMetric(DynamicAllocationMetrics.OPTIMIZER_GROUP_SCALE_UP_TOTAL, group.getName())); @@ -750,7 +756,13 @@ public void testWatchAfterDisposeDoesNotRegisterMetrics() { ResourceGroup group = buildSlowDraResourceGroup(TEST_GROUP_NAME + "-19", 0); DefaultOptimizingService formerLeader = new DefaultOptimizingService( - new Configurations(), catalogManager(), optimizerManager(), tableService(), null, null); + new Configurations(), + catalogManager(), + optimizerManager(), + tableService(), + processServiceService(), + null, + null); formerLeader.dispose(); formerLeader.updateResourceGroup(group); diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java new file mode 100644 index 0000000000..572d3b28d9 --- /dev/null +++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestTableProcessQuery.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.dashboard; + +import org.apache.amoro.AmoroTable; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.ServerTableIdentifier; +import org.apache.amoro.TableFormat; +import org.apache.amoro.config.Configurations; +import org.apache.amoro.optimizing.MetricsSummary; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.server.catalog.CatalogManager; +import org.apache.amoro.server.catalog.ServerCatalog; +import org.apache.amoro.server.persistence.PersistentBase; +import org.apache.amoro.server.persistence.mapper.TableMetaMapper; +import org.apache.amoro.server.persistence.mapper.TableProcessMapper; +import org.apache.amoro.server.table.DerbyPersistence; +import org.apache.amoro.server.table.TableManager; +import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.table.descriptor.OptimizingProcessInfo; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +class TestTableProcessQuery extends PersistentBase { + private final Persistence persistence = new Persistence(); + private final ServerTableIdentifier table = + ServerTableIdentifier.of(1L, "catalog", "db", "tbl", TableFormat.PAIMON); + private final String action = PaimonActions.EXPIRE_SNAPSHOTS.getName(); + + @AfterEach + void cleanup() { + persistence.clear(); + } + + @Test + void cleanupQueryPreservesResultsAndLimitsEveryQueryToTargetTable() { + doAs(TableMetaMapper.class, mapper -> mapper.insertTable(table)); + insert(table.getId(), 11, action, ProcessStatus.SUCCESS); + insert(table.getId(), 12, action, ProcessStatus.FAILED); + insert(table.getId(), 13, "MINOR", ProcessStatus.SUCCESS); + insert(table.getId() + 1, 14, action, ProcessStatus.SUCCESS); + TableProcessQuery query = new TableProcessQuery(); + Pair, Integer> page = + query.list(table, List.of(action), " ", null, 1, 0); + Assertions.assertEquals(2, page.getRight()); + OptimizingProcessInfo result = page.getLeft().get(0); + Assertions.assertEquals("12", result.getProcessId()); + Assertions.assertEquals("catalog", result.getCatalogName()); + Assertions.assertEquals("db", result.getDbName()); + Assertions.assertEquals("tbl", result.getTableName()); + Assertions.assertEquals(Map.of("outcome", "noop"), result.getSummary()); + Assertions.assertEquals(0, result.getTotalTasks()); + Assertions.assertNotNull(result.getInputFiles()); + Assertions.assertNotNull(result.getOutputFiles()); + Assertions.assertEquals( + "11", + query + .list(table, List.of(action), null, ProcessStatus.SUCCESS, 10, 0) + .getLeft() + .get(0) + .getProcessId()); + Assertions.assertEquals( + 0, query.list(table, Collections.emptyList(), null, null, 10, 0).getRight()); + Assertions.assertEquals( + 0, query.list(table, List.of(action), "unknown", null, 10, 0).getRight()); + Assertions.assertEquals(0, query.list(table, List.of(action), null, null, 10, 50).getRight()); + } + + @Test + void serverRoutingRetainsIcebergTypePriorityAndOptimizationMetrics() { + doAs(TableMetaMapper.class, mapper -> mapper.insertTable(table)); + insert(table.getId(), 11, action, ProcessStatus.SUCCESS); + Map metrics = + Map.of( + MetricsSummary.INPUT_DATA_FILES, + "4", + MetricsSummary.INPUT_DATA_SIZE, + "40", + MetricsSummary.OUTPUT_DATA_FILES, + "1", + MetricsSummary.OUTPUT_DATA_SIZE, + "10"); + doAs( + TableProcessMapper.class, + mapper -> + mapper.insertProcess( + table.getId(), + 12L, + "", + ProcessStatus.SUCCESS, + "MINOR", + "default", + "local", + 0, + 1000L, + Map.of(), + metrics)); + AmoroTable iceberg = Mockito.mock(AmoroTable.class); + Mockito.when(iceberg.id()).thenReturn(TableIdentifier.of("catalog", "db", "tbl")); + Mockito.when(iceberg.format()).thenReturn(TableFormat.ICEBERG); + CatalogManager catalogs = Mockito.mock(CatalogManager.class); + ServerCatalog catalog = Mockito.mock(ServerCatalog.class); + Mockito.when(catalogs.getServerCatalog("catalog")).thenReturn(catalog); + Mockito.doReturn(iceberg).when(catalog).loadTable("db", "tbl"); + ServerTableDescriptor descriptor = + new ServerTableDescriptor(catalogs, Mockito.mock(TableManager.class), new Configurations()); + org.apache.amoro.api.TableIdentifier id = iceberg.id().buildTableIdentifier(); + Assertions.assertEquals( + "11", + descriptor + .getOptimizingProcessesInfo(id, null, "CLEANUP", null, 10, 0) + .getLeft() + .get(0) + .getProcessId()); + for (String category : new String[] {"CLEANUP", "OPTIMIZING", null}) { + OptimizingProcessInfo minor = + descriptor + .getOptimizingProcessesInfo(id, "MINOR", category, null, 10, 0) + .getLeft() + .get(0); + Assertions.assertEquals("12", minor.getProcessId()); + Assertions.assertEquals(4, minor.getInputFiles().getFileCnt()); + Assertions.assertEquals(10L, minor.getOutputFiles().getTotalSize()); + } + Assertions.assertEquals( + 0, descriptor.getOptimizingProcessesInfo(id, "unknown", "CLEANUP", null, 10, 0).getRight()); + Assertions.assertEquals( + 0, descriptor.getOptimizingProcessesInfo(id, null, null, null, 10, 0).getRight()); + Assertions.assertEquals( + 0, descriptor.getOptimizingProcessesInfo(id, null, "unknown", null, 10, 0).getRight()); + Mockito.when(iceberg.id()).thenReturn(TableIdentifier.of("catalog", "db", "unregistered")); + Assertions.assertEquals( + 0, descriptor.getOptimizingProcessesInfo(id, null, "CLEANUP", null, 10, 0).getRight()); + Mockito.when(catalog.loadTable("db", "tbl")) + .thenThrow(new IllegalStateException("table unavailable")); + Assertions.assertThrows( + IllegalStateException.class, + () -> descriptor.getOptimizingProcessesInfo(id, null, "CLEANUP", null, 10, 0)); + } + + private void insert(long tableId, long processId, String type, ProcessStatus status) { + doAs( + TableProcessMapper.class, + mapper -> + mapper.insertProcess( + tableId, + processId, + "", + status, + type, + "default", + "local", + 0, + 1000L, + Collections.emptyMap(), + Map.of("outcome", "noop"))); + } + + private static class Persistence extends DerbyPersistence { + void clear() { + super.after(); + } + } +} diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java new file mode 100644 index 0000000000..f53f9c8102 --- /dev/null +++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/controller/TestProcessCancellation.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.dashboard.controller; + +import io.javalin.http.Context; +import org.apache.amoro.Action; +import org.apache.amoro.Constants; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.ServerTableIdentifier; +import org.apache.amoro.TableFormat; +import org.apache.amoro.api.OptimizingService; +import org.apache.amoro.client.OptimizingClientPools; +import org.apache.amoro.config.Configurations; +import org.apache.amoro.process.LocalExecutionEngine; +import org.apache.amoro.process.LocalProcess; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.process.TableProcess; +import org.apache.amoro.server.DefaultOptimizingService; +import org.apache.amoro.server.catalog.CatalogManager; +import org.apache.amoro.server.persistence.PersistentBase; +import org.apache.amoro.server.persistence.TableRuntimeMeta; +import org.apache.amoro.server.persistence.mapper.TableMetaMapper; +import org.apache.amoro.server.persistence.mapper.TableProcessMapper; +import org.apache.amoro.server.process.ProcessService; +import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.DefaultTableRuntimeStore; +import org.apache.amoro.server.table.DerbyPersistence; +import org.apache.amoro.server.table.TableManager; +import org.apache.amoro.server.table.TableService; +import org.apache.amoro.shade.thrift.org.apache.thrift.TMultiplexedProcessor; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.amoro.shade.thrift.org.apache.thrift.protocol.TMultiplexedProtocol; +import org.apache.amoro.shade.thrift.org.apache.thrift.server.TThreadPoolServer; +import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TServerSocket; +import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TSocket; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; +import java.util.stream.Stream; + +/** + * Exercise the dashboard controller through real Thrift and the production cancellation service. + */ +class TestProcessCancellation extends PersistentBase { + private final Persistence persistence = new Persistence(); + private final AwaitableEngine engine = new AwaitableEngine(); + private final TableService tables = Mockito.mock(TableService.class); + private final TableManager tableManager = Mockito.mock(TableManager.class); + private final Context ctx = Mockito.mock(Context.class); + private final CountDownLatch entered = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private final CountDownLatch finished = new CountDownLatch(1); + private final ExecutorService workers = Executors.newFixedThreadPool(2); + private ServerTableIdentifier identifier; + private ProcessService processes; + private DefaultOptimizingService endpoint; + private TableController controller; + private TThreadPoolServer server; + private Thread serverThread; + private TSocket connection; + private OptimizingService.Client client; + private MockedStatic clientLookup; + private long processId; + private boolean closed; + private final CountDownLatch completionEntered = new CountDownLatch(1); + private final CountDownLatch releaseCompletion = new CountDownLatch(1); + private volatile boolean pauseBeforePersistence; + private volatile boolean pauseCallback; + + @BeforeEach + void setup() throws Exception { + engine.open(Collections.emptyMap()); + ProcessService.ExecuteEngineManager engines = + Mockito.mock(ProcessService.ExecuteEngineManager.class); + Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine)); + processes = + new ProcessService(tables, Collections.emptyList(), engines) { + @Override + public TableProcess untrackTableProcessInstance(ServerTableIdentifier table, long id) { + TableProcess process = super.untrackTableProcessInstance(table, id); + if (pauseCallback && process != null) { + awaitCompletionRelease(); + } + return process; + } + }; + processes.getTableHandlerChain().initialize(Collections.emptyList()); + endpoint = + new DefaultOptimizingService( + new Configurations(), null, null, tables, processes, null, null); + TMultiplexedProcessor processor = new TMultiplexedProcessor(); + processor.registerProcessor( + Constants.THRIFT_OPTIMIZING_SERVICE_NAME, new OptimizingService.Processor<>(endpoint)); + ServerSocket socket = new ServerSocket(0); + server = + new TThreadPoolServer( + new TThreadPoolServer.Args(new TServerSocket(socket)) + .processor(processor) + .executorService(workers)); + serverThread = new Thread(server::serve, "cancel-regression-thrift"); + serverThread.start(); + await(server::isServing); + connection = new TSocket("localhost", socket.getLocalPort(), 5000); + connection.open(); + client = + new OptimizingService.Client( + new TMultiplexedProtocol( + new TBinaryProtocol(connection), Constants.THRIFT_OPTIMIZING_SERVICE_NAME)); + // Replace only client discovery; the controller uses a real transport and service + // implementation. + clientLookup = Mockito.mockStatic(OptimizingClientPools.class); + clientLookup + .when(() -> OptimizingClientPools.getClient(Mockito.anyString())) + .thenReturn(client); + CatalogManager catalogs = Mockito.mock(CatalogManager.class); + Mockito.when(catalogs.catalogExist("catalog")).thenReturn(true); + Mockito.when(tableManager.getServerTableIdentifier(Mockito.any())) + .thenAnswer(call -> identifier); + Mockito.when(tableManager.getTableProcessMeta(Mockito.anyLong())) + .thenAnswer( + call -> + getAs( + TableProcessMapper.class, + mapper -> mapper.getProcessMeta(call.getArgument(0)))); + controller = new TableController(catalogs, tableManager, null, new Configurations()); + Mockito.when(ctx.pathParam("catalog")).thenReturn("catalog"); + Mockito.when(ctx.pathParam("db")).thenReturn("db"); + Mockito.when(ctx.pathParam("table")).thenReturn("tbl"); + } + + @AfterEach + void close() throws Exception { + close(10, TimeUnit.SECONDS); + } + + private void close(long timeout, TimeUnit unit) throws Exception { + if (closed) { + return; + } + release.countDown(); + if (clientLookup != null) { + clientLookup.close(); + clientLookup = null; + } + if (connection != null) { + connection.close(); + } + if (server != null) { + server.stop(); + } + workers.shutdownNow(); + if (serverThread != null) { + serverThread.join(5000); + Assertions.assertFalse(serverThread.isAlive(), "Thrift server did not terminate"); + } + Assertions.assertTrue( + workers.awaitTermination(10, TimeUnit.SECONDS), "Thrift workers did not terminate"); + if (endpoint != null) { + endpoint.dispose(); + } + if (processes != null) { + processes.dispose(); + } + engine.close(); + engine.awaitWorkers(); + if (processes != null) { + Assertions.assertTrue( + processes.awaitTermination(timeout, unit), + "Process callbacks did not terminate; Derby must remain available"); + } + // A timed-out or interrupted drain leaves persistence intact so teardown can be retried. + persistence.clear(); + closed = true; + } + + @ParameterizedTest + @MethodSource("formats") + void dashboardCancelsCleanupAndRejectsRepeatedCancellation(TableFormat format) throws Exception { + startCleanup(format); + controller.cancelOptimizingProcess(ctx); + Assertions.assertEquals(ProcessStatus.CANCELED, status()); + Assertions.assertTrue(processes.getTableProcessInstances(identifier).isEmpty()); + Assertions.assertThrows( + IllegalStateException.class, () -> controller.cancelOptimizingProcess(ctx)); + Mockito.verify(ctx, Mockito.times(1)).json(Mockito.any()); + Assertions.assertFalse(client.cancelProcess(Long.MAX_VALUE)); + } + + static Stream formats() { + return Stream.of(TableFormat.ICEBERG, TableFormat.PAIMON); + } + + @Test + void dashboardRejectsCancellationForAnotherTable() throws Exception { + startCleanup(TableFormat.ICEBERG); + Mockito.when(tableManager.getServerTableIdentifier(Mockito.any())) + .thenReturn( + ServerTableIdentifier.of( + identifier.getId() + 1, "catalog", "db", "other", TableFormat.ICEBERG)); + Assertions.assertThrows( + IllegalStateException.class, () -> controller.cancelOptimizingProcess(ctx)); + Assertions.assertEquals(ProcessStatus.RUNNING, status()); + Mockito.verify(ctx, Mockito.never()).json(Mockito.any()); + Assertions.assertTrue(client.cancelProcess(processId)); + } + + @Test + void completedCleanupIsNotReportedAsCanceled() throws Exception { + startCleanup(TableFormat.ICEBERG); + release.countDown(); + await(() -> status() == ProcessStatus.SUCCESS); + Assertions.assertThrows( + IllegalStateException.class, () -> controller.cancelOptimizingProcess(ctx)); + Assertions.assertEquals(ProcessStatus.SUCCESS, status()); + Mockito.verify(ctx, Mockito.never()).json(Mockito.any()); + } + + @ParameterizedTest + @MethodSource("cleanupPhases") + void cleanupTimeoutPreservesPersistenceUntilCompletionExits( + TableFormat format, boolean beforePersistence) throws Exception { + startCleanup(format); + pauseBeforePersistence = beforePersistence; + pauseCallback = !beforePersistence; + try { + // Model a failure before the test reaches its explicit cancellation. Hold the real tracer + // either before its terminal write or in its completion callback, after it was untracked. + release.countDown(); + Assertions.assertTrue(completionEntered.await(10, TimeUnit.SECONDS)); + AssertionError timeout = + Assertions.assertThrows(AssertionError.class, () -> close(0, TimeUnit.NANOSECONDS)); + Assertions.assertTrue(timeout.getMessage().contains("Process callbacks did not terminate")); + Assertions.assertFalse(processes.awaitTermination(0, TimeUnit.NANOSECONDS)); + Assertions.assertEquals( + beforePersistence ? ProcessStatus.RUNNING : ProcessStatus.SUCCESS, status()); + } finally { + releaseCompletion.countDown(); + // Retry the same teardown only after the held worker can finish; @AfterEach is idempotent. + close(); + } + Assertions.assertTrue(processes.awaitTermination(0, TimeUnit.NANOSECONDS)); + Assertions.assertNull( + getAs(TableProcessMapper.class, mapper -> mapper.getProcessMeta(processId))); + } + + static Stream cleanupPhases() { + return formats() + .flatMap(format -> Stream.of(Arguments.of(format, true), Arguments.of(format, false))); + } + + private void awaitCompletionRelease() { + completionEntered.countDown(); + try { + Assertions.assertTrue( + releaseCompletion.await(40, TimeUnit.SECONDS), "Completion gate timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + private void startCleanup(TableFormat format) throws Exception { + identifier = ServerTableIdentifier.of(1L, "catalog", "db", "tbl", format); + doAs(TableMetaMapper.class, mapper -> mapper.insertTable(identifier)); + TableRuntimeMeta meta = new TableRuntimeMeta(); + meta.setTableId(identifier.getId()); + meta.setGroupName("default"); + meta.setStatusCode(0); + DefaultTableRuntimeStore store = + new DefaultTableRuntimeStore( + identifier, meta, DefaultTableRuntime.REQUIRED_STATES, Collections.emptyList()); + DefaultTableRuntime runtime = new DefaultTableRuntime(store, () -> null); + store.setTableRuntime(runtime); + Mockito.when(tables.getRuntime(identifier.getId())).thenReturn(runtime); + processes.register(runtime, new BlockingCleanup(runtime)); + Assertions.assertTrue(entered.await(10, TimeUnit.SECONDS)); + processId = processes.getTableProcessInstances(identifier).keySet().iterator().next(); + await(() -> status() == ProcessStatus.RUNNING); + Mockito.when(ctx.pathParam("processId")).thenReturn(String.valueOf(processId)); + } + + private ProcessStatus status() { + return getAs(TableProcessMapper.class, mapper -> mapper.getProcessMeta(processId)).getStatus(); + } + + private static void await(BooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + while (!condition.getAsBoolean() && System.nanoTime() < deadline) { + Thread.sleep(20); + } + Assertions.assertTrue(condition.getAsBoolean(), "Timed out waiting for cancellation lifecycle"); + } + + private class BlockingCleanup extends TableProcess implements LocalProcess { + BlockingCleanup(DefaultTableRuntime runtime) { + super(runtime, engine); + } + + @Override + public Action getAction() { + return PaimonActions.EXPIRE_SNAPSHOTS; + } + + @Override + public Map getProcessParameters() { + return Collections.emptyMap(); + } + + @Override + public Map getSummary() { + if (pauseBeforePersistence && finished.getCount() == 0) { + awaitCompletionRelease(); + } + return Collections.emptyMap(); + } + + @Override + public void run() { + entered.countDown(); + try { + if (!release.await(20, TimeUnit.SECONDS)) { + throw new IllegalStateException("Test release timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } finally { + finished.countDown(); + } + } + } + + private static class AwaitableEngine extends LocalExecutionEngine { + private final List pools = new ArrayList<>(); + + @Override + protected ThreadPoolExecutor newFixedPool(String tag, int size) { + ThreadPoolExecutor pool = super.newFixedPool(tag, size); + pools.add(pool); + return pool; + } + + void awaitWorkers() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + for (ThreadPoolExecutor pool : pools) { + Assertions.assertTrue( + pool.awaitTermination(Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS), + "Local workers did not terminate; Derby must remain available"); + } + } + } + + private static class Persistence extends DerbyPersistence { + void clear() { + super.after(); + } + } +} diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java b/amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java new file mode 100644 index 0000000000..1f4b33e4d0 --- /dev/null +++ b/amoro-ams/src/test/java/org/apache/amoro/server/process/TestProcessCancellationConcurrency.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.process; + +import org.apache.amoro.Action; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.ServerTableIdentifier; +import org.apache.amoro.TableFormat; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.config.Configurations; +import org.apache.amoro.process.ActionCoordinator; +import org.apache.amoro.process.LocalExecutionEngine; +import org.apache.amoro.process.LocalProcess; +import org.apache.amoro.process.ProcessEvent; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.process.TableProcess; +import org.apache.amoro.server.DefaultOptimizingService; +import org.apache.amoro.server.persistence.PersistentBase; +import org.apache.amoro.server.persistence.TableRuntimeMeta; +import org.apache.amoro.server.persistence.mapper.TableMetaMapper; +import org.apache.amoro.server.persistence.mapper.TableProcessMapper; +import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.DefaultTableRuntimeStore; +import org.apache.amoro.server.table.DerbyPersistence; +import org.apache.amoro.server.table.TableService; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +/** Deterministic cancellation races using the real service, store and local execution engine. */ +class TestProcessCancellationConcurrency { + static Stream formats() { + return Stream.of(TableFormat.ICEBERG, TableFormat.PAIMON); + } + + @ParameterizedTest + @MethodSource("formats") + void cancellationBeforeIdentifierPublicationCancelsQueuedWork(TableFormat format) + throws Exception { + try (Fixture f = new Fixture(format)) { + Work blocker = f.work(() -> await(f.releasePool, 30)); + f.engine.submitDirect(blocker); + await(blocker.started); + Work target = f.work(() -> {}); + f.service.register(f.runtime, target); + await(f.engine.submitted); + Assertions.assertEquals(0, target.runs.get()); + + Assertions.assertTrue(f.endpoint.cancelProcess(f.service.store.getProcessId())); + Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus()); + f.engine.releaseSubmit.countDown(); + await(f.engine.canceled); + Assertions.assertEquals(ProcessStatus.CANCELED, f.engine.getStatus(f.engine.id)); + + // A marker behind the canceled task proves the queue drained without invoking its body. + Work marker = f.work(() -> {}); + f.engine.submitDirect(marker); + f.releasePool.countDown(); + await(marker.finished); + Assertions.assertEquals(0, target.runs.get()); + Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty()); + } + } + + @ParameterizedTest + @MethodSource("formats") + void cancellationBetweenFailureAndRetryBecomesCanceled(TableFormat format) throws Exception { + try (Fixture f = new Fixture(format)) { + f.service.pause = Pause.AFTER_FAILURE; + f.engine.releaseSubmit.countDown(); + Work target = + f.work( + () -> { + throw new IllegalStateException("controlled failure"); + }); + f.service.register(f.runtime, target); + await(f.service.transitionReached); + Assertions.assertEquals(ProcessStatus.FAILED, f.service.store.getStatus()); + + Assertions.assertTrue(f.endpoint.cancelProcess(f.service.store.getProcessId())); + Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus()); + f.service.releaseTransition.countDown(); + await(f.service.callbackFinished); + Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus()); + Assertions.assertEquals(0, f.service.store.getRetryNumber()); + Assertions.assertEquals(1, f.engine.submissions.get()); + Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty()); + } + } + + @ParameterizedTest + @MethodSource("formats") + void rejectedCancellationDoesNotPreventRetry(TableFormat format) throws Exception { + try (Fixture f = new Fixture(format)) { + f.service.pause = Pause.AFTER_FAILURE; + f.engine.releaseSubmit.countDown(); + AtomicInteger attempts = new AtomicInteger(); + Work target = + f.work( + () -> { + if (attempts.getAndIncrement() == 0) { + throw new IllegalStateException("first attempt fails"); + } + }); + f.service.register(f.runtime, target); + await(f.service.transitionReached); + Assertions.assertFalse( + f.service.store.tryTransitState( + ProcessStatus.RUNNING, + ProcessEvent.CANCEL_REQUESTED, + f.service.store.getExternalProcessIdentifier(), + "invalid target", + Collections.emptyMap(), + Collections.emptyMap())); + Assertions.assertEquals(ProcessStatus.FAILED, f.service.store.getStatus()); + Assertions.assertFalse(f.service.getTableProcessInstances(f.identifier).isEmpty()); + f.service.releaseTransition.countDown(); + await(f.service.successRecorded); + await(f.service.callbackFinished); + Assertions.assertEquals(ProcessStatus.SUCCESS, f.service.store.getStatus()); + Assertions.assertEquals(1, f.service.store.getRetryNumber()); + Assertions.assertEquals(2, f.engine.submissions.get()); + } + } + + @ParameterizedTest + @MethodSource("formats") + void tableRemovalBetweenFailureAndRetryCancels(TableFormat format) throws Exception { + try (Fixture f = new Fixture(format)) { + f.service.pause = Pause.AFTER_FAILURE; + f.engine.releaseSubmit.countDown(); + Work target = + f.work( + () -> { + throw new IllegalStateException("controlled failure"); + }); + f.service.register(f.runtime, target); + await(f.service.transitionReached); + f.service.getTableHandlerChain().fireTableRemoved(f.runtime); + Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus()); + f.service.releaseTransition.countDown(); + await(f.service.callbackFinished); + Assertions.assertEquals(0, f.service.store.getRetryNumber()); + Assertions.assertEquals(1, f.engine.submissions.get()); + Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty()); + } + } + + @ParameterizedTest + @MethodSource("formats") + void completionWinsCancellation(TableFormat format) throws Exception { + try (Fixture f = new Fixture(format)) { + f.service.pause = Pause.AFTER_SUCCESS; + f.engine.releaseSubmit.countDown(); + f.service.register(f.runtime, f.work(() -> {})); + await(f.service.transitionReached); + Assertions.assertFalse(f.endpoint.cancelProcess(f.service.store.getProcessId())); + Assertions.assertEquals(ProcessStatus.SUCCESS, f.service.store.getStatus()); + Assertions.assertFalse(f.service.getTableProcessInstances(f.identifier).isEmpty()); + f.service.releaseTransition.countDown(); + await(f.service.callbackFinished); + Assertions.assertEquals(ProcessStatus.SUCCESS, f.service.store.getStatus()); + Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty()); + } + } + + @ParameterizedTest + @MethodSource("formats") + void cancellationWinsCompletion(TableFormat format) throws Exception { + try (Fixture f = new Fixture(format)) { + f.service.pause = Pause.BEFORE_SUCCESS; + f.engine.releaseSubmit.countDown(); + f.service.register(f.runtime, f.work(() -> {})); + await(f.service.transitionReached); + Assertions.assertTrue(f.endpoint.cancelProcess(f.service.store.getProcessId())); + f.service.releaseTransition.countDown(); + await(f.service.callbackFinished); + Assertions.assertEquals(ProcessStatus.CANCELED, f.service.store.getStatus()); + Assertions.assertEquals(0, f.service.store.getRetryNumber()); + Assertions.assertEquals(1, f.engine.submissions.get()); + Assertions.assertTrue(f.service.getTableProcessInstances(f.identifier).isEmpty()); + } + } + + private enum Pause { + NONE, + AFTER_FAILURE, + BEFORE_SUCCESS, + AFTER_SUCCESS + } + + private static void await(CountDownLatch latch) { + await(latch, 10); + } + + private static void await(CountDownLatch latch, int seconds) { + try { + Assertions.assertTrue( + latch.await(seconds, TimeUnit.SECONDS), "Timed out waiting for test gate"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + private static class Fixture extends PersistentBase implements AutoCloseable { + final Persistence persistence = new Persistence(); + final CountDownLatch releasePool = new CountDownLatch(1); + final GatedEngine engine = new GatedEngine(); + final ServerTableIdentifier identifier; + final DefaultTableRuntime runtime; + final ControlledService service; + final DefaultOptimizingService endpoint; + + Fixture(TableFormat format) throws Exception { + identifier = ServerTableIdentifier.of(1L, "cancel-races", "db", "tbl", format); + doAs(TableMetaMapper.class, mapper -> mapper.insertTable(identifier)); + TableRuntimeMeta meta = new TableRuntimeMeta(); + meta.setTableId(identifier.getId()); + meta.setGroupName("default"); + meta.setStatusCode(0); + DefaultTableRuntimeStore state = + new DefaultTableRuntimeStore( + identifier, meta, DefaultTableRuntime.REQUIRED_STATES, Collections.emptyList()); + runtime = new DefaultTableRuntime(state, () -> null); + state.setTableRuntime(runtime); + TableService tables = Mockito.mock(TableService.class); + Mockito.when(tables.getRuntime(identifier.getId())).thenReturn(runtime); + engine.open(Map.of("pool.default.thread-count", "1")); + ProcessService.ExecuteEngineManager engines = + Mockito.mock(ProcessService.ExecuteEngineManager.class); + Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine)); + ActionCoordinator coordinator = + new MockActionCoordinator(engine) { + @Override + public Action action() { + return PaimonActions.EXPIRE_SNAPSHOTS; + } + + @Override + public int parallelism() { + return 1; + } + }; + service = new ControlledService(tables, List.of(coordinator), engines); + service.getTableHandlerChain().initialize(Collections.emptyList()); + endpoint = + new DefaultOptimizingService( + new Configurations(), null, null, tables, service, null, null); + } + + Work work(Runnable action) { + return new Work(runtime, engine, action); + } + + @Override + public void close() { + engine.releaseSubmit.countDown(); + service.releaseTransition.countDown(); + if (service.store != null) { + endpoint.cancelProcess(service.store.getProcessId()); + } + releasePool.countDown(); + Work marker = work(() -> {}); + engine.submitDirect(marker); + await(marker.finished); + service + .getActionCoordinators() + .values() + .forEach(schedulers -> schedulers.forEach(ActionCoordinatorScheduler::dispose)); + endpoint.dispose(); + service.dispose(); + engine.close(); + persistence.clear(); + } + } + + private static class ControlledService extends ProcessService { + volatile DefaultTableProcessStore store; + volatile Pause pause = Pause.NONE; + final AtomicBoolean paused = new AtomicBoolean(); + final CountDownLatch transitionReached = new CountDownLatch(1); + final CountDownLatch releaseTransition = new CountDownLatch(1); + final CountDownLatch completionObserved = new CountDownLatch(1); + final CountDownLatch callbackFinished = new CountDownLatch(1); + final CountDownLatch successRecorded = new CountDownLatch(1); + + ControlledService( + TableService tables, List coordinators, ExecuteEngineManager engines) { + super(tables, coordinators, engines); + } + + @Override + public TableProcess untrackTableProcessInstance(ServerTableIdentifier table, long processId) { + TableProcess process = super.untrackTableProcessInstance(table, processId); + if (completionObserved.getCount() == 0) { + callbackFinished.countDown(); + } + return process; + } + + void pauseTransition() { + if (paused.compareAndSet(false, true)) { + transitionReached.countDown(); + await(releaseTransition); + } + } + + @Override + protected DefaultTableProcessStore persistTableProcess(TableProcess process) { + DefaultTableProcessStore initial = super.persistTableProcess(process); + TableProcessMeta meta = + getAs(TableProcessMapper.class, mapper -> mapper.getProcessMeta(initial.getProcessId())); + store = + new DefaultTableProcessStore( + initial.getProcessId(), + process.getTableRuntime(), + meta, + process.getAction(), + ActionCoordinatorScheduler.PROCESS_MAX_RETRY_NUMBER) { + @Override + public boolean tryTransitState( + ProcessStatus status, + ProcessEvent event, + String external, + String reason, + Map parameters, + Map summary) { + if (pause == Pause.BEFORE_SUCCESS && status == ProcessStatus.SUCCESS) { + pauseTransition(); + } + boolean accepted = + super.tryTransitState(status, event, external, reason, parameters, summary); + if (accepted + && ((pause == Pause.AFTER_FAILURE && status == ProcessStatus.FAILED) + || (pause == Pause.AFTER_SUCCESS && status == ProcessStatus.SUCCESS))) { + pauseTransition(); + } + if (event == ProcessEvent.COMPLETE_SUCCESS || event == ProcessEvent.COMPLETE_FAILED) { + completionObserved.countDown(); + } + if (accepted && status == ProcessStatus.SUCCESS) { + successRecorded.countDown(); + } + return accepted; + } + }; + return store; + } + } + + private static class GatedEngine extends LocalExecutionEngine { + final CountDownLatch submitted = new CountDownLatch(1); + final CountDownLatch releaseSubmit = new CountDownLatch(1); + final CountDownLatch canceled = new CountDownLatch(1); + volatile String id; + final AtomicInteger submissions = new AtomicInteger(); + + String submitDirect(TableProcess process) { + return super.submitTableProcess(process); + } + + @Override + public String submitTableProcess(TableProcess process) { + submissions.incrementAndGet(); + id = super.submitTableProcess(process); + submitted.countDown(); + await(releaseSubmit); + return id; + } + + @Override + public ProcessStatus tryCancelTableProcess(TableProcess process, String identifier) { + ProcessStatus status = super.tryCancelTableProcess(process, identifier); + if (status == ProcessStatus.CANCELED) { + canceled.countDown(); + } + return status; + } + } + + private static class Work extends TableProcess implements LocalProcess { + final AtomicInteger runs = new AtomicInteger(); + final CountDownLatch started = new CountDownLatch(1); + final CountDownLatch finished = new CountDownLatch(1); + final Runnable action; + + Work(TableRuntime runtime, LocalExecutionEngine engine, Runnable action) { + super(runtime, engine); + this.action = action; + } + + @Override + public Action getAction() { + return PaimonActions.EXPIRE_SNAPSHOTS; + } + + @Override + public Map getProcessParameters() { + return Collections.emptyMap(); + } + + @Override + public Map getSummary() { + return Collections.emptyMap(); + } + + @Override + public void run() { + runs.incrementAndGet(); + started.countDown(); + try { + action.run(); + } finally { + finished.countDown(); + } + } + } + + private static class Persistence extends DerbyPersistence { + void clear() { + super.after(); + } + } +} diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java new file mode 100644 index 0000000000..da71e8490d --- /dev/null +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableExpirationConfiguration.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.table; + +import org.apache.amoro.IcebergActions; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.process.LocalExecutionEngine; +import org.apache.amoro.server.process.iceberg.IcebergProcessFactory; +import org.apache.amoro.server.table.cleanup.TableRuntimeCleanupState; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class TestTableExpirationConfiguration { + @ParameterizedTest + @CsvSource({ + "omitted,omitted,true", + "true,omitted,true", + "false,omitted,false", + "omitted,false,false", + "true,false,false", + "false,true,false" + }) + void icebergExpirationKeepsExistingTableSwitchAndGcGate( + String tableExpire, String gc, boolean expected) { + Map properties = new HashMap<>(); + if (!"omitted".equals(tableExpire)) { + properties.put("table-expire.enabled", tableExpire); + } + if (!"omitted".equals(gc)) { + properties.put("gc.enabled", gc); + } + TableRuntime runtime = Mockito.mock(TableRuntime.class); + Mockito.when(runtime.getTableConfiguration()) + .thenReturn(TableConfigurations.parseTableConfig(properties)); + Mockito.when(runtime.getState(DefaultTableRuntime.CLEANUP_STATE_KEY)) + .thenReturn(new TableRuntimeCleanupState()); + IcebergProcessFactory factory = new IcebergProcessFactory(); + factory.open(Map.of("expire-snapshots.interval", "1h")); + factory.availableExecuteEngines(List.of(Mockito.mock(LocalExecutionEngine.class))); + Assertions.assertEquals( + expected, factory.trigger(runtime, IcebergActions.EXPIRE_SNAPSHOTS).isPresent()); + } +} diff --git a/amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java new file mode 100644 index 0000000000..23aba1cd30 --- /dev/null +++ b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/PaimonWarehouseLifecycle.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.process; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; + +/** Explicitly launched failure fixture; its name excludes it from Surefire test discovery. */ +class PaimonWarehouseLifecycle extends TestPaimonProcessIntegration { + static PaimonWarehouseLifecycle instance; + final CountDownLatch release = new CountDownLatch(1); + final AtomicReference worker = new AtomicReference<>(); + + PaimonWarehouseLifecycle() { + instance = this; + } + + @Test + void schedulerTimeout() throws Exception { + startSchedulerWithBlockedRead(release, worker); + // Return normally: failure must come from the inherited, real @AfterEach cleanup. + } + + @Test + void bodyFailure() { + Assertions.fail("simulated test failure"); + } +} diff --git a/amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java new file mode 100644 index 0000000000..991fa16b45 --- /dev/null +++ b/amoro-ams/src/test/paimon/org/apache/amoro/server/process/TestPaimonProcessIntegration.java @@ -0,0 +1,1036 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.process; + +import org.apache.amoro.Action; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.ServerTableIdentifier; +import org.apache.amoro.TableFormat; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.config.Configurations; +import org.apache.amoro.formats.paimon.PaimonCatalog; +import org.apache.amoro.formats.paimon.PaimonCatalogFactory; +import org.apache.amoro.formats.paimon.process.PaimonProcessFactory; +import org.apache.amoro.formats.paimon.process.PaimonSnapshotsExpiringProcess; +import org.apache.amoro.process.ActionCoordinator; +import org.apache.amoro.process.LocalExecutionEngine; +import org.apache.amoro.process.ProcessFactory; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.process.TableProcess; +import org.apache.amoro.server.DefaultOptimizingService; +import org.apache.amoro.server.catalog.CatalogManager; +import org.apache.amoro.server.catalog.ServerCatalog; +import org.apache.amoro.server.dashboard.ServerTableDescriptor; +import org.apache.amoro.server.manager.AbstractPluginManager; +import org.apache.amoro.server.manager.PluginConfiguration; +import org.apache.amoro.server.persistence.PersistentBase; +import org.apache.amoro.server.persistence.TableRuntimeMeta; +import org.apache.amoro.server.persistence.mapper.TableMetaMapper; +import org.apache.amoro.server.persistence.mapper.TableProcessMapper; +import org.apache.amoro.server.table.DefaultActionCoordinator; +import org.apache.amoro.server.table.DefaultTableRuntimeFactory; +import org.apache.amoro.server.table.DefaultTableRuntimeStore; +import org.apache.amoro.server.table.DerbyPersistence; +import org.apache.amoro.server.table.TableManager; +import org.apache.amoro.server.table.TableService; +import org.apache.amoro.shade.guava32.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.amoro.table.TableRuntimeFactory; +import org.apache.amoro.table.descriptor.OptimizingProcessInfo; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.CloseableIterator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.mockito.Mockito; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +class TestPaimonProcessIntegration extends PersistentBase { + private Path warehouse; + private final Persistence persistence = new Persistence(); + private final ServerTableIdentifier identifier = + ServerTableIdentifier.of(1L, "catalog", "db", "tbl", TableFormat.PAIMON); + private final Clock clock = Mockito.mock(Clock.class); + private final ObservedFactory factory = new ObservedFactory(clock); + private final AwaitableEngine engine = new AwaitableEngine(); + private Catalog nativeCatalog; + private PaimonCatalog catalog; + private TableRuntime runtime; + private AwaitableService service; + private ServerTableDescriptor descriptor; + private CountDownLatch loadEntered; + private CountDownLatch releaseLoad; + private boolean gateLocalLoad; + private TableService scheduledTables; + private CountDownLatch nativeReadEntered; + private CountDownLatch releaseNativeRead; + private boolean closed; + private Duration schedulerWaitTimeout = Duration.ofSeconds(40); + private ScheduledExecutorService continuousWriter; + private ScheduledFuture continuousWrites; + private final CompletableFuture resourcesCleared = new CompletableFuture<>(); + + @BeforeEach + void setup() throws Exception { + // JUnit TempDir cleanup runs even after a failed teardown, while workers may still be alive. + warehouse = Files.createTempDirectory("amoro-paimon-"); + Mockito.when(clock.millis()).thenAnswer(call -> System.currentTimeMillis()); + nativeCatalog = + PaimonCatalogFactory.paimonCatalog( + Map.of("warehouse", warehouse.toString()), new Configuration()); + nativeCatalog.createDatabase("db", false); + nativeCatalog.createTable( + Identifier.create("db", "tbl"), + Schema.newBuilder().column("id", DataTypes.INT()).option("write-only", "true").build(), + false); + catalog = new PaimonCatalog(nativeCatalog, "catalog"); + doAs(TableMetaMapper.class, mapper -> mapper.insertTable(identifier)); + engine.open(Map.of("pool.paimon-expire-snapshots.thread-count", "1")); + factory.open(Map.of("expire-snapshots.interval", "1h")); + factory.availableExecuteEngines(List.of(engine)); + DefaultTableRuntimeFactory runtimeFactory = new DefaultTableRuntimeFactory(); + runtimeFactory.initialize(List.of(factory)); + runtimeFactory.withTableLoader( + id -> { + if (releaseLoad != null + && (!gateLocalLoad || Thread.currentThread().getName().startsWith("local-"))) { + loadEntered.countDown(); + try { + if (!releaseLoad.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting for test load gate"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + // The regression owns this gate and releases it in finally; it models slow native IO. + if (nativeReadEntered != null && Thread.currentThread().getName().startsWith("local-")) { + nativeReadEntered.countDown(); + awaitGate(releaseNativeRead); + } + return catalog.loadTable(id.getDatabase(), id.getTableName()); + }); + TableRuntimeFactory.TableRuntimeCreator creator = + runtimeFactory.accept(identifier, Map.of()).orElseThrow(); + TableRuntimeMeta meta = new TableRuntimeMeta(); + meta.setTableId(identifier.getId()); + meta.setGroupName("default"); + meta.setStatusCode(0); + DefaultTableRuntimeStore store = + new DefaultTableRuntimeStore( + identifier, meta, creator.requiredStateKeys(), Collections.emptyList()); + runtime = creator.create(store); + store.setTableRuntime(runtime); + ProcessService.ExecuteEngineManager engines = + Mockito.mock(ProcessService.ExecuteEngineManager.class); + Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine)); + service = + new AwaitableService( + Mockito.mock(TableService.class), runtimeFactory.supportedCoordinators(), engines); + // Initialize lifecycle without scheduling automatic checks over test data. + service.getTableHandlerChain().initialize(Collections.emptyList()); + CatalogManager manager = Mockito.mock(CatalogManager.class); + ServerCatalog serverCatalog = Mockito.mock(ServerCatalog.class); + Mockito.when(manager.getServerCatalog("catalog")).thenReturn(serverCatalog); + Mockito.when(serverCatalog.loadTable("db", "tbl")) + .thenAnswer(call -> catalog.loadTable("db", "tbl")); + descriptor = + new ServerTableDescriptor(manager, Mockito.mock(TableManager.class), new Configurations()); + } + + @AfterEach + synchronized void close() throws Exception { + if (closed) { + return; + } + if (scheduledTables != null) { + Mockito.when(scheduledTables.contains(identifier.getId())).thenReturn(false); + } + factory.stopObserving(); + if (service != null) { + service + .getActionCoordinators() + .values() + .forEach(schedulers -> schedulers.forEach(ActionCoordinatorScheduler::dispose)); + } + stopContinuousWriter(); + if (releaseLoad != null) { + releaseLoad.countDown(); + } + factory.awaitSchedulerWorkers(schedulerWaitTimeout); + if (service != null) { + // Keep the executor accepting bounded failure retries until its completion callback untracks. + await(() -> service.getTableProcessInstances(identifier).isEmpty()); + service.dispose(); + Assertions.assertTrue( + service.awaitTermination(40, TimeUnit.SECONDS), + "Process callbacks did not terminate; catalog and Derby must remain available"); + } + engine.close(); + // Cancellation completes the engine Future before an already-running body necessarily exits. + engine.awaitWorkers(); + if (nativeCatalog != null) { + nativeCatalog.close(); + } + persistence.clear(); + if (warehouse != null) { + FileUtils.deleteDirectory(warehouse.toFile()); + } + closed = true; + resourcesCleared.complete(null); + } + + @Test + void realFrameworkPersistsPaimonResultsAndServesCleanupQueries() throws Exception { + service.register( + runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow()); + await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.SUCCESS); + OptimizingProcessInfo info = rows().get(0); + Assertions.assertEquals("noop", info.getSummary().get("outcome")); + Assertions.assertEquals("0", info.getSummary().get("reported-expired-snapshots")); + Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0); + Assertions.assertEquals( + Map.of(PaimonActions.EXPIRE_SNAPSHOTS.getName(), "Expire Snapshots"), + descriptor.getTableProcessTypes( + identifier.getIdentifier().buildTableIdentifier(), "CLEANUP")); + Assertions.assertEquals(0, info.getTotalTasks()); + Assertions.assertNotNull(info.getInputFiles()); + await(() -> service.getTableProcessInstances(identifier).isEmpty()); + } + + @Test + void independentChangelogTableExpiresThroughProcessService() throws Exception { + nativeCatalog.alterTable( + Identifier.create("db", "tbl"), + List.of( + SchemaChange.setOption("snapshot.num-retained.min", "1"), + SchemaChange.setOption("snapshot.num-retained.max", "2"), + SchemaChange.setOption("snapshot.time-retained", "100 d"), + SchemaChange.setOption("changelog.num-retained.max", "5")), + false); + FileStoreTable table = (FileStoreTable) nativeCatalog.getTable(Identifier.create("db", "tbl")); + for (int id = 1; id <= 8; id++) { + append(table, id); + } + service.register( + runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow()); + await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.SUCCESS); + OptimizingProcessInfo info = rows().get(0); + Assertions.assertEquals("executed", info.getSummary().get("outcome")); + Assertions.assertEquals("6", info.getSummary().get("reported-expired-snapshots")); + Assertions.assertEquals("5", info.getSummary().get("changelog.num-retained.max")); + Assertions.assertEquals(7L, table.snapshotManager().earliestSnapshotId()); + Assertions.assertEquals(6, table.changelogManager().safelyGetAllChangelogs().size()); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6, 7, 8), readIds(table)); + Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0); + await(() -> service.getTableProcessInstances(identifier).isEmpty()); + } + + @Test + void failureSummaryIsPersistedAndRetriesAreBounded() throws Exception { + nativeCatalog.alterTable( + Identifier.create("db", "tbl"), + SchemaChange.setOption("snapshot.expire.limit", "0"), + false); + service.register( + runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow()); + await(() -> !rows().isEmpty() && service.getTableProcessInstances(identifier).isEmpty()); + Assertions.assertEquals(1, rows().size()); + OptimizingProcessInfo info = rows().get(0); + Assertions.assertEquals(ProcessStatus.FAILED, info.getStatus()); + Assertions.assertEquals("config", info.getSummary().get("error-stage")); + Assertions.assertNotNull(info.getSummary().get("error-message")); + Assertions.assertFalse(info.getSummary().containsKey("outcome")); + Assertions.assertEquals( + 3, + getAs( + TableProcessMapper.class, + mapper -> mapper.getProcessMeta(Long.parseLong(info.getProcessId()))) + .getRetryNumber()); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + } + + @Test + void recoveredLostLocalTaskRunsWithCurrentSettings() throws Exception { + doAs( + TableProcessMapper.class, + mapper -> + mapper.insertProcess( + identifier.getId(), + 42L, + "lost-local-process", + ProcessStatus.RUNNING, + PaimonActions.EXPIRE_SNAPSHOTS.getName(), + "default", + "local", + 0, + System.currentTimeMillis(), + Map.of(), + Map.of())); + nativeCatalog.alterTable( + Identifier.create("db", "tbl"), + SchemaChange.setOption("amoro.expire-snapshots.enabled", "false"), + false); + service.recoverProcesses(List.of(runtime)); + await(() -> rows().get(0).getStatus() == ProcessStatus.SUCCESS); + Assertions.assertEquals("42", rows().get(0).getProcessId()); + Assertions.assertEquals("skipped", rows().get(0).getSummary().get("outcome")); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + await(() -> service.getTableProcessInstances(identifier).isEmpty()); + } + + @Test + void pluginDefaultsDoNotEnableMaintenanceAndExplicitConfigAdmitsRuntime() throws Exception { + for (List configs : + List.of( + Collections.emptyList(), + List.of( + new PluginConfiguration( + "paimon", false, 100, Map.of("expire-snapshots.interval", "1h"))), + List.of(PluginConfiguration.emptyConfig("paimon")), + List.of( + new PluginConfiguration( + "paimon", + true, + 100, + Map.of( + "expire-snapshots.enabled", + "false", + "expire-snapshots.interval", + "1h"))))) { + try (Plugins manager = new Plugins(configs)) { + manager.initialize(); + DefaultTableRuntimeFactory runtimes = new DefaultTableRuntimeFactory(); + runtimes.initialize(manager.installedPlugins()); + Assertions.assertTrue(runtimes.accept(identifier, Map.of()).isEmpty()); + } + } + PluginConfiguration enabled = + PluginConfiguration.fromJSONObject( + new ObjectMapper() + .readTree( + "{\"name\":\"paimon\",\"properties\":{\"expire-snapshots.interval\":\"1h\"}}")); + try (Plugins manager = new Plugins(List.of(enabled))) { + manager.initialize(); + DefaultTableRuntimeFactory runtimes = new DefaultTableRuntimeFactory(); + runtimes.initialize(manager.installedPlugins()); + Assertions.assertTrue(runtimes.accept(identifier, Map.of()).isPresent()); + Assertions.assertEquals(1, runtimes.supportedCoordinators().size()); + } + } + + @Test + void cancelServiceEntryCancelsPaimonCleanup() throws Exception { + TableProcess process = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow(); + loadEntered = new CountDownLatch(1); + releaseLoad = new CountDownLatch(1); + TableService tables = Mockito.mock(TableService.class); + Mockito.when(tables.getRuntime(identifier.getId())).thenReturn(runtime); + DefaultOptimizingService endpoint = + new DefaultOptimizingService(new Configurations(), null, null, tables, service, null, null); + try { + service.register(runtime, process); + Assertions.assertTrue(loadEntered.await(10, TimeUnit.SECONDS)); + await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.RUNNING); + long processId = Long.parseLong(rows().get(0).getProcessId()); + Assertions.assertTrue(endpoint.cancelProcess(processId)); + Assertions.assertEquals(ProcessStatus.CANCELED, rows().get(0).getStatus()); + Assertions.assertTrue(service.getTableProcessInstances(identifier).isEmpty()); + Assertions.assertFalse(endpoint.cancelProcess(processId)); + Assertions.assertFalse(endpoint.cancelProcess(Long.MAX_VALUE)); + } finally { + releaseLoad.countDown(); + await(() -> process.getSummary().containsKey("outcome")); + endpoint.dispose(); + } + } + + @Test + void sameActionIsNotSubmittedTwiceAndTableRemovalCancelsTrackedProcess() throws Exception { + TableProcess process = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow(); + loadEntered = new CountDownLatch(1); + releaseLoad = new CountDownLatch(1); + service.register(runtime, process); + Assertions.assertTrue(loadEntered.await(10, TimeUnit.SECONDS)); + service.register(runtime, new PaimonSnapshotsExpiringProcess(runtime, engine)); + await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.RUNNING); + service.getTableHandlerChain().fireTableRemoved(runtime); + Assertions.assertEquals(ProcessStatus.CANCELED, rows().get(0).getStatus()); + Assertions.assertTrue(service.getTableProcessInstances(identifier).isEmpty()); + releaseLoad.countDown(); + // Cancellation is not a rollback or proof that the underlying invocation stopped. + await(() -> process.getSummary().containsKey("outcome")); + Assertions.assertEquals(ProcessStatus.CANCELED, rows().get(0).getStatus()); + } + + @Test + void periodicSchedulerWaitsForStartupAndSuccessInterval() throws Exception { + loadEntered = new CountDownLatch(1); + releaseLoad = new CountDownLatch(1); + gateLocalLoad = true; + factory.controlChecks = true; + long started = startPeriodicScheduling(1_000); + + Check first = factory.nextCheck(); + Assertions.assertTrue(first.started - started >= Duration.ofSeconds(11).toNanos()); + Assertions.assertTrue(first.proceed().isPresent()); + Assertions.assertTrue(loadEntered.await(10, TimeUnit.SECONDS)); + + // The real scheduler continues checking while the first local invocation is blocked. + Check second = factory.nextCheck(); + Assertions.assertTrue(second.started - first.finished >= Duration.ofSeconds(1).toNanos()); + Assertions.assertTrue(second.proceed().isPresent()); + Check third = factory.nextCheck(); + Assertions.assertEquals(1, rows().size(), "An in-flight action must not be registered twice"); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + releaseLoad.countDown(); + await(() -> runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0); + long successTime = runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME); + Mockito.when(clock.millis()).thenReturn(successTime + 999); + Assertions.assertTrue(third.proceed().isEmpty(), "A check just after success must be skipped"); + + // Another real scheduler interval can pass while the injected time remains below the boundary. + Check fourth = factory.nextCheck(); + Assertions.assertTrue(fourth.started - third.finished >= Duration.ofSeconds(1).toNanos()); + Assertions.assertTrue(fourth.proceed().isEmpty()); + + // Completion tracing has its own polling interval. Let it finish before the next check. + await(() -> rows().get(0).getStatus() == ProcessStatus.SUCCESS); + await(() -> service.getTableProcessInstances(identifier).isEmpty()); + Mockito.when(clock.millis()).thenReturn(successTime + 1_000); + Check fifth = factory.nextCheck(); + Assertions.assertTrue(fifth.proceed().isPresent()); + await( + () -> + rows().size() == 2 + && rows().stream().allMatch(r -> r.getStatus() == ProcessStatus.SUCCESS)); + } + + @ParameterizedTest + @CsvSource({"100,8", "300,12"}) + void periodicExpirationCatchesUpWhileWritesContinue(long intervalMillis, int expireLimit) + throws Exception { + nativeCatalog.alterTable( + Identifier.create("db", "tbl"), + List.of( + SchemaChange.setOption("snapshot.num-retained.min", "2"), + SchemaChange.setOption("snapshot.num-retained.max", "2"), + SchemaChange.setOption("snapshot.time-retained", "100 d"), + SchemaChange.setOption("snapshot.expire.limit", String.valueOf(expireLimit))), + false); + FileStoreTable table = (FileStoreTable) nativeCatalog.getTable(Identifier.create("db", "tbl")); + append(table, 1); + table.createTag("first", 1); + for (int id = 2; id <= 24; id++) { + append(table, id); + } + Assertions.assertEquals(22, eligibleSnapshots(table)); + startPeriodicScheduling(intervalMillis); + factory.nextCheck(); + + AtomicInteger committed = new AtomicInteger(24); + AtomicReference writeFailure = new AtomicReference<>(); + continuousWriter = Executors.newSingleThreadScheduledExecutor(); + continuousWrites = + continuousWriter.scheduleWithFixedDelay( + () -> { + try { + append(table, committed.get() + 1); + committed.incrementAndGet(); + } catch (Throwable e) { + writeFailure.set(e); + throw new IllegalStateException("Continuous test writer failed", e); + } + }, + 0, + 1, + TimeUnit.SECONDS); + try { + // At most one new snapshot per second. Allow for the framework's 5-second status poll, + // not just the much shorter coordinator interval, when choosing expiration capacity. + await( + () -> { + Assertions.assertNull(writeFailure.get(), "Writer must remain healthy"); + return committed.get() >= 27 && eligibleSnapshots(table) == 0; + }, + Duration.ofSeconds(90)); + int firstCatchUp = committed.get(); + // Catch up again after further writes: draining only the initial static backlog is not + // enough. + await( + () -> { + Assertions.assertNull(writeFailure.get(), "Writer must remain healthy"); + return committed.get() >= firstCatchUp + 3 && eligibleSnapshots(table) == 0; + }, + Duration.ofSeconds(90)); + Assertions.assertFalse( + continuousWrites.isDone(), "Convergence must happen before the writer stops"); + } finally { + stopContinuousWriter(); + } + Assertions.assertNull(writeFailure.get()); + Mockito.when(scheduledTables.contains(identifier.getId())).thenReturn(false); + await(() -> service.getTableProcessInstances(identifier).isEmpty()); + Assertions.assertTrue( + table.snapshotManager().snapshotExists(table.snapshotManager().latestSnapshotId() - 1)); + List completed = rows(); + Assertions.assertTrue( + completed.size() >= 3, "Convergence must span multiple periodic processes"); + for (OptimizingProcessInfo process : completed) { + Assertions.assertEquals(ProcessStatus.SUCCESS, process.getStatus()); + int expired = Integer.parseInt(process.getSummary().get("reported-expired-snapshots")); + Assertions.assertTrue(expired >= 0 && expired <= expireLimit); + Assertions.assertEquals( + String.valueOf(expireLimit), process.getSummary().get("snapshot.expire.limit")); + } + Assertions.assertEquals( + IntStream.rangeClosed(1, committed.get()).boxed().collect(Collectors.toList()), + readIds(table)); + Assertions.assertEquals(List.of(1), readIds(table.copy(Map.of("scan.tag-name", "first")))); + } + + private void stopContinuousWriter() throws InterruptedException { + if (continuousWriter == null) { + return; + } + if (continuousWrites != null) { + continuousWrites.cancel(false); + } + continuousWriter.shutdown(); + try { + if (!continuousWriter.awaitTermination(15, TimeUnit.SECONDS)) { + continuousWriter.shutdownNow(); + Assertions.assertTrue( + continuousWriter.awaitTermination(40, TimeUnit.SECONDS), + "Continuous writer did not terminate; its table must remain available"); + } + } catch (InterruptedException e) { + continuousWriter.shutdownNow(); + Thread.currentThread().interrupt(); + throw e; + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void cleanupAfterFailureWaitsForNativeWorkAndCallbacks(boolean nativeStillRunning) + throws Exception { + ExecutorService closer = Executors.newSingleThreadExecutor(); + Future cleanup = null; + try { + if (nativeStillRunning) { + nativeReadEntered = new CountDownLatch(1); + releaseNativeRead = new CountDownLatch(1); + } else { + service.pauseCallback = true; + } + service.register( + runtime, factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow()); + if (nativeStillRunning) { + Assertions.assertTrue(nativeReadEntered.await(10, TimeUnit.SECONDS)); + await(() -> rows().size() == 1 && rows().get(0).getStatus() == ProcessStatus.RUNNING); + Assertions.assertTrue( + service.cancelProcess(identifier, Long.parseLong(rows().get(0).getProcessId()))); + } else { + Assertions.assertTrue(service.callbackEntered.await(10, TimeUnit.SECONDS)); + } + // Both paths are already untracked; neither proves the native worker/callback has returned. + Assertions.assertTrue(service.getTableProcessInstances(identifier).isEmpty()); + cleanup = + closer.submit( + (Callable) + () -> { + try { + throw new AssertionError("simulated test abort"); + } finally { + close(); + } + }); + CompletableFuture draining = nativeStillRunning ? engine.draining : service.draining; + CompletableFuture.anyOf(draining, resourcesCleared).get(30, TimeUnit.SECONDS); + Assertions.assertFalse(resourcesCleared.isDone(), "Resources cleared before workers exited"); + Assertions.assertTrue(draining.isDone()); + Assertions.assertFalse(cleanup.isDone()); + Assertions.assertEquals(1, rows().size(), "Derby must remain usable while work is pending"); + } finally { + if (releaseNativeRead != null) { + releaseNativeRead.countDown(); + } + service.releaseCallback.countDown(); + try { + if (cleanup != null) { + Future result = cleanup; + ExecutionException failure = + Assertions.assertThrows( + ExecutionException.class, () -> result.get(40, TimeUnit.SECONDS)); + Assertions.assertInstanceOf(AssertionError.class, failure.getCause()); + Assertions.assertEquals("simulated test abort", failure.getCause().getMessage()); + // Keep worker cleanup independent of the assertions above. + Assertions.assertTrue(service.awaitTermination(40, TimeUnit.SECONDS)); + engine.awaitWorkers(); + } + } finally { + closer.shutdownNow(); + Assertions.assertTrue(closer.awaitTermination(40, TimeUnit.SECONDS)); + } + } + Assertions.assertTrue(resourcesCleared.isDone()); + } + + @Test + void cleanupTimeoutPreservesResourcesUntilSchedulerExits() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference schedulerThread = new AtomicReference<>(); + catalog = Mockito.spy(catalog); + Mockito.doAnswer( + call -> { + if (Thread.currentThread().getName().startsWith("async-")) { + schedulerThread.set(Thread.currentThread()); + entered.countDown(); + // Model catalog IO that cannot respond to cancellation until the read completes. + boolean released = false; + while (!released) { + try { + release.await(); + released = true; + } catch (InterruptedException ignored) { + // The regression releases the external operation in finally. + } + } + throw new IllegalStateException("Test catalog read released"); + } + return call.callRealMethod(); + }) + .when(catalog) + .loadTable("db", "tbl"); + try { + startPeriodicScheduling(100); + Assertions.assertTrue(entered.await(20, TimeUnit.SECONDS)); + // Keep the production scheduler's real 30s + 30s stop path. Only the fixture's + // additional wait is zero, so the still-blocked worker deterministically times out. + schedulerWaitTimeout = Duration.ZERO; + AssertionError failure = Assertions.assertThrows(AssertionError.class, this::close); + Assertions.assertTrue(failure.getMessage().contains("Scheduler worker did not terminate")); + Assertions.assertFalse(resourcesCleared.isDone()); + Assertions.assertTrue(schedulerThread.get().isAlive()); + Assertions.assertNotNull(nativeCatalog.getTable(Identifier.create("db", "tbl"))); + Assertions.assertEquals( + List.of(identifier), + getAs(TableMetaMapper.class, TableMetaMapper::selectAllTableIdentifiers)); + } finally { + release.countDown(); + if (schedulerThread.get() != null) { + schedulerThread.get().join(40_000); + Assertions.assertFalse(schedulerThread.get().isAlive()); + } + schedulerWaitTimeout = Duration.ofSeconds(40); + close(); + } + Assertions.assertTrue(resourcesCleared.isDone()); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void junitTeardownOwnsWarehouseLifetime(boolean schedulerBlocked) throws Exception { + // The launched fixture needs exclusive ownership of the shared Derby tables. + close(); + PaimonWarehouseLifecycle.instance = null; + SummaryGeneratingListener listener = new SummaryGeneratingListener(); + try { + // Run a real Jupiter lifecycle. Expected inner failures are inspected here, not swallowed + // by a test extension before JUnit gets a chance to clean its method-scoped resources. + LauncherFactory.create() + .execute( + LauncherDiscoveryRequestBuilder.request() + .selectors( + DiscoverySelectors.selectMethod( + PaimonWarehouseLifecycle.class, + schedulerBlocked ? "schedulerTimeout" : "bodyFailure")) + .configurationParameter("junit.jupiter.execution.parallel.enabled", "false") + .build(), + listener); + Assertions.assertEquals(1, listener.getSummary().getTestsFoundCount()); + Assertions.assertEquals(1, listener.getSummary().getTestsFailedCount()); + Assertions.assertEquals(1, listener.getSummary().getFailures().size()); + String message = listener.getSummary().getFailures().get(0).getException().getMessage(); + Assertions.assertTrue( + message.contains( + schedulerBlocked ? "Scheduler worker did not terminate" : "simulated test failure"), + message); + PaimonWarehouseLifecycle inner = PaimonWarehouseLifecycle.instance; + Assertions.assertNotNull(inner); + TestPaimonProcessIntegration fixture = inner; + if (schedulerBlocked) { + Assertions.assertTrue(inner.worker.get().isAlive()); + Assertions.assertFalse(fixture.resourcesCleared.isDone()); + Assertions.assertTrue( + Files.exists(fixture.warehouse), + "JUnit removed warehouse while work was still running"); + Assertions.assertNotNull(fixture.nativeCatalog.getTable(Identifier.create("db", "tbl"))); + } else { + Assertions.assertTrue(fixture.resourcesCleared.isDone()); + Assertions.assertFalse(Files.exists(fixture.warehouse)); + } + } finally { + PaimonWarehouseLifecycle inner = PaimonWarehouseLifecycle.instance; + if (inner != null) { + inner.release.countDown(); + if (inner.worker.get() != null) { + inner.worker.get().join(40_000); + Assertions.assertFalse(inner.worker.get().isAlive()); + } + TestPaimonProcessIntegration fixture = inner; + fixture.schedulerWaitTimeout = Duration.ofSeconds(40); + fixture.close(); + Assertions.assertFalse(Files.exists(fixture.warehouse)); + } + PaimonWarehouseLifecycle.instance = null; + } + } + + void startSchedulerWithBlockedRead(CountDownLatch release, AtomicReference worker) + throws Exception { + CountDownLatch entered = new CountDownLatch(1); + catalog = Mockito.spy(catalog); + Mockito.doAnswer( + call -> { + if (Thread.currentThread().getName().startsWith("async-")) { + worker.set(Thread.currentThread()); + entered.countDown(); + Assertions.assertTrue( + Uninterruptibles.awaitUninterruptibly(release, 3, TimeUnit.MINUTES), + "External read gate timed out"); + throw new IllegalStateException("Test catalog read released"); + } + return call.callRealMethod(); + }) + .when(catalog) + .loadTable("db", "tbl"); + startPeriodicScheduling(100); + Assertions.assertTrue(entered.await(20, TimeUnit.SECONDS)); + schedulerWaitTimeout = Duration.ZERO; + } + + private static void awaitGate(CountDownLatch gate) { + try { + Assertions.assertTrue(gate.await(40, TimeUnit.SECONDS), "Test gate timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + private static class AwaitableEngine extends LocalExecutionEngine { + private final List workers = new ArrayList<>(); + private final CompletableFuture draining = new CompletableFuture<>(); + + @Override + protected ThreadPoolExecutor newFixedPool(String tag, int size) { + ThreadPoolExecutor pool = super.newFixedPool(tag, size); + workers.add(pool); + return pool; + } + + void awaitWorkers() throws InterruptedException { + draining.complete(null); + long deadline = System.nanoTime() + Duration.ofSeconds(40).toNanos(); + for (ThreadPoolExecutor worker : workers) { + Assertions.assertTrue( + worker.awaitTermination( + Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS), + "Local worker did not terminate; catalog and Derby must remain available"); + } + } + } + + private static class AwaitableService extends ProcessService { + private final CompletableFuture draining = new CompletableFuture<>(); + private final CountDownLatch callbackEntered = new CountDownLatch(1); + private final CountDownLatch releaseCallback = new CountDownLatch(1); + private volatile boolean pauseCallback; + + AwaitableService( + TableService tables, List coordinators, ExecuteEngineManager engines) { + super(tables, coordinators, engines); + } + + @Override + public TableProcess untrackTableProcessInstance(ServerTableIdentifier table, long processId) { + TableProcess process = super.untrackTableProcessInstance(table, processId); + if (pauseCallback && process != null) { + callbackEntered.countDown(); + awaitGate(releaseCallback); + } + return process; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + draining.complete(null); + return super.awaitTermination(timeout, unit); + } + } + + private static long eligibleSnapshots(FileStoreTable table) { + // This fixture has no consumer and retains exactly the latest two snapshots. A tag protects + // its referenced data separately; it does not turn those two retained snapshots into backlog. + // Read the oldest boundary first so concurrent commits/expiration can only overestimate lag. + long earliest = table.snapshotManager().earliestSnapshotId(); + return Math.max(0, table.snapshotManager().latestSnapshotId() - earliest - 1); + } + + private static void append(FileStoreTable table, int id) throws Exception { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(GenericRow.of(id)); + commit.commit(write.prepareCommit()); + } + } + + private static List readIds(org.apache.paimon.table.Table table) throws Exception { + ReadBuilder builder = table.newReadBuilder(); + List ids = new ArrayList<>(); + try (CloseableIterator rows = + builder.newRead().createReader(builder.newScan().plan()).toCloseableIterator()) { + while (rows.hasNext()) { + ids.add(rows.next().getInt(0)); + } + } + Collections.sort(ids); + return ids; + } + + private long startPeriodicScheduling(long intervalMillis) throws InterruptedException { + service + .getActionCoordinators() + .values() + .forEach(schedulers -> schedulers.forEach(ActionCoordinatorScheduler::dispose)); + service.dispose(); + Assertions.assertTrue(service.awaitTermination(40, TimeUnit.SECONDS)); + factory.open(Map.of("expire-snapshots.interval", intervalMillis + "ms")); + scheduledTables = Mockito.mock(TableService.class); + Mockito.when(scheduledTables.contains(identifier.getId())).thenReturn(true); + ProcessService.ExecuteEngineManager engines = + Mockito.mock(ProcessService.ExecuteEngineManager.class); + Mockito.when(engines.installedPlugins()).thenReturn(List.of(engine)); + service = + new AwaitableService( + scheduledTables, + List.of( + new DefaultActionCoordinator( + TableFormat.PAIMON, PaimonActions.EXPIRE_SNAPSHOTS, factory)), + engines); + factory.observing = true; + // Use the production startup delay and recurring scheduler, not direct trigger/register calls. + long started = System.nanoTime(); + service.getTableHandlerChain().initialize(List.of(runtime)); + return started; + } + + private static class Check { + private final long started = System.nanoTime(); + private long finished; + private final CountDownLatch permit = new CountDownLatch(1); + private final CompletableFuture> result = new CompletableFuture<>(); + + Optional proceed() throws Exception { + permit.countDown(); + return result.get(10, TimeUnit.SECONDS); + } + } + + /** Observe the factory SPI without replacing scheduling, admission or expiration behavior. */ + private static class ObservedFactory extends PaimonProcessFactory { + private final BlockingQueue checks = new LinkedBlockingQueue<>(); + private volatile boolean observing; + private volatile boolean controlChecks; + private volatile boolean stopped; + private volatile Check current; + private final Set schedulerWorkers = new LinkedHashSet<>(); + + ObservedFactory(Clock clock) { + super(clock); + } + + @Override + public Optional trigger(TableRuntime runtime, Action action) { + if (!observing) { + return super.trigger(runtime, action); + } + synchronized (this) { + // Close admission atomically with recording workers: a late check cannot access Catalog. + if (stopped) { + return Optional.empty(); + } + schedulerWorkers.add(Thread.currentThread()); + } + Check check = new Check(); + current = check; + checks.add(check); + try { + if (controlChecks && !check.permit.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting for scheduled check gate"); + } + Optional process = + stopped ? Optional.empty() : super.trigger(runtime, action); + check.finished = System.nanoTime(); + check.result.complete(process); + return process; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + check.result.completeExceptionally(e); + throw new IllegalStateException(e); + } catch (RuntimeException e) { + check.result.completeExceptionally(e); + throw e; + } + } + + Check nextCheck() throws Exception { + Check check = checks.poll(30, TimeUnit.SECONDS); + Assertions.assertNotNull(check, "The production scheduler did not run a check"); + return check; + } + + void awaitSchedulerWorkers(Duration timeout) throws InterruptedException { + List workers; + synchronized (this) { + workers = new ArrayList<>(schedulerWorkers); + } + long deadline = System.nanoTime() + timeout.toNanos(); + for (Thread worker : workers) { + long remaining = deadline - System.nanoTime(); + if (remaining > 0) { + TimeUnit.NANOSECONDS.timedJoin(worker, remaining); + } + // dispose() can return after logging a timeout. Only actual thread exit is sufficient; + // it also covers process registration and scheduler cleanup after trigger() returns. + Assertions.assertFalse( + worker.isAlive(), + "Scheduler worker did not terminate; catalog and Derby must remain available"); + } + } + + synchronized void stopObserving() { + // Releasing a pending check during teardown must not register another native invocation. + stopped = true; + controlChecks = false; + if (current != null) { + current.permit.countDown(); + } + } + } + + private List rows() { + return descriptor + .getOptimizingProcessesInfo( + identifier.getIdentifier().buildTableIdentifier(), null, "CLEANUP", null, 10, 0) + .getLeft(); + } + + private static void await(BooleanSupplier condition) throws Exception { + await(condition, Duration.ofSeconds(40)); + } + + private static void await(BooleanSupplier condition, Duration timeout) throws Exception { + long end = System.nanoTime() + timeout.toNanos(); + while (System.nanoTime() < end) { + if (condition.getAsBoolean()) { + return; + } + Thread.sleep(20); + } + Assertions.fail("Timed out waiting for process lifecycle"); + } + + private static class Plugins extends AbstractPluginManager + implements AutoCloseable { + private final List configs; + + Plugins(List configs) { + super("process-factories"); + this.configs = configs; + } + + @Override + protected List loadPluginConfigurations() { + return configs; + } + } + + private static class Persistence extends DerbyPersistence { + void clear() { + super.after(); + } + } +} diff --git a/amoro-common/src/main/java/org/apache/amoro/PaimonActions.java b/amoro-common/src/main/java/org/apache/amoro/PaimonActions.java new file mode 100644 index 0000000000..fb8a40ee7f --- /dev/null +++ b/amoro-common/src/main/java/org/apache/amoro/PaimonActions.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro; + +/** Maintenance actions supported by Paimon tables. */ +public final class PaimonActions { + public static final Action EXPIRE_SNAPSHOTS = Action.register("expire-snapshots"); + + private PaimonActions() {} +} diff --git a/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java b/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java index 5999de9f01..951a46bb4a 100755 --- a/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java +++ b/amoro-common/src/main/java/org/apache/amoro/process/LocalExecutionEngine.java @@ -163,7 +163,8 @@ private ThreadPoolExecutor getPool(String tag) { return pools.get(DEFAULT_POOL); } - private ThreadPoolExecutor newFixedPool(String tag, int size) { + /** Create a worker pool; subclasses can retain it to await termination in tests. */ + protected ThreadPoolExecutor newFixedPool(String tag, int size) { return new ThreadPoolExecutor( size, size, diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java index 2af18a9417..9da07674b5 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonCatalog.java @@ -19,7 +19,6 @@ package org.apache.amoro.formats.paimon; import org.apache.amoro.AlreadyExistsException; -import org.apache.amoro.AmoroTable; import org.apache.amoro.DatabaseNotEmptyException; import org.apache.amoro.FormatCatalog; import org.apache.amoro.NoSuchDatabaseException; @@ -87,16 +86,22 @@ public void dropDatabase(String database) { } @Override - public AmoroTable loadTable(String database, String table) { + public PaimonTable loadTable(String database, String table) { try { return new PaimonTable( TableIdentifier.of(name, database, table), - catalog.getTable(Identifier.create(database, table))); + catalog.getTable(Identifier.create(database, table)), + () -> reloadTable(database, table)); } catch (Catalog.TableNotExistException e) { throw new NoSuchTableException(e); } } + private PaimonTable reloadTable(String database, String table) { + catalog.invalidateTable(Identifier.create(database, table)); + return loadTable(database, table); + } + @Override public List listTables(String database) { try { diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java index 27a5f57068..c4c6e6ea3e 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java @@ -27,16 +27,31 @@ import org.apache.paimon.table.Table; import java.util.Map; +import java.util.function.Supplier; public class PaimonTable implements AmoroTable { private final TableIdentifier tableIdentifier; private final Table table; + private final Supplier refresh; public PaimonTable(TableIdentifier tableIdentifier, Table table) { + this(tableIdentifier, table, null); + } + + PaimonTable(TableIdentifier tableIdentifier, Table table, Supplier refresh) { this.tableIdentifier = tableIdentifier; this.table = table; + this.refresh = refresh; + } + + /** Reload this table through its owning catalog, invalidating any cached metadata. */ + public PaimonTable refresh() { + if (refresh == null) { + throw new IllegalStateException("Paimon table has no catalog refresh callback"); + } + return refresh.get(); } @Override diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java index 7263dc8a5a..a25874b358 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java @@ -21,6 +21,7 @@ import static org.apache.paimon.operation.FileStoreScan.Plan.groupByPartFiles; import org.apache.amoro.AmoroTable; +import org.apache.amoro.PaimonActions; import org.apache.amoro.TableFormat; import org.apache.amoro.api.CommitMetaProducer; import org.apache.amoro.process.ProcessStatus; @@ -479,8 +480,20 @@ public Map getTableOptimizingTypes(AmoroTable amoroTable) { return types; } + @Override + public Map getTableProcessTypes(AmoroTable table, String processCategory) { + if (ProcessCategory.CLEANUP.getName().equalsIgnoreCase(processCategory)) { + return Collections.singletonMap(PaimonActions.EXPIRE_SNAPSHOTS.getName(), "Expire Snapshots"); + } + return FormatTableDescriptor.super.getTableProcessTypes(table, processCategory); + } + @Override public List getProcessTypesByCategory(String processCategory) { + if (ProcessCategory.CLEANUP.getName().equalsIgnoreCase(processCategory)) { + return Collections.singletonList(PaimonActions.EXPIRE_SNAPSHOTS.getName()); + } + if (ProcessCategory.OPTIMIZING.getName().equalsIgnoreCase(processCategory)) { return OPTIMIZING_TYPES; } diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java new file mode 100644 index 0000000000..c018f7a94d --- /dev/null +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonProcessFactory.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.paimon.process; + +import org.apache.amoro.Action; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.TableFormat; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.config.ConfigOption; +import org.apache.amoro.config.ConfigOptions; +import org.apache.amoro.config.Configurations; +import org.apache.amoro.process.ExecuteEngine; +import org.apache.amoro.process.LocalExecutionEngine; +import org.apache.amoro.process.ProcessFactory; +import org.apache.amoro.process.ProcessTriggerStrategy; +import org.apache.amoro.process.RecoverProcessFailedException; +import org.apache.amoro.process.TableProcess; +import org.apache.amoro.process.TableProcessStore; +import org.apache.amoro.table.StateKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Clock; +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** Paimon maintenance, scheduled and recovered by the common Process framework. */ +public class PaimonProcessFactory implements ProcessFactory { + private static final Logger LOG = LoggerFactory.getLogger(PaimonProcessFactory.class); + public static final ConfigOption EXPIRE_ENABLED = + ConfigOptions.key("expire-snapshots.enabled").booleanType().defaultValue(true); + public static final ConfigOption EXPIRE_INTERVAL = + ConfigOptions.key("expire-snapshots.interval").durationType().noDefaultValue(); + public static final StateKey LAST_SUCCESS_TIME = + StateKey.stateKey("paimon.snapshot-expiration.last-success-time").longType().defaultValue(0L); + + private final Clock clock; + + /** SPI entry point using the system clock. */ + public PaimonProcessFactory() { + this(Clock.systemUTC()); + } + + /** Allow subclasses to supply a clock without changing scheduling or retention policy. */ + protected PaimonProcessFactory(Clock clock) { + this.clock = Objects.requireNonNull(clock, "clock"); + } + + private ExecuteEngine localEngine; + private final Map actions = new HashMap<>(); + + @Override + public String name() { + return "paimon"; + } + + @Override + public void open(Map properties) { + actions.clear(); + Configurations configs = + Configurations.fromMap(properties == null ? Collections.emptyMap() : properties); + if (!configs.getBoolean(EXPIRE_ENABLED)) { + return; + } + if (properties == null || !properties.containsKey(EXPIRE_INTERVAL.key())) { + LOG.info("Paimon snapshot expiration is inactive: no explicit {}", EXPIRE_INTERVAL.key()); + return; + } + Duration interval = configs.getDuration(EXPIRE_INTERVAL); + if (interval.toMillis() < 1) { + throw new IllegalArgumentException("Paimon expire-snapshots.interval must be at least 1 ms"); + } + actions.put(PaimonActions.EXPIRE_SNAPSHOTS, ProcessTriggerStrategy.triggerAtFixRate(interval)); + } + + @Override + public void close() { + actions.clear(); + } + + @Override + public List> requiredStates() { + return Collections.singletonList(LAST_SUCCESS_TIME); + } + + @Override + public Map> supportedActions() { + return actions.isEmpty() + ? Collections.emptyMap() + : Collections.singletonMap(TableFormat.PAIMON, Set.copyOf(actions.keySet())); + } + + @Override + public ProcessTriggerStrategy triggerStrategy(TableFormat format, Action action) { + return format == TableFormat.PAIMON + ? actions.getOrDefault(action, ProcessTriggerStrategy.METADATA_TRIGGER) + : ProcessTriggerStrategy.METADATA_TRIGGER; + } + + @Override + public void availableExecuteEngines(Collection engines) { + localEngine = null; + for (ExecuteEngine engine : engines) { + if (engine instanceof LocalExecutionEngine) { + localEngine = engine; + break; + } + } + } + + @Override + public Optional trigger(TableRuntime runtime, Action action) { + if (runtime.getFormat() != TableFormat.PAIMON || !actions.containsKey(action)) { + return Optional.empty(); + } + + if (PaimonActions.EXPIRE_SNAPSHOTS.equals(action)) { + return triggerExpireSnapshot(runtime); + } + + return Optional.empty(); + } + + private Optional triggerExpireSnapshot(TableRuntime runtime) { + if (localEngine == null) { + return Optional.empty(); + } + ProcessTriggerStrategy strategy = actions.get(PaimonActions.EXPIRE_SNAPSHOTS); + if (clock.millis() - runtime.getState(LAST_SUCCESS_TIME) + < strategy.getTriggerInterval().toMillis()) { + return Optional.empty(); + } + String reason = PaimonSnapshotExpiration.skipReason(PaimonSnapshotExpiration.refresh(runtime)); + if (reason != null) { + if ("governance-disabled".equals(reason)) { + LOG.debug( + "Skip Paimon snapshot expiration for {}: {}", runtime.getTableIdentifier(), reason); + } else { + LOG.info( + "Skip Paimon snapshot expiration for {}: {}", runtime.getTableIdentifier(), reason); + } + return Optional.empty(); + } + return Optional.of(new PaimonSnapshotsExpiringProcess(runtime, localEngine)); + } + + @Override + public TableProcess recover(TableRuntime runtime, TableProcessStore store) + throws RecoverProcessFailedException { + if (localEngine == null + || runtime.getFormat() != TableFormat.PAIMON + || !PaimonActions.EXPIRE_SNAPSHOTS.equals(store.getAction())) { + throw new RecoverProcessFailedException( + "Cannot recover Paimon snapshot expiration: unsupported action/format or no local engine"); + } + return new PaimonSnapshotsExpiringProcess(runtime, localEngine); + } +} diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java new file mode 100644 index 0000000000..b1cf799728 --- /dev/null +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotExpiration.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.paimon.process; + +import org.apache.amoro.AmoroTable; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.formats.paimon.PaimonTable; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.table.AppendOnlyFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.PrimaryKeyFileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.BranchManager; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Table eligibility and native retention configuration for snapshot expiration. */ +final class PaimonSnapshotExpiration { + private static final String ENABLED = "amoro.expire-snapshots.enabled"; + private static final boolean ENABLED_DEFAULT = true; + + private PaimonSnapshotExpiration() {} + + static PaimonTable refresh(TableRuntime runtime) { + AmoroTable table = runtime.loadTable(); + if (!(table instanceof PaimonTable)) { + throw new IllegalArgumentException("Snapshot expiration requires a Paimon table"); + } + return ((PaimonTable) table).refresh(); + } + + /** Null means that this table is eligible; other values are safe diagnostic reason codes. */ + static String skipReason(PaimonTable table) { + Map properties = table.properties(); + if (!Boolean.parseBoolean(properties.getOrDefault(ENABLED, String.valueOf(ENABLED_DEFAULT)))) { + return "governance-disabled"; + } + Table nativeTable = table.originalTable(); + if (!(nativeTable instanceof PrimaryKeyFileStoreTable) + && !(nativeTable instanceof AppendOnlyFileStoreTable)) { + return "unsupported-table-type"; + } + FileStoreTable fileStoreTable = (FileStoreTable) nativeTable; + if (!BranchManager.isMainBranch(fileStoreTable.snapshotManager().branch())) { + return "non-default-branch"; + } + CoreOptions options = fileStoreTable.store().options(); + if (!options.writeOnly()) { + return "write-only-required"; + } + return null; + } + + static ExpireConfig configuration(FileStoreTable table) { + ExpireConfig config = table.store().options().expireConfig(); + validate( + config.getSnapshotRetainMin(), + config.getSnapshotRetainMax(), + config.getSnapshotTimeRetain(), + config.getSnapshotMaxDeletes()); + validate( + config.getChangelogRetainMin(), + config.getChangelogRetainMax(), + config.getChangelogTimeRetain(), + config.getChangelogMaxDeletes()); + return config; + } + + private static void validate(int min, int max, Duration time, int limit) { + if (min < 1 || max < min || time == null || time.isNegative() || limit <= 0) { + throw new IllegalArgumentException("Invalid Paimon retention configuration"); + } + // The native expiration implementation operates in milliseconds. + time.toMillis(); + } + + static Map summary(ExpireConfig config) { + Map result = new LinkedHashMap<>(); + result.put("snapshot.num-retained.min", String.valueOf(config.getSnapshotRetainMin())); + result.put("snapshot.num-retained.max", String.valueOf(config.getSnapshotRetainMax())); + result.put("snapshot.time-retained", config.getSnapshotTimeRetain().toString()); + result.put("snapshot.expire.limit", String.valueOf(config.getSnapshotMaxDeletes())); + result.put("changelog.num-retained.min", String.valueOf(config.getChangelogRetainMin())); + result.put("changelog.num-retained.max", String.valueOf(config.getChangelogRetainMax())); + result.put("changelog.time-retained", config.getChangelogTimeRetain().toString()); + result.put("effective-changelog-max-deletes", String.valueOf(config.getChangelogMaxDeletes())); + return result; + } +} diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java new file mode 100644 index 0000000000..4a6222cd1d --- /dev/null +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/process/PaimonSnapshotsExpiringProcess.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.paimon.process; + +import org.apache.amoro.Action; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.formats.paimon.PaimonTable; +import org.apache.amoro.process.ExecuteEngine; +import org.apache.amoro.process.LocalProcess; +import org.apache.amoro.process.TableProcess; +import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.table.FileStoreTable; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** One native snapshot expiration invocation using the current table's retention settings. */ +public class PaimonSnapshotsExpiringProcess extends TableProcess implements LocalProcess { + public static final String TAG = "paimon-expire-snapshots"; + private volatile Map summary = Collections.emptyMap(); + + public PaimonSnapshotsExpiringProcess(TableRuntime runtime, ExecuteEngine engine) { + super(runtime, engine); + } + + @Override + public String tag() { + return TAG; + } + + @Override + public void run() { + Map result = new LinkedHashMap<>(); + publish(result); + String stage = "load"; + try { + PaimonTable table = PaimonSnapshotExpiration.refresh(tableRuntime); + stage = "config"; + String reason = PaimonSnapshotExpiration.skipReason(table); + if (reason != null) { + result.put("outcome", "skipped"); + result.put("skip-reason", reason); + publish(result); + return; + } + FileStoreTable nativeTable = (FileStoreTable) table.originalTable(); + ExpireConfig config = PaimonSnapshotExpiration.configuration(nativeTable); + result.putAll(PaimonSnapshotExpiration.summary(config)); + publish(result); + stage = "expire"; + int expired = nativeTable.newExpireSnapshots().config(config).expire(); + result.put("outcome", expired == 0 ? "noop" : "executed"); + result.put("reported-expired-snapshots", String.valueOf(expired)); + result.put("result-scope", "last-completed-invocation"); + publish(result); + stage = "state-update"; + tableRuntime.updateState( + PaimonProcessFactory.LAST_SUCCESS_TIME, ignored -> System.currentTimeMillis()); + } catch (Exception e) { + result.put("error-stage", stage); + // Exception messages can contain catalog credentials or signed object-store URLs. + result.put( + "error-message", + "Snapshot expiration failed during " + + stage + + " (" + + e.getClass().getSimpleName() + + "). See AMS logs for details."); + publish(result); + LOG.error( + "Paimon snapshot expiration failed during {} for {}", stage, getTableIdentifier(), e); + throw new RuntimeException("Paimon snapshot expiration failed during " + stage, e); + } + } + + private void publish(Map result) { + summary = Collections.unmodifiableMap(new LinkedHashMap<>(result)); + } + + @Override + public Action getAction() { + return PaimonActions.EXPIRE_SNAPSHOTS; + } + + @Override + public Map getProcessParameters() { + return Collections.emptyMap(); + } + + @Override + public Map getSummary() { + return summary; + } +} diff --git a/amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory b/amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory new file mode 100644 index 0000000000..9f49765bd9 --- /dev/null +++ b/amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# * +# http://www.apache.org/licenses/LICENSE-2.0 +# * +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.amoro.formats.paimon.process.PaimonProcessFactory diff --git a/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java new file mode 100644 index 0000000000..6c90b206dd --- /dev/null +++ b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/TestPaimonTableRefresh.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.paimon; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.paimon.catalog.CachingCatalog; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.types.DataTypes; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; + +class TestPaimonTableRefresh { + @TempDir Path warehouse; + + @Test + void refreshInvalidatesCachedTableWithoutClosingCatalog() throws Exception { + try (Catalog original = + PaimonCatalogFactory.paimonCatalog( + Collections.singletonMap("warehouse", warehouse.toString()), new Configuration()); + Catalog cached = new CachingCatalog(original, new Options())) { + original.createDatabase("db", false); + Identifier id = Identifier.create("db", "tbl"); + original.createTable(id, Schema.newBuilder().column("id", DataTypes.INT()).build(), false); + PaimonCatalog catalog = new PaimonCatalog(cached, "catalog"); + PaimonTable before = (PaimonTable) catalog.loadTable("db", "tbl"); + original.alterTable( + id, SchemaChange.setOption("amoro.expire-snapshots.enabled", "true"), false); + assertNull(before.properties().get("amoro.expire-snapshots.enabled")); + assertSame(before.originalTable(), catalog.loadTable("db", "tbl").originalTable()); + PaimonTable fresh = before.refresh(); + assertEquals("true", fresh.properties().get("amoro.expire-snapshots.enabled")); + original.alterTable(id, SchemaChange.setOption("snapshot.num-retained.min", "2"), false); + assertEquals("2", fresh.refresh().properties().get("snapshot.num-retained.min")); + assertTrue(catalog.tableExists("db", "tbl")); + assertThrows( + IllegalStateException.class, + () -> new PaimonTable(before.id(), before.originalTable()).refresh()); + } + } +} diff --git a/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java new file mode 100644 index 0000000000..378bcfb6bc --- /dev/null +++ b/amoro-format-paimon/src/test/java/org/apache/amoro/formats/paimon/process/TestPaimonSnapshotExpiration.java @@ -0,0 +1,707 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.paimon.process; + +import org.apache.amoro.Action; +import org.apache.amoro.AmoroTable; +import org.apache.amoro.PaimonActions; +import org.apache.amoro.ServerTableIdentifier; +import org.apache.amoro.TableFormat; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.config.TableConfiguration; +import org.apache.amoro.formats.paimon.PaimonCatalog; +import org.apache.amoro.formats.paimon.PaimonCatalogFactory; +import org.apache.amoro.metrics.MetricRegistry; +import org.apache.amoro.process.LocalExecutionEngine; +import org.apache.amoro.process.LocalProcess; +import org.apache.amoro.process.ProcessFactory; +import org.apache.amoro.process.RecoverProcessFailedException; +import org.apache.amoro.process.TableProcess; +import org.apache.amoro.process.TableProcessStore; +import org.apache.amoro.table.StateKey; +import org.apache.hadoop.conf.Configuration; +import org.apache.paimon.catalog.CachingCatalog; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.consumer.Consumer; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.StreamTableScan; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.CloseableIterator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.function.Function; +import java.util.stream.StreamSupport; + +class TestPaimonSnapshotExpiration { + @TempDir Path warehouse; + Catalog original; + Catalog cached; + PaimonCatalog catalog; + final Identifier id = Identifier.create("db", "tbl"); + final LocalExecutionEngine engine = new LocalExecutionEngine(); + + @BeforeEach + void setup() throws Exception { + original = + PaimonCatalogFactory.paimonCatalog( + Collections.singletonMap("warehouse", warehouse.toString()), new Configuration()); + original.createDatabase("db", false); + cached = new CachingCatalog(original, new Options()); + catalog = new PaimonCatalog(cached, "catalog"); + } + + @AfterEach + void close() throws Exception { + cached.close(); + engine.close(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void expiresUsingTableRetentionAndLimitAndPreservesCurrentData(boolean primaryKey) + throws Exception { + createTable( + primaryKey, + Map.of( + "snapshot.num-retained.min", + "1", + "snapshot.num-retained.max", + "2", + "snapshot.time-retained", + "100 d", + "snapshot.expire.limit", + "1")); + append(1, 4); + Runtime runtime = new Runtime(); + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + process.run(); + Assertions.assertEquals(2L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals("executed", process.getSummary().get("outcome")); + Assertions.assertEquals("1", process.getSummary().get("reported-expired-snapshots")); + Assertions.assertEquals("2", process.getSummary().get("snapshot.num-retained.max")); + Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0); + Assertions.assertEquals(List.of(1, 2, 3, 4), readIds(table())); + process.run(); + Assertions.assertEquals(3L, table().snapshotManager().earliestSnapshotId()); + process.run(); + Assertions.assertEquals("noop", process.getSummary().get("outcome")); + Assertions.assertEquals("0", process.getSummary().get("reported-expired-snapshots")); + } + + @Test + void nativeDefaultsAndEmptyTableAreReportedWithoutOverrides() throws Exception { + createTable(false, Collections.emptyMap()); + PaimonSnapshotsExpiringProcess process = + new PaimonSnapshotsExpiringProcess(new Runtime(), engine); + process.run(); + Assertions.assertEquals("noop", process.getSummary().get("outcome")); + Assertions.assertEquals("10", process.getSummary().get("snapshot.num-retained.min")); + Assertions.assertEquals("2147483647", process.getSummary().get("snapshot.num-retained.max")); + Assertions.assertEquals("PT1H", process.getSummary().get("snapshot.time-retained")); + Assertions.assertEquals("50", process.getSummary().get("snapshot.expire.limit")); + Assertions.assertEquals("10", process.getSummary().get("changelog.num-retained.min")); + } + + @ParameterizedTest + @CsvSource({ + "false,changelog.num-retained.min,2", + "true,changelog.num-retained.min,2", + "false,changelog.num-retained.max,5", + "true,changelog.num-retained.max,5", + "false,changelog.time-retained,200 d", + "true,changelog.time-retained,200 d" + }) + void independentChangelogRetentionPreservesHistory(boolean primaryKey, String key, String value) + throws Exception { + Map options = + new HashMap<>( + Map.of( + "snapshot.num-retained.min", "1", + "snapshot.num-retained.max", "2", + "snapshot.time-retained", "100 d")); + if (primaryKey) { + options.put("changelog-producer", "input"); + } + options.put(key, value); + createTable(primaryKey, options); + append(1, 8); + Assertions.assertTrue(table().store().options().changelogLifecycleDecoupled()); + Runtime runtime = new Runtime(); + TableProcess process = factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow(); + ((LocalProcess) process).run(); + Assertions.assertEquals("executed", process.getSummary().get("outcome")); + Assertions.assertEquals("6", process.getSummary().get("reported-expired-snapshots")); + Assertions.assertEquals(7L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertTrue(runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME) > 0); + Assertions.assertEquals(List.of(1L, 2L, 3L, 4L, 5L, 6L), archivedChangelogIds()); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6, 7, 8), readIds(table())); + + // Resume a consumer from an expired snapshot, reading actual retained changelog files. + ReadBuilder reader = table().newReadBuilder(); + StreamTableScan scan = reader.newStreamScan(); + scan.restore(1L); + List history = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + try (CloseableIterator rows = + reader.newRead().createReader(scan.plan()).toCloseableIterator()) { + while (rows.hasNext()) { + history.add(rows.next().getInt(0)); + } + } + } + Collections.sort(history); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6, 7, 8), history); + ((LocalProcess) process).run(); + Assertions.assertEquals("noop", process.getSummary().get("outcome")); + // A separate changelog expiry would delete some of these archives; snapshot expiry must not. + Assertions.assertEquals(List.of(1L, 2L, 3L, 4L, 5L, 6L), archivedChangelogIds()); + } + + @Test + void queuedAndRecoveredProcessesAcceptIndependentChangelogRetention() throws Exception { + createTable(false, Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "2")); + append(1, 4); + Runtime runtime = new Runtime(); + PaimonProcessFactory factory = factory(); + TableProcess queued = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow(); + original.alterTable(id, SchemaChange.setOption("changelog.num-retained.max", "5"), false); + ((LocalProcess) queued).run(); + Assertions.assertEquals("executed", queued.getSummary().get("outcome")); + Assertions.assertEquals("5", queued.getSummary().get("changelog.num-retained.max")); + Assertions.assertEquals(List.of(1L, 2L), archivedChangelogIds()); + append(5, 6); + TableProcessStore store = Mockito.mock(TableProcessStore.class); + Mockito.when(store.getAction()).thenReturn(PaimonActions.EXPIRE_SNAPSHOTS); + TableProcess recovered = factory.recover(runtime, store); + original.alterTable(id, SchemaChange.setOption("changelog.num-retained.max", "6"), false); + ((LocalProcess) recovered).run(); + Assertions.assertEquals("executed", recovered.getSummary().get("outcome")); + Assertions.assertEquals("6", recovered.getSummary().get("changelog.num-retained.max")); + Assertions.assertEquals(5L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals(List.of(1L, 2L, 3L, 4L), archivedChangelogIds()); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6), readIds(table())); + } + + @ParameterizedTest + @CsvSource({ + "amoro.expire-snapshots.enabled,false,governance-disabled", + "write-only,false,write-only-required" + }) + void queuedProcessRechecksEligibility(String key, String value, String reason) throws Exception { + createTable(false, Collections.emptyMap()); + Runtime runtime = new Runtime(); + PaimonProcessFactory factory = factory(); + TableProcess process = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow(); + original.alterTable(id, SchemaChange.setOption(key, value), false); + ((LocalProcess) process).run(); + Assertions.assertEquals("skipped", process.getSummary().get("outcome")); + Assertions.assertEquals(reason, process.getSummary().get("skip-reason")); + Assertions.assertFalse(process.getSummary().containsKey("reported-expired-snapshots")); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + } + + @Test + void queuedAndRecoveredProcessesReadLatestPolicy() throws Exception { + createTable( + true, + Map.of( + "snapshot.num-retained.min", + "1", + "snapshot.num-retained.max", + "3", + "snapshot.time-retained", + "100 d", + "snapshot.expire.limit", + "1")); + append(1, 6); + Runtime runtime = new Runtime(); + PaimonProcessFactory factory = factory(); + TableProcess queued = factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).orElseThrow(); + original.alterTable( + id, + List.of( + SchemaChange.setOption("snapshot.num-retained.max", "2"), + SchemaChange.setOption("snapshot.expire.limit", "10")), + false); + ((LocalProcess) queued).run(); + Assertions.assertEquals(5L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals("10", queued.getSummary().get("snapshot.expire.limit")); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + Assertions.assertTrue(queued.getProcessParameters().isEmpty()); + TableProcessStore store = Mockito.mock(TableProcessStore.class); + Mockito.when(store.getAction()).thenReturn(PaimonActions.EXPIRE_SNAPSHOTS); + original.alterTable(id, SchemaChange.setOption("snapshot.num-retained.max", "1"), false); + TableProcess recovered = factory.recover(runtime, store); + ((LocalProcess) recovered).run(); + Assertions.assertEquals(6L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5, 6), readIds(table())); + } + + @Test + void nativeResultSurvivesStateFailureAndRetryReplacesSummary() throws Exception { + createTable(false, Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "1")); + append(1, 3); + Runtime runtime = new Runtime(); + runtime.failStateUpdate = true; + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + Assertions.assertThrows(RuntimeException.class, process::run); + Map failed = process.getSummary(); + Assertions.assertEquals("executed", failed.get("outcome")); + Assertions.assertEquals("2", failed.get("reported-expired-snapshots")); + Assertions.assertEquals("state-update", failed.get("error-stage")); + Assertions.assertEquals(3L, table().snapshotManager().earliestSnapshotId()); + original.alterTable( + id, SchemaChange.setOption("amoro.expire-snapshots.enabled", "false"), false); + process.run(); + Assertions.assertEquals("skipped", process.getSummary().get("outcome")); + Assertions.assertFalse(process.getSummary().containsKey("reported-expired-snapshots")); + Assertions.assertFalse(process.getSummary().containsKey("error-message")); + Assertions.assertEquals("executed", failed.get("outcome")); + Assertions.assertThrows( + UnsupportedOperationException.class, () -> failed.put("outcome", "bad")); + } + + @Test + void loadingFailureIsNotSkippedAndSummaryDoesNotLeakCredentials() throws Exception { + createTable(false, Collections.emptyMap()); + Runtime runtime = new Runtime(); + runtime.loader = + () -> { + throw new IllegalStateException("password=secret https://user:secret@host"); + }; + Assertions.assertThrows( + IllegalStateException.class, + () -> factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS)); + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + Assertions.assertThrows(RuntimeException.class, process::run); + Assertions.assertEquals("load", process.getSummary().get("error-stage")); + Assertions.assertFalse(process.getSummary().containsKey("outcome")); + Assertions.assertFalse(process.getSummary().get("error-message").contains("secret")); + runtime.loader = () -> catalog.loadTable("db", "tbl"); + process.run(); + Assertions.assertEquals("noop", process.getSummary().get("outcome")); + Assertions.assertFalse(process.getSummary().containsKey("error-message")); + } + + @Test + void invalidLimitFailsWithoutDeletingSnapshots() throws Exception { + createTable( + false, + Map.of( + "snapshot.num-retained.min", + "1", + "snapshot.num-retained.max", + "1", + "snapshot.expire.limit", + "0")); + append(1, 3); + PaimonSnapshotsExpiringProcess process = + new PaimonSnapshotsExpiringProcess(new Runtime(), engine); + Assertions.assertThrows(RuntimeException.class, process::run); + Assertions.assertEquals("config", process.getSummary().get("error-stage")); + Assertions.assertFalse(process.getSummary().containsKey("outcome")); + Assertions.assertEquals(1L, table().snapshotManager().earliestSnapshotId()); + } + + @Test + void nativeTagAndConsumerProtectionRemainEffective() throws Exception { + createTable(true, Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "1")); + append(1, 1); + table().createTag("first", 1); + append(2, 5); + table().consumerManager().resetConsumer("reader", new Consumer(3)); + PaimonSnapshotsExpiringProcess process = + new PaimonSnapshotsExpiringProcess(new Runtime(), engine); + process.run(); + Assertions.assertTrue(table().snapshotManager().snapshotExists(3)); + Assertions.assertEquals(List.of(1), readIds(table().copy(Map.of("scan.tag-name", "first")))); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5), readIds(table())); + table().consumerManager().deleteConsumer("reader"); + process.run(); + Assertions.assertEquals(5L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals(List.of(1), readIds(table().copy(Map.of("scan.tag-name", "first")))); + } + + @ParameterizedTest + @ValueSource(strings = {"omitted", "true", "false"}) + void tableExpirationSwitchDefaultsToEnabled(String enabled) throws Exception { + createTable( + false, + "omitted".equals(enabled) + ? Map.of("snapshot.num-retained.min", "1", "snapshot.num-retained.max", "1") + : Map.of( + "snapshot.num-retained.min", "1", + "snapshot.num-retained.max", "1", + "amoro.expire-snapshots.enabled", enabled)); + append(1, 3); + Runtime runtime = new Runtime(); + boolean expected = !"false".equals(enabled); + Assertions.assertEquals( + expected, factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + process.run(); + Assertions.assertEquals(expected ? 3L : 1L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals(expected ? "executed" : "skipped", process.getSummary().get("outcome")); + if (!expected) { + Assertions.assertEquals("governance-disabled", process.getSummary().get("skip-reason")); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + original.alterTable(id, SchemaChange.removeOption("amoro.expire-snapshots.enabled"), false); + Assertions.assertTrue(factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + process.run(); + Assertions.assertEquals(3L, table().snapshotManager().earliestSnapshotId()); + } + } + + @Test + void pluginExpirationSwitchControlsActionRegistration() throws Exception { + createTable(false, Collections.emptyMap()); + Runtime runtime = new Runtime(); + PaimonProcessFactory factory = factory(); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + factory.open(Map.of("expire-snapshots.enabled", "false", "expire-snapshots.interval", "1h")); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + // A disabled action does not require or parse its interval, as in the Iceberg factory. + factory.open( + Map.of("expire-snapshots.enabled", "false", "expire-snapshots.interval", "invalid")); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + factory.open(Map.of("expire-snapshots.enabled", "false")); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + factory.open(Map.of("expire-snapshots.enabled", "true")); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + factory.open(Map.of("expire-snapshots.enabled", "true", "expire-snapshots.interval", "1h")); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + } + + @Test + void factoryDispatchOnlyAcceptsRegisteredPaimonActions() { + PaimonProcessFactory factory = factory(); + TableRuntime runtime = Mockito.mock(TableRuntime.class); + Action unsupported = Action.register("test-paimon-unsupported"); + Mockito.when(runtime.getFormat()).thenReturn(TableFormat.PAIMON); + Assertions.assertTrue(factory.trigger(runtime, unsupported).isEmpty()); + Mockito.when(runtime.getFormat()).thenReturn(TableFormat.ICEBERG); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + factory.close(); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + Mockito.when(runtime.getFormat()).thenReturn(TableFormat.PAIMON); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + // Rejected dispatches must not read table state or load and refresh the table. + Mockito.verify(runtime, Mockito.atLeastOnce()).getFormat(); + Mockito.verifyNoMoreInteractions(runtime); + } + + @Test + void factoryRequiresExplicitIntervalAndLocalEngine() throws Exception { + PaimonProcessFactory factory = new PaimonProcessFactory(); + factory.open(Collections.emptyMap()); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + factory.open(Map.of("expire-snapshots.interval", "1h")); + Assertions.assertEquals( + java.time.Duration.ofHours(1), + factory + .triggerStrategy(TableFormat.PAIMON, PaimonActions.EXPIRE_SNAPSHOTS) + .getTriggerInterval()); + createTable(false, Collections.emptyMap()); + Runtime runtime = new Runtime(); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + TableProcessStore store = Mockito.mock(TableProcessStore.class); + Mockito.when(store.getAction()).thenReturn(PaimonActions.EXPIRE_SNAPSHOTS); + Assertions.assertThrows( + RecoverProcessFailedException.class, () -> factory.recover(runtime, store)); + factory.availableExecuteEngines(List.of(engine)); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + factory.open(Collections.emptyMap()); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + Assertions.assertTrue( + StreamSupport.stream(ServiceLoader.load(ProcessFactory.class).spliterator(), false) + .anyMatch(f -> f instanceof PaimonProcessFactory)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "0ms", "-1s", "1ns", "invalid", "9223372036854775807d"}) + void invalidIntervalsDoNotRegisterActions(String interval) { + PaimonProcessFactory factory = new PaimonProcessFactory(); + Assertions.assertThrows( + RuntimeException.class, () -> factory.open(Map.of("expire-snapshots.interval", interval))); + Assertions.assertTrue(factory.supportedActions().isEmpty()); + } + + @Test + void defaultBranchAndOrdinaryTableScopeAreCheckedBeforeExecution() throws Exception { + createTable(true, Collections.emptyMap()); + append(1, 1); + table().createBranch("development"); + Runtime runtime = new Runtime(); + runtime.loader = () -> catalog.loadTable("db", "tbl$branch_development"); + Assertions.assertTrue(factory().trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + process.run(); + Assertions.assertEquals("non-default-branch", process.getSummary().get("skip-reason")); + runtime.loader = () -> catalog.loadTable("db", "tbl$snapshots"); + process.run(); + Assertions.assertEquals("skipped", process.getSummary().get("outcome")); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void localEngineUsesDedicatedPoolOrExistingDefaultFallback(boolean configured) throws Exception { + createTable(false, Collections.emptyMap()); + engine.open(configured ? Map.of("pool.paimon-expire-snapshots.thread-count", "1") : Map.of()); + Runtime runtime = new Runtime(); + java.util.concurrent.atomic.AtomicReference thread = + new java.util.concurrent.atomic.AtomicReference<>(); + runtime.loader = + () -> { + thread.set(Thread.currentThread().getName()); + return catalog.loadTable("db", "tbl"); + }; + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + String executionId = engine.submitTableProcess(process); + long deadline = System.nanoTime() + java.time.Duration.ofSeconds(10).toNanos(); + while (engine.getStatus(executionId) == org.apache.amoro.process.ProcessStatus.RUNNING + && System.nanoTime() < deadline) { + Thread.sleep(10); + } + Assertions.assertEquals( + org.apache.amoro.process.ProcessStatus.SUCCESS, engine.getStatus(executionId)); + Assertions.assertTrue( + thread.get().startsWith(configured ? "local-paimon-expire-snapshots-" : "local-default-")); + } + + @Test + void nativeFailureProducesFailureSummaryWithoutCallingChangelogExpiration() throws Exception { + createTable(true, Collections.emptyMap()); + org.apache.paimon.table.PrimaryKeyFileStoreTable nativeTable = + Mockito.spy((org.apache.paimon.table.PrimaryKeyFileStoreTable) table()); + org.apache.paimon.table.ExpireSnapshots expiration = + Mockito.mock(org.apache.paimon.table.ExpireSnapshots.class); + Mockito.doReturn(expiration).when(nativeTable).newExpireSnapshots(); + Mockito.when(expiration.config(Mockito.any())).thenReturn(expiration); + Mockito.when(expiration.expire()).thenThrow(new IllegalStateException("secret-token=hidden")); + Catalog failingCatalog = Mockito.mock(Catalog.class); + Mockito.when(failingCatalog.getTable(id)).thenReturn(nativeTable); + PaimonCatalog wrapper = new PaimonCatalog(failingCatalog, "catalog"); + Runtime runtime = new Runtime(); + runtime.loader = () -> wrapper.loadTable("db", "tbl"); + PaimonSnapshotsExpiringProcess process = new PaimonSnapshotsExpiringProcess(runtime, engine); + Assertions.assertThrows(RuntimeException.class, process::run); + Assertions.assertEquals("expire", process.getSummary().get("error-stage")); + Assertions.assertFalse(process.getSummary().containsKey("reported-expired-snapshots")); + Assertions.assertFalse(process.getSummary().get("error-message").contains("hidden")); + Mockito.verify(nativeTable, Mockito.never()).newExpireChangelog(); + Assertions.assertEquals(0L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + } + + @Test + void nativeTimeAndMinimumRetentionAreRespectedAndChangelogSettingsPreserved() throws Exception { + createTable( + false, + Map.of( + "snapshot.num-retained.min", + "2", + "snapshot.time-retained", + "100 d", + "changelog.num-retained.min", + "1", + "changelog.time-retained", + "1 d", + "snapshot.expire.limit", + "7")); + append(1, 5); + PaimonSnapshotsExpiringProcess process = + new PaimonSnapshotsExpiringProcess(new Runtime(), engine); + process.run(); + Assertions.assertEquals(1L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals("noop", process.getSummary().get("outcome")); + Assertions.assertEquals("1", process.getSummary().get("changelog.num-retained.min")); + Assertions.assertEquals("PT24H", process.getSummary().get("changelog.time-retained")); + Assertions.assertEquals("7", process.getSummary().get("effective-changelog-max-deletes")); + original.alterTable( + id, + List.of( + SchemaChange.setOption("snapshot.time-retained", "0 ms"), + SchemaChange.setOption("changelog.time-retained", "0 ms")), + false); + process.run(); + Assertions.assertEquals(4L, table().snapshotManager().earliestSnapshotId()); + Assertions.assertEquals(List.of(1, 2, 3, 4, 5), readIds(table())); + } + + @Test + void factoryUsesControlledTimeAtSuccessIntervalBoundary() throws Exception { + createTable(false, Collections.emptyMap()); + Runtime runtime = new Runtime(); + runtime.updateState(PaimonProcessFactory.LAST_SUCCESS_TIME, ignored -> 10_000L); + java.time.Clock clock = Mockito.mock(java.time.Clock.class); + PaimonProcessFactory factory = new PaimonProcessFactory(clock); + factory.open(Map.of("expire-snapshots.interval", "1s")); + factory.availableExecuteEngines(List.of(engine)); + + Mockito.when(clock.millis()).thenReturn(10_999L); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isEmpty()); + Mockito.when(clock.millis()).thenReturn(11_000L); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + Mockito.when(clock.millis()).thenReturn(11_001L); + Assertions.assertTrue(factory.trigger(runtime, PaimonActions.EXPIRE_SNAPSHOTS).isPresent()); + Assertions.assertEquals(10_000L, runtime.getState(PaimonProcessFactory.LAST_SUCCESS_TIME)); + } + + private PaimonProcessFactory factory() { + PaimonProcessFactory factory = new PaimonProcessFactory(); + factory.open(Map.of("expire-snapshots.interval", "1h")); + factory.availableExecuteEngines(List.of(engine)); + return factory; + } + + void createTable(boolean primaryKey, Map options) throws Exception { + Schema.Builder builder = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("write-only", "true") + .option("bucket", primaryKey ? "1" : "-1"); + if (primaryKey) { + builder.primaryKey("id"); + } + options.forEach(builder::option); + original.createTable(id, builder.build(), false); + } + + FileStoreTable table() throws Exception { + return (FileStoreTable) original.getTable(id); + } + + private List archivedChangelogIds() throws Exception { + List ids = new ArrayList<>(); + table().changelogManager().safelyGetAllChangelogs().forEach(log -> ids.add(log.id())); + Collections.sort(ids); + return ids; + } + + void append(int start, int end) throws Exception { + for (int i = start; i <= end; i++) { + BatchWriteBuilder builder = table().newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(GenericRow.of(i)); + commit.commit(write.prepareCommit()); + } + } + } + + static List readIds(org.apache.paimon.table.Table table) throws Exception { + ReadBuilder builder = table.newReadBuilder(); + List ids = new ArrayList<>(); + try (CloseableIterator rows = + builder.newRead().createReader(builder.newScan().plan()).toCloseableIterator()) { + while (rows.hasNext()) { + ids.add(rows.next().getInt(0)); + } + } + Collections.sort(ids); + return ids; + } + + class Runtime implements TableRuntime { + private final Map, Object> states = new HashMap<>(); + java.util.function.Supplier> loader = () -> catalog.loadTable("db", "tbl"); + boolean failStateUpdate; + + @Override + public AmoroTable loadTable() { + return loader.get(); + } + + @Override + public ServerTableIdentifier getTableIdentifier() { + return ServerTableIdentifier.of(1L, "catalog", "db", "tbl", TableFormat.PAIMON); + } + + @Override + public T getState(StateKey key) { + return (T) states.getOrDefault(key, key.getDefaultValue()); + } + + @Override + public void updateState(StateKey key, Function updater) { + if (failStateUpdate) { + throw new IllegalStateException("injected state failure"); + } + states.put(key, updater.apply(getState(key))); + } + + @Override + public List getProcessStates() { + return Collections.emptyList(); + } + + @Override + public List getProcessStates(Action action) { + return Collections.emptyList(); + } + + @Override + public String getGroupName() { + return "default"; + } + + @Override + public TableConfiguration getTableConfiguration() { + throw new UnsupportedOperationException(); + } + + @Override + public Map getTableConfig() { + throw new UnsupportedOperationException(); + } + + @Override + public void registerMetric(MetricRegistry registry) {} + + @Override + public void unregisterMetric() {} + } +} diff --git a/amoro-web/src/views/tables/components/Optimizing.vue b/amoro-web/src/views/tables/components/Optimizing.vue index dac192b47d..d41cc1547b 100644 --- a/amoro-web/src/views/tables/components/Optimizing.vue +++ b/amoro-web/src/views/tables/components/Optimizing.vue @@ -163,7 +163,7 @@ async function refreshOptimizingProcesses() { } } -async function cancel() { +async function cancel(targetProcessId: string | number = processId.value) { Modal.confirm({ title: t(props.cancelModalTitleKey), onOk: async () => { @@ -171,7 +171,7 @@ async function cancel() { loading.value = true await cancelOptimizingProcess({ ...sourceData, - processId: processId.value?.toString(), + processId: targetProcessId.toString(), }) cancelDisabled.value = true refresh() @@ -305,7 +305,8 @@ onMounted(() => {