Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions .github/workflows/core-hadoop3-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,56 @@ jobs:
if: matrix.spark == '3.3'
run: ./mvnw clean install -pl '!amoro-ams,!dist' -Djacoco.flink.skip=true -B -ntp -Pjava${{ matrix.jdk }} -Pspark-${{ matrix.spark }}

# Standard-directory tests already run in the full build above.
- name: Test Paimon AMS integration
if: matrix.spark == '3.5'
run: |
./mvnw test -pl amoro-ams -am -B -ntp \
-Pjava${{ matrix.jdk }},support-paimon-format,spark-3.5,skip-dashboard-build \
-Dtest=TestPaimonProcessIntegration \
-Dsurefire.failIfNoSpecifiedTests=false
python3 - <<'PYTHON'
import xml.etree.ElementTree as ET
report = ET.parse('amoro-ams/target/surefire-reports/TEST-org.apache.amoro.server.process.TestPaimonProcessIntegration.xml').getroot()
assert int(report.get('tests', '0')) > 0, 'Paimon AMS integration tests did not run'
assert all(int(report.get(key, '0')) == 0 for key in ('failures', 'errors', 'skipped')), report.attrib
PYTHON

- name: Code coverage
uses: codecov/codecov-action@v3
with:
verbose: true
flags: core

paimon-1-1-1:
name: Paimon 1.1.1 format regression with JDK 17
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: maven
check-latest: false
- name: Test Paimon 1.1.1 and Scala 2.13
run: |
./mvnw clean test -pl amoro-format-paimon -am -B -ntp \
-Pjava17,scala-2.13 \
-Dtest=TestPaimonSnapshotExpiration,TestPaimonTableRefresh \
-Dsurefire.failIfNoSpecifiedTests=false
python3 - <<'PYTHON'
import xml.etree.ElementTree as ET
for test_class in (
'org.apache.amoro.formats.paimon.process.TestPaimonSnapshotExpiration',
'org.apache.amoro.formats.paimon.TestPaimonTableRefresh',
):
report = ET.parse(f'amoro-format-paimon/target/surefire-reports/TEST-{test_class}.xml').getroot()
assert int(report.get('tests', '0')) > 0, f'{test_class} did not run'
assert all(int(report.get(key, '0')) == 0 for key in ('failures', 'errors', 'skipped')), report.attrib
PYTHON

spark-optimizer-4-1:
name: Build Spark 4.1 Optimizer with JDK 17
runs-on: ubuntu-latest
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ resources
!javadoc/resources
!amoro-ams/src/main/resources/
!amoro-ams/src/main/resources/**
!amoro-format-paimon/src/main/resources/
!amoro-format-paimon/src/main/resources/META-INF/
!amoro-format-paimon/src/main/resources/META-INF/services/
!amoro-format-paimon/src/main/resources/META-INF/services/org.apache.amoro.process.ProcessFactory

!dist/src/main/amoro-bin/bin/
!dist/src/main/amoro-bin/conf/
57 changes: 57 additions & 0 deletions amoro-ams/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,12 @@
<version>${swagger-ui.version}</version>
</dependency>

<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<scope>test</scope>
</dependency>

<!-- testcontainers dependencies -->
<dependency>
<groupId>org.testcontainers</groupId>
Expand Down Expand Up @@ -603,6 +609,41 @@
</dependencies>

<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${maven-build-helper-plugin.version}</version>
<executions>
<execution>
<id>paimon-integration-tests</id>
<goals>
<goal>add-test-source</goal>
</goals>
<phase>generate-test-sources</phase>
<configuration>
<sources>
<source>src/test/paimon</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<configuration>
<java>
<includes>
<include>src/main/java/**/*.java</include>
<include>src/test/**/*.java</include>
</includes>
</java>
</configuration>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
Expand Down Expand Up @@ -706,6 +747,14 @@
<artifactId>amoro-format-paimon</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>support-hudi-format</id>
Expand Down Expand Up @@ -757,6 +806,14 @@
<artifactId>amoro-format-lance</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>fail-on-no-git-dir</id>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ private void startOptimizingService() throws Exception {
catalogManager,
optimizerManager,
tableService,
processService,
bucketAssignStore,
haContainer);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import org.apache.amoro.server.persistence.mapper.OptimizerMapper;
import org.apache.amoro.server.persistence.mapper.ResourceMapper;
import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
import org.apache.amoro.server.process.ProcessService;
import org.apache.amoro.server.process.TableProcessMeta;
import org.apache.amoro.server.resource.Containers;
import org.apache.amoro.server.resource.OptimizerInstance;
Expand Down Expand Up @@ -137,6 +138,7 @@ public class DefaultOptimizingService extends StatedPersistentBase
private final CatalogManager catalogManager;
private final OptimizerManager optimizerManager;
private final TableService tableService;
private final ProcessService processService;
private final RuntimeHandlerChain tableHandlerChain;
private final ExecutorService planExecutor;
private final BucketAssignStore bucketAssignStore;
Expand All @@ -148,6 +150,7 @@ public DefaultOptimizingService(
CatalogManager catalogManager,
OptimizerManager optimizerManager,
TableService tableService,
ProcessService processService,
BucketAssignStore bucketAssignStore,
HighAvailabilityContainer haContainer) {
this.optimizerTouchTimeout =
Expand All @@ -170,6 +173,7 @@ public DefaultOptimizingService(
this.groupMaxKeepingAttempts =
serviceConfig.getInteger(AmoroManagementConf.OPTIMIZER_GROUP_MAX_KEEPING_ATTEMPTS);
this.tableService = tableService;
this.processService = processService;
this.catalogManager = catalogManager;
this.optimizerManager = optimizerManager;
this.bucketAssignStore = bucketAssignStore;
Expand Down Expand Up @@ -499,16 +503,18 @@ public boolean cancelProcess(long processId) {
return false;
}
long tableId = processMeta.getTableId();
DefaultTableRuntime tableRuntime = (DefaultTableRuntime) tableService.getRuntime(tableId);
TableRuntime tableRuntime = tableService.getRuntime(tableId);
if (tableRuntime == null) {
return false;
}
OptimizingProcess process = tableRuntime.getOptimizingProcess();
if (process == null || process.getProcessId() != processId) {
return false;
if (tableRuntime instanceof DefaultTableRuntime) {
OptimizingProcess process = ((DefaultTableRuntime) tableRuntime).getOptimizingProcess();
if (process != null && process.getProcessId() == processId) {
process.close(true);
return true;
}
}
process.close(true);
return true;
return processService.cancelProcess(tableRuntime.getTableIdentifier(), processId);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,11 @@ public Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(
if (identifier == null) {
return Pair.of(Collections.emptyList(), 0);
}
if (TableProcessQuery.handles(this, processCategory, type)) {
return new TableProcessQuery()
.list(
identifier, getProcessTypesByCategory(processCategory), type, status, limit, offset);
}
int total = 0;
// page helper is 1-based
int pageNumber = (offset / limit) + 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,16 @@ public Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(
int offset) {
AmoroTable<?> amoroTable = loadTable(tableIdentifier);
FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format());
if (TableProcessQuery.handles(formatTableDescriptor, processCategory, type)) {
return new TableProcessQuery()
.list(
amoroTable.id(),
formatTableDescriptor.getProcessTypesByCategory(processCategory),
type,
status,
limit,
offset);
}
return formatTableDescriptor.getOptimizingProcessesInfo(
amoroTable, type, processCategory, status, limit, offset);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.amoro.server.dashboard;

import com.github.pagehelper.Page;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import org.apache.amoro.ServerTableIdentifier;
import org.apache.amoro.process.ProcessStatus;
import org.apache.amoro.server.persistence.PersistentBase;
import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
import org.apache.amoro.server.persistence.mapper.TableProcessMapper;
import org.apache.amoro.server.process.TableProcessMeta;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.descriptor.FilesStatistics;
import org.apache.amoro.table.descriptor.FormatTableDescriptor;
import org.apache.amoro.table.descriptor.OptimizingProcessInfo;
import org.apache.amoro.table.descriptor.ProcessCategory;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;

import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

/** Format-independent queries of AMS cleanup records, without optimizing task metrics. */
public class TableProcessQuery extends PersistentBase {
static boolean handles(FormatTableDescriptor descriptor, String category, String type) {
return ProcessCategory.CLEANUP.getName().equalsIgnoreCase(category)
&& (StringUtils.isBlank(type)
|| descriptor.getProcessTypesByCategory(category).contains(type));
}

Pair<List<OptimizingProcessInfo>, Integer> list(
TableIdentifier table,
List<String> actions,
String type,
ProcessStatus status,
int limit,
int offset) {
ServerTableIdentifier identifier =
getAs(
TableMetaMapper.class,
mapper ->
mapper.selectTableIdentifier(
table.getCatalog(), table.getDatabase(), table.getTableName()));
return list(identifier, actions, type, status, limit, offset);
}

public Pair<List<OptimizingProcessInfo>, Integer> list(
ServerTableIdentifier table,
List<String> actions,
String type,
ProcessStatus status,
int limit,
int offset) {
String filterType = StringUtils.isBlank(type) ? null : type;
if (table == null || (filterType == null && (actions == null || actions.isEmpty()))) {
return Pair.of(Collections.emptyList(), 0);
}
try (Page<?> ignored = PageHelper.startPage(offset / limit + 1, limit, true)) {
List<TableProcessMeta> records =
getAs(
TableProcessMapper.class,
mapper ->
mapper.listProcessMeta(
table.getId(), filterType, filterType == null ? actions : null, status));
PageInfo<TableProcessMeta> page = new PageInfo<>(records);
if (page.getSize() == 0) {
return Pair.of(Collections.emptyList(), 0);
}
return Pair.of(
records.stream().map(meta -> toInfo(table, meta)).collect(Collectors.toList()),
(int) page.getTotal());
}
}

private static OptimizingProcessInfo toInfo(ServerTableIdentifier table, TableProcessMeta meta) {
OptimizingProcessInfo result = new OptimizingProcessInfo();
result.setTableId(meta.getTableId());
result.setCatalogName(table.getCatalog());
result.setDbName(table.getDatabase());
result.setTableName(table.getTableName());
result.setProcessId(String.valueOf(meta.getProcessId()));
result.setStartTime(meta.getCreateTime());
result.setFinishTime(meta.getFinishTime());
result.setDuration(
(meta.getFinishTime() > 0 ? meta.getFinishTime() : System.currentTimeMillis())
- meta.getCreateTime());
result.setOptimizingType(meta.getProcessType());
result.setStatus(meta.getStatus());
result.setFailReason(meta.getFailMessage());
result.setInputFiles(new FilesStatistics());
result.setOutputFiles(new FilesStatistics());
result.setSummary(meta.getSummary());
return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -706,7 +706,10 @@ public void cancelOptimizingProcess(Context ctx) {
OptimizingClientPools.getClient(
AmsUtil.getAMSThriftAddress(serviceConfig, Constants.THRIFT_OPTIMIZING_SERVICE_NAME));
try {
client.cancelProcess(processId);
Preconditions.checkState(
client.cancelProcess(processId),
"Process could not be canceled: it is no longer active or is not managed by this AMS, processId: %s",
processId);
} catch (TException e) {
throw new IllegalStateException("Failed to cancel optimizing process:" + e.getMessage());
}
Expand Down
Loading
Loading