From 1f578c913b4a9d388010ee31a99ce0dea048b98d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 12 Oct 2026 17:08:20 +0300 Subject: [PATCH 01/20] IGNITE-29165 [ducktests] Add reproducers of a full disk volume under a server node --- .../ignitetest/services/utils/ignite_aware.py | 32 ++++++++++++++++++- .../ignite_configuration/data_storage.py | 2 ++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index e0f81ab35b785..0717b2106cdeb 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -48,6 +48,9 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMix """ The base class to build services aware of Ignite. """ + # Source name of the volumes mount_tmpfs makes, so that clean_node finds them in /proc/mounts. + TMPFS_SOURCE = "ignitetest-tmpfs" + @constructible class NetPart(IntEnum): """ @@ -79,6 +82,8 @@ def __init__(self, context, config, num_nodes, startup_timeout_sec, shutdown_tim self.disconnected_nodes = [] + self.tmpfs_volumes = {} + @property def product(self): return str(self.config.version) @@ -109,7 +114,20 @@ def await_started(self, nodes=None): self.await_event("Topology snapshot", self.startup_timeout_sec, nodes=nodes, from_the_beginning=True) + def mount_tmpfs(self, path, size_mb): + """ + Puts a size-limited tmpfs over a directory on every node of the service, so that writes into it fail with + "No space left on device" once size_mb is used. The mount is made when a node starts, before the service + creates its directories and configs, and is removed when the node is cleaned. Mounting needs sudo, which the + ducker user has. + :param path: Absolute path on the node, e.g. log_dir, or persistent_root for one volume holding everything. + :param size_mb: Volume size in megabytes. + """ + self.tmpfs_volumes[path] = size_mb + def start_node(self, node, **kwargs): + self.__mount_tmpfs_volumes(node) + self.init_shared(node) self.init_persistent(node) @@ -186,7 +204,19 @@ def clean(self, **kwargs): def clean_node(self, node, **kwargs): super().clean_node(node, **kwargs) - node.account.ssh("rm -rf -- %s" % self.persistent_root, allow_fail=False) + # A mount point can't be removed, so the volumes go first, the nested ones before their parents. Every volume + # mount_tmpfs made on the node goes, including one left by an aborted run: they share the source name. + node.account.ssh(f"awk '$1 == \"{self.TMPFS_SOURCE}\" {{print $2}}' /proc/mounts | sort -r | " + f"xargs -r -n1 sudo umount -l; rm -rf -- {self.persistent_root}", allow_fail=False) + + def __mount_tmpfs_volumes(self, node): + """ + Mounts the volumes registered by mount_tmpfs, the parents before the nested ones. + """ + for path, size_mb in sorted(self.tmpfs_volumes.items()): + node.account.ssh(f"mountpoint -q {path} || (mkdir -p {path} && sudo mount -t tmpfs " + f"-o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} {path})", + allow_fail=False) def init_persistent(self, node): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py index 54e5be421668e..6dfa2739ee783 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py @@ -42,6 +42,7 @@ class DataStorageConfiguration(NamedTuple): """ Ignite DataStorage configuration """ + cdc_wal_directory_max_size: int = None cdc_wal_path: str = None checkpoint_frequency: int = None default: DataRegionConfiguration = DataRegionConfiguration() @@ -50,6 +51,7 @@ class DataStorageConfiguration(NamedTuple): metrics_rate_time_interval: int = None page_size: int = None regions: list = [] + storage_path: str = None wal_archive_path: str = None wal_buffer_size: int = None wal_compaction_enabled: bool = None From 07d1009f5105f1f39adfc8587ab3de5bd2cd26cc Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 12 Oct 2026 17:08:39 +0300 Subject: [PATCH 02/20] IGNITE-29165 Add disk-full load application --- .../disk_full/DiskFullLoadApplication.java | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java new file mode 100644 index 0000000000000..a99d825062ad6 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Thin-client load for the disk-full tests. Puts values of {@code valueSize} bytes under increasing keys, so the + * partitions and the WAL keep growing, and every {@code binaryTypeEvery}-th put stores a value of a new binary type, + * so the server keeps writing binary metadata. A failed put is counted and retried after a pause, so the load + * outlives a server node that stops. Once a second it records its counters as {@code progress-> <-}. + */ +public class DiskFullLoadApplication extends IgniteAwareApplication { + /** Successful puts after which the application reports it is initialized. */ + private static final int WARM_UP_PUTS = 1000; + + /** Pause after a failed put, in milliseconds. */ + private static final long ERROR_PAUSE = 200; + + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + String cacheName = jsonNode.get("cacheName").asText(); + int binaryTypeEvery = jsonNode.get("binaryTypeEvery").asInt(); + byte[] val = new byte[jsonNode.get("valueSize").asInt()]; + + ClientCache cache = client.getOrCreateCache(cacheName); + + long key = 0; + long puts = 0; + long errs = 0; + long nextReport = System.currentTimeMillis() + 1000; + + while (active()) { + try { + if (key % binaryTypeEvery == 0) + cache.put(key, client.binary().builder("DiskFullType" + key).setField("val", val).build()); + else + cache.put(key, val); + + key++; + puts++; + + // The first record comes before the marker, so the test can read the counters once start() returns. + if (puts == WARM_UP_PUTS) { + recordResult("progress", puts + " " + errs); + + markInitialized(); + } + } + catch (Exception e) { + errs++; + + log.warn("Put failed [key=" + key + ", err=" + e + ']'); + + Thread.sleep(ERROR_PAUSE); + } + + long now = System.currentTimeMillis(); + + if (now >= nextReport) { + recordResult("progress", puts + " " + errs); + + nextReport = now + 1000; + } + } + + markFinished(); + } +} From 849d9fe1a48b8414090404d22d61f52576ce877a Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 12 Oct 2026 17:14:23 +0300 Subject: [PATCH 03/20] IGNITE-29165 Add disk-full tests --- .../tests/ignitetest/tests/disk_full_test.py | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 modules/ducktests/tests/ignitetest/tests/disk_full_test.py diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py new file mode 100644 index 0000000000000..ab6320f6958a8 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -0,0 +1,335 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Fills one volume of a persistent server node and checks what the node does. +""" +import os +import time + +from ducktape.mark import matrix +from ducktape.utils.util import wait_until + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteThinClientConfiguration, \ + DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_spec import envs_to_exports +from ignitetest.services.utils.path import IgnitePathAware +from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + +MB = 1024 * 1024 + +FILLER = "filler" + +STDOUT = "ignite.out" + +STDERR = "ignite.err" + + +class StockLogConfigTemplate: + """ + The distribution's own config/ignite-log4j.xml, with its log directory, ${sys:IGNITE_HOME}/work/log, moved to the + service's log directory. The file is read on the test driver, which, like every ducker container, has the + distributions under install_root. + """ + @staticmethod + def render(service, **kwargs): + """ + :param service: Ignite service the config is for. + :return: Text of the log4j2 config. + """ + with open(os.path.join(service.home_dir, "config", "ignite-log4j.xml"), encoding="utf-8") as stock: + return stock.read().replace("${sys:IGNITE_HOME}/work/log", service.log_dir) + + +def production_launch_spec(base_spec): + """ + :param base_spec: Spec class to inherit from: the service's own, so that a NodeSpec set in globals still applies. + :return: Spec class that starts a server node the way a production setup does: with the distribution's own + config/ignite-log4j.xml, and with stdout and stderr redirected to files in the log directory instead of + piped through tee into console.log. A JVM that fails to start leaves its output in those files. + """ + class ProductionLaunchSpec(base_spec): + """ + Starts a server node the way a production setup does. + """ + def config_templates(self): + return [(name, StockLogConfigTemplate() if name == IgnitePathAware.IGNITE_LOG_CONFIG_NAME else template) + for name, template in super().config_templates()] + + def command(self, node): + return "%s %s %s %s > %s 2> %s &" % \ + (envs_to_exports(self.envs()), + self.service.script("ignite.sh"), + self._jvm_opts(), + self.config_file_path(), + os.path.join(self.service.log_dir, STDOUT), + os.path.join(self.service.log_dir, STDERR)) + + return ProductionLaunchSpec + + +class DiskFullTest(IgniteTest): + """ + Each test mounts a size-limited tmpfs over a directory of a persistent server node, runs a thin-client load, + fills the volume and checks what the node does. The tests record today's behavior: which full volume a node + survives and which one stops it. + """ + LOAD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication" + + @cluster(num_nodes=2) + @ignite_versions(str(DEV_BRANCH)) + @matrix(layout=["separate", "separate_jfr", "shared_with_db"]) + def test_log_volume_full(self, ignite_version, layout): + """ + Fills the volume holding the node's logs: its log4j2 files, stdout, stderr and gc.log. The node starts the way + a production setup does (production_launch_spec). + - separate: the logs alone on a 32 MB volume. The node goes on serving puts and control.sh answers. The + node's log stops growing while the volume is full and grows again once space is freed. + - separate_jfr: the same volume, plus a JFR recording with its repository in the log directory. The JVM + exits. + - shared_with_db: one 256 MB volume holds the logs and the work directory with the partitions, the WAL and + its archive. The node stops, as with a full data volume. Whether the reason reaches the log depends on + the space left at that moment, so the test checks only that the process ends. + """ + ignite = IgniteService(self.test_context, self.__server_config(ignite_version), num_nodes=1) + + jfr_opts = [] + + if layout == "separate_jfr": + jfr_opts = [f"-XX:StartFlightRecording=filename={ignite.log_dir}/recording.jfr", + f"-XX:FlightRecorderOptions=repository={ignite.log_dir}/jfr"] + + ignite.spec = production_launch_spec(ignite.spec.__class__)( + service=ignite, jvm_opts=ignite.spec.user_jvm_opts + jfr_opts, + merge_with_default=ignite.spec.merge_with_default) + + if layout == "shared_with_db": + ignite.mount_tmpfs(ignite.persistent_root, 256) + else: + ignite.mount_tmpfs(ignite.log_dir, 32) + + node = ignite.nodes[0] + + load = self.__start(ignite) + + try: + fill(ignite, ignite.log_dir) + + if layout == "separate": + log_size = ignite.get_file_size(node, node.log_file) + + self.__check_serves(ignite, load) + + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + + assert ignite.get_file_size(node, node.log_file) == log_size, \ + "The node's log grew while its volume was full" + + free(ignite, ignite.log_dir) + + wait_until(lambda: ignite.get_file_size(node, node.log_file) > log_size, timeout_sec=30, + err_msg="The node's log did not grow after space was freed") + else: + self.__check_stops(ignite) + finally: + free(ignite, ignite.log_dir) + + load.stop() + + @cluster(num_nodes=2) + @ignite_versions(str(DEV_BRANCH)) + @matrix(dir=["wal", "archive", "cdc", "db", "work"]) + def test_volume_full(self, ignite_version, dir): + """ + Fills the volume of one node directory. The WAL, its archive, the partitions and the work directory each have + a directory of their own, and the one under test gets a size-limited volume. The node starts with the default + launch. + - wal: the WAL work directory, archiver on. Its segments are preallocated, so the node goes on serving puts. + - archive: the WAL archive. The archiver fails, and the node stops. + - cdc: CDC on, with no consumer and no cdcWalDirectoryMaxSize, and its directory on the archive's volume, + since hard links need one file system. The links keep archived segments, the volume fills up by itself, + and the node stops when the archiver fails. + - db: the partition files. The checkpoint fails, and the node stops. + - work: the work directory with the binary metadata. Writing the next binary type fails, and the node stops. + A node that stops must log the failure handler's FailureContext of the expected type, caused by "No space + left on device". + """ + ignite = IgniteService(self.test_context, self.__server_config(ignite_version), num_nodes=1) + + root = ignite.persistent_root + + volumes = { + "wal": (os.path.join(root, "wal"), 64), + "archive": (os.path.join(root, "archive"), 64), + "cdc": (os.path.join(root, "archive"), 96), + "db": (os.path.join(root, "storage"), 128), + "work": (ignite.work_dir, 32) + } + + failures = { + "archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), + "cdc": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), + "db": ("CRITICAL_ERROR", "StorageException"), + "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore") + } + + path, size_mb = volumes[dir] + + data_storage = ignite.config.data_storage._replace( + storage_path=volumes["db"][0], wal_path=volumes["wal"][0], wal_archive_path=volumes["archive"][0], + max_wal_archive_size=32 * MB) + + if dir == "cdc": + data_storage = data_storage._replace(cdc_wal_path=os.path.join(volumes["cdc"][0], "cdc"), + default=data_storage.default._replace(cdc_enabled=True)) + + ignite.config = ignite.config._replace(data_storage=data_storage) + + ignite.mount_tmpfs(path, size_mb) + + node = ignite.nodes[0] + + load = self.__start(ignite) + + try: + if dir != "cdc": + fill(ignite, path) + + if dir == "wal": + segments = ignite.exec_command(node, f"ls {path}/*/*.wal | wc -l").strip() + + assert int(segments) > 0, f"No WAL segments on the volume {path}" + + self.__check_serves(ignite, load) + else: + self.__check_stops(ignite, timeout_sec=120 if dir == "cdc" else 60) + + check_failure(ignite, *failures[dir]) + finally: + free(ignite, path) + + load.stop() + + @staticmethod + def __server_config(ignite_version): + """ + :return: Config of a persistent server node with small WAL segments, frequent checkpoints and a thin-client + connector. + """ + return IgniteConfiguration( + version=IgniteVersion(ignite_version), + client_connector_configuration=ClientConnectorConfiguration(), + metrics_log_frequency=5000, + data_storage=DataStorageConfiguration( + checkpoint_frequency=1000, + wal_segment_size=4 * MB, + max_wal_archive_size=64 * MB, + default=DataRegionConfiguration(persistence_enabled=True, max_size=256 * MB))) + + def __start(self, ignite): + """ + Starts the node, activates the cluster and starts the load. + :return: Load application service, already putting data and recording its progress. + """ + ignite.start() + + ControlUtility(ignite).activate() + + address = f"{ignite.nodes[0].account.hostname}:{ignite.config.client_connector_configuration.port}" + + load = IgniteApplicationService( + self.test_context, + IgniteThinClientConfiguration(addresses=[address], version=ignite.config.version), + java_class_name=self.LOAD_APP, + params={"cacheName": "disk-full", "valueSize": 1024, "binaryTypeEvery": 1000}) + + load.start() + + return load + + @staticmethod + def __check_serves(ignite, load, duration_sec=15): + """ + Checks that the node stays up and the load's puts go on without errors. + """ + puts, errors = progress(load) + + time.sleep(duration_sec) + + assert ignite.alive(ignite.nodes[0]), "The node stopped" + + puts_after, errors_after = progress(load) + + assert puts_after > puts, f"No puts in {duration_sec} s" + assert errors_after == errors, f"{errors_after - errors} puts failed in {duration_sec} s" + + @staticmethod + def __check_stops(ignite, timeout_sec=60): + """ + Checks that the node's process ends. + """ + wait_until(lambda: not ignite.alive(ignite.nodes[0]), timeout_sec=timeout_sec, + err_msg=f"The node is still up {timeout_sec} s after its volume filled up") + + +def fill(ignite, path): + """ + Writes a file into the directory until its volume is full, and checks that less than 1 MB stays free. + """ + node = ignite.nodes[0] + + node.account.ssh(f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M", allow_fail=True) + + free_bytes = int(ignite.exec_command(node, f"df --output=avail -B1 {path} | tail -1").strip()) + + assert free_bytes < MB, f"{free_bytes} bytes are free on the volume {path} after the fill" + + +def free(ignite, path): + """ + Removes the file fill() wrote. + """ + ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=True) + + +def check_failure(ignite, failure_type, cause): + """ + Checks that the node's log records the failure handler's FailureContext of the given type, with the cause and + "No space left on device" in the lines that follow it. + """ + node = ignite.nodes[0] + + failure = ignite.exec_command(node, f"grep -h -m1 -F -A60 'FailureContext [type={failure_type}' " + f"{node.log_file} || true") + + assert failure, f"No {failure_type} failure in the node's log" + assert cause in failure, f"The {failure_type} failure is not caused by '{cause}': {failure[:2000]}" + assert "No space left on device" in failure, f"The {failure_type} failure is not ENOSPC: {failure[:2000]}" + + +def progress(load): + """ + :return: Puts and failed puts the load application recorded last. + """ + puts, errors = load.extract_results("progress")[-1].split() + + return int(puts), int(errors) From cbd4f6bcef7d69efb3205480e3334732934867f1 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 15 Oct 2026 15:19:01 +0300 Subject: [PATCH 04/20] IGNITE-29165 Split tmpfs unmount from clean_node --- .../ignitetest/services/utils/ignite_aware.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 0717b2106cdeb..b75f27b6e62c0 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -48,7 +48,7 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMix """ The base class to build services aware of Ignite. """ - # Source name of the volumes mount_tmpfs makes, so that clean_node finds them in /proc/mounts. + # Mount source of the volumes mount_tmpfs makes: clean_node finds them by it. TMPFS_SOURCE = "ignitetest-tmpfs" @constructible @@ -116,12 +116,7 @@ def await_started(self, nodes=None): def mount_tmpfs(self, path, size_mb): """ - Puts a size-limited tmpfs over a directory on every node of the service, so that writes into it fail with - "No space left on device" once size_mb is used. The mount is made when a node starts, before the service - creates its directories and configs, and is removed when the node is cleaned. Mounting needs sudo, which the - ducker user has. - :param path: Absolute path on the node, e.g. log_dir, or persistent_root for one volume holding everything. - :param size_mb: Volume size in megabytes. + Mounts a tmpfs of size_mb over the path on every node when it starts, so writes there fail once it is full. """ self.tmpfs_volumes[path] = size_mb @@ -204,20 +199,26 @@ def clean(self, **kwargs): def clean_node(self, node, **kwargs): super().clean_node(node, **kwargs) - # A mount point can't be removed, so the volumes go first, the nested ones before their parents. Every volume - # mount_tmpfs made on the node goes, including one left by an aborted run: they share the source name. - node.account.ssh(f"awk '$1 == \"{self.TMPFS_SOURCE}\" {{print $2}}' /proc/mounts | sort -r | " - f"xargs -r -n1 sudo umount -l; rm -rf -- {self.persistent_root}", allow_fail=False) + self.__unmount_tmpfs_volumes(node) + + node.account.ssh("rm -rf -- %s" % self.persistent_root, allow_fail=False) def __mount_tmpfs_volumes(self, node): """ - Mounts the volumes registered by mount_tmpfs, the parents before the nested ones. + Mounts the volumes registered by mount_tmpfs, unless they are still mounted from the node's previous start. """ - for path, size_mb in sorted(self.tmpfs_volumes.items()): + for path, size_mb in self.tmpfs_volumes.items(): node.account.ssh(f"mountpoint -q {path} || (mkdir -p {path} && sudo mount -t tmpfs " f"-o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} {path})", allow_fail=False) + def __unmount_tmpfs_volumes(self, node): + """ + Unmounts every volume mount_tmpfs made on the node, also one left by an aborted run. + """ + node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET | xargs -r -n1 sudo umount -l", + allow_fail=False) + def init_persistent(self, node): """ Init persistent directory. From 15d0c71e09887c43750c802ae50145614918e5bf Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 15 Oct 2026 15:21:05 +0300 Subject: [PATCH 05/20] IGNITE-29165 Rework disk-full tests after review --- .../tests/ignitetest/tests/disk_full_test.py | 318 ++++++++---------- 1 file changed, 143 insertions(+), 175 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index ab6320f6958a8..eab54e05099cd 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -14,7 +14,7 @@ # limitations under the License. """ -Fills one volume of a persistent server node and checks what the node does. +Fills one volume of a persistent server node under load and checks whether the node survives. """ import os import time @@ -28,8 +28,7 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteThinClientConfiguration, \ DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration -from ignitetest.services.utils.ignite_spec import envs_to_exports -from ignitetest.services.utils.path import IgnitePathAware +from ignitetest.services.utils.jvm_utils import merge_jvm_settings from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest @@ -37,218 +36,172 @@ MB = 1024 * 1024 -FILLER = "filler" - -STDOUT = "ignite.out" - -STDERR = "ignite.err" - - -class StockLogConfigTemplate: - """ - The distribution's own config/ignite-log4j.xml, with its log directory, ${sys:IGNITE_HOME}/work/log, moved to the - service's log directory. The file is read on the test driver, which, like every ducker container, has the - distributions under install_root. - """ - @staticmethod - def render(service, **kwargs): - """ - :param service: Ignite service the config is for. - :return: Text of the log4j2 config. - """ - with open(os.path.join(service.home_dir, "config", "ignite-log4j.xml"), encoding="utf-8") as stock: - return stock.read().replace("${sys:IGNITE_HOME}/work/log", service.log_dir) - - -def production_launch_spec(base_spec): - """ - :param base_spec: Spec class to inherit from: the service's own, so that a NodeSpec set in globals still applies. - :return: Spec class that starts a server node the way a production setup does: with the distribution's own - config/ignite-log4j.xml, and with stdout and stderr redirected to files in the log directory instead of - piped through tee into console.log. A JVM that fails to start leaves its output in those files. - """ - class ProductionLaunchSpec(base_spec): - """ - Starts a server node the way a production setup does. - """ - def config_templates(self): - return [(name, StockLogConfigTemplate() if name == IgnitePathAware.IGNITE_LOG_CONFIG_NAME else template) - for name, template in super().config_templates()] +# A full tmpfs still takes writes into the rest of a file's last page. +PAGE = 4096 - def command(self, node): - return "%s %s %s %s > %s 2> %s &" % \ - (envs_to_exports(self.envs()), - self.service.script("ignite.sh"), - self._jvm_opts(), - self.config_file_path(), - os.path.join(self.service.log_dir, STDOUT), - os.path.join(self.service.log_dir, STDERR)) +FILLER = "filler" - return ProductionLaunchSpec +# Failure type and failed writer the node logs when the volume fills. +FAILURES = { + "wal_archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), + "partitions": ("CRITICAL_ERROR", "StorageException"), + "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore") +} class DiskFullTest(IgniteTest): """ - Each test mounts a size-limited tmpfs over a directory of a persistent server node, runs a thin-client load, - fills the volume and checks what the node does. The tests record today's behavior: which full volume a node - survives and which one stops it. + Each test puts a size-limited tmpfs over a directory of the node, fills it and records what the node does today. """ LOAD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication" @cluster(num_nodes=2) @ignite_versions(str(DEV_BRANCH)) - @matrix(layout=["separate", "separate_jfr", "shared_with_db"]) - def test_log_volume_full(self, ignite_version, layout): + def test_node_survives_full_log_volume(self, ignite_version): """ - Fills the volume holding the node's logs: its log4j2 files, stdout, stderr and gc.log. The node starts the way - a production setup does (production_launch_spec). - - separate: the logs alone on a 32 MB volume. The node goes on serving puts and control.sh answers. The - node's log stops growing while the volume is full and grows again once space is freed. - - separate_jfr: the same volume, plus a JFR recording with its repository in the log directory. The JVM - exits. - - shared_with_db: one 256 MB volume holds the logs and the work directory with the partitions, the WAL and - its archive. The node stops, as with a full data volume. Whether the reason reaches the log depends on - the space left at that moment, so the test checks only that the process ends. + The log volume holds the log4j2 log, console.log and gc.log. """ - ignite = IgniteService(self.test_context, self.__server_config(ignite_version), num_nodes=1) - - jfr_opts = [] - - if layout == "separate_jfr": - jfr_opts = [f"-XX:StartFlightRecording=filename={ignite.log_dir}/recording.jfr", - f"-XX:FlightRecorderOptions=repository={ignite.log_dir}/jfr"] - - ignite.spec = production_launch_spec(ignite.spec.__class__)( - service=ignite, jvm_opts=ignite.spec.user_jvm_opts + jfr_opts, - merge_with_default=ignite.spec.merge_with_default) + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.log_dir, 32) - if layout == "shared_with_db": - ignite.mount_tmpfs(ignite.persistent_root, 256) - else: - ignite.mount_tmpfs(ignite.log_dir, 32) + load = self.__start(ignite) - node = ignite.nodes[0] + check_log_grows(ignite, seconds=15) - load = self.__start(ignite) + fill_volume(ignite, ignite.log_dir) + assert free_space(ignite, ignite.log_dir) == 0 - try: - fill(ignite, ignite.log_dir) + self.__check_serves_load(ignite, load, seconds=15) + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" - if layout == "separate": - log_size = ignite.get_file_size(node, node.log_file) + free_volume(ignite, ignite.log_dir) - self.__check_serves(ignite, load) + check_log_grows(ignite, seconds=15) - assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + load.stop() - assert ignite.get_file_size(node, node.log_file) == log_size, \ - "The node's log grew while its volume was full" + @cluster(num_nodes=2) + @ignite_versions(str(DEV_BRANCH)) + def test_jvm_exits_on_full_jfr_volume(self, ignite_version): + """ + JFR ends the JVM once it can't write its recording. + """ + ignite = self.__ignite(ignite_version) + jfr_opts = ["-XX:StartFlightRecording", f"-XX:FlightRecorderOptions=repository={ignite.jfr_dir}"] + ignite.spec.jvm_opts = merge_jvm_settings(ignite.spec.jvm_opts, jfr_opts) + ignite.mount_tmpfs(ignite.jfr_dir, 32) - free(ignite, ignite.log_dir) + self.__start(ignite) - wait_until(lambda: ignite.get_file_size(node, node.log_file) > log_size, timeout_sec=30, - err_msg="The node's log did not grow after space was freed") - else: - self.__check_stops(ignite) - finally: - free(ignite, ignite.log_dir) + fill_volume(ignite, ignite.jfr_dir) + assert free_space(ignite, ignite.jfr_dir) == 0 - load.stop() + self.__check_stops(ignite) @cluster(num_nodes=2) @ignite_versions(str(DEV_BRANCH)) - @matrix(dir=["wal", "archive", "cdc", "db", "work"]) - def test_volume_full(self, ignite_version, dir): + def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version): """ - Fills the volume of one node directory. The WAL, its archive, the partitions and the work directory each have - a directory of their own, and the one under test gets a size-limited volume. The node starts with the default - launch. - - wal: the WAL work directory, archiver on. Its segments are preallocated, so the node goes on serving puts. - - archive: the WAL archive. The archiver fails, and the node stops. - - cdc: CDC on, with no consumer and no cdcWalDirectoryMaxSize, and its directory on the archive's volume, - since hard links need one file system. The links keep archived segments, the volume fills up by itself, - and the node stops when the archiver fails. - - db: the partition files. The checkpoint fails, and the node stops. - - work: the work directory with the binary metadata. Writing the next binary type fails, and the node stops. - A node that stops must log the failure handler's FailureContext of the expected type, caused by "No space - left on device". + The default layout: logs in the work directory, next to the data. """ - ignite = IgniteService(self.test_context, self.__server_config(ignite_version), num_nodes=1) + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.persistent_root, 256) - root = ignite.persistent_root + self.__start(ignite) - volumes = { - "wal": (os.path.join(root, "wal"), 64), - "archive": (os.path.join(root, "archive"), 64), - "cdc": (os.path.join(root, "archive"), 96), - "db": (os.path.join(root, "storage"), 128), - "work": (ignite.work_dir, 32) - } + fill_volume(ignite, ignite.log_dir) + assert free_space(ignite, ignite.log_dir) == 0 - failures = { - "archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), - "cdc": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), - "db": ("CRITICAL_ERROR", "StorageException"), - "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore") - } + self.__check_stops(ignite) - path, size_mb = volumes[dir] + @cluster(num_nodes=2) + @ignite_versions(str(DEV_BRANCH)) + def test_node_survives_full_wal_volume(self, ignite_version): + """ + WAL segments are preallocated, so puts go on into a full WAL volume. + """ + ignite = self.__ignite(ignite_version) + wal = os.path.join(ignite.persistent_root, "wal") + ignite.mount_tmpfs(wal, 64) - data_storage = ignite.config.data_storage._replace( - storage_path=volumes["db"][0], wal_path=volumes["wal"][0], wal_archive_path=volumes["archive"][0], - max_wal_archive_size=32 * MB) + load = self.__start(ignite) - if dir == "cdc": - data_storage = data_storage._replace(cdc_wal_path=os.path.join(volumes["cdc"][0], "cdc"), - default=data_storage.default._replace(cdc_enabled=True)) + segments = ignite.exec_command(ignite.nodes[0], f"ls {wal}/*/*.wal | wc -l") + assert int(segments) > 0, "No WAL segments on the volume" - ignite.config = ignite.config._replace(data_storage=data_storage) + fill_volume(ignite, wal) + assert free_space(ignite, wal) == 0 - ignite.mount_tmpfs(path, size_mb) + self.__check_serves_load(ignite, load, seconds=15) - node = ignite.nodes[0] + load.stop() - load = self.__start(ignite) + @cluster(num_nodes=2) + @ignite_versions(str(DEV_BRANCH)) + @matrix(volume=["wal_archive", "partitions", "work"]) + def test_node_stops_on_full_volume(self, ignite_version, volume): + """ + The work directory holds the binary metadata. + """ + ignite = self.__ignite(ignite_version) + path = os.path.join(ignite.persistent_root, volume) + ignite.mount_tmpfs(path, 64) - try: - if dir != "cdc": - fill(ignite, path) + self.__start(ignite) - if dir == "wal": - segments = ignite.exec_command(node, f"ls {path}/*/*.wal | wc -l").strip() + fill_volume(ignite, path) + assert free_space(ignite, path) == 0 - assert int(segments) > 0, f"No WAL segments on the volume {path}" + self.__check_stops(ignite) + check_failure(ignite, *FAILURES[volume]) - self.__check_serves(ignite, load) - else: - self.__check_stops(ignite, timeout_sec=120 if dir == "cdc" else 60) + @cluster(num_nodes=2) + @ignite_versions(str(DEV_BRANCH)) + def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): + """ + CDC with no consumer and no cdcWalDirectoryMaxSize keeps every archived segment. + """ + ignite = self.__ignite(ignite_version) + archive = os.path.join(ignite.persistent_root, "wal_archive") + data_storage = ignite.config.data_storage - check_failure(ignite, *failures[dir]) - finally: - free(ignite, path) + # CDC hard-links archived segments, so its directory must be on the archive's volume. + ignite.config = ignite.config._replace(data_storage=data_storage._replace( + cdc_wal_path=os.path.join(archive, "cdc"), default=data_storage.default._replace(cdc_enabled=True))) + ignite.mount_tmpfs(archive, 96) - load.stop() + self.__start(ignite) - @staticmethod - def __server_config(ignite_version): + self.__check_stops(ignite, timeout_sec=120) + check_failure(ignite, *FAILURES["wal_archive"]) + + def __ignite(self, ignite_version): """ - :return: Config of a persistent server node with small WAL segments, frequent checkpoints and a thin-client - connector. + :return: Persistent server node with the partitions, the WAL and its archive in directories of their own. """ - return IgniteConfiguration( + ignite = IgniteService(self.test_context, IgniteConfiguration( version=IgniteVersion(ignite_version), client_connector_configuration=ClientConnectorConfiguration(), - metrics_log_frequency=5000, + metrics_log_frequency=1000, data_storage=DataStorageConfiguration( checkpoint_frequency=1000, wal_segment_size=4 * MB, - max_wal_archive_size=64 * MB, - default=DataRegionConfiguration(persistence_enabled=True, max_size=256 * MB))) + max_wal_archive_size=32 * MB, + default=DataRegionConfiguration(persistence_enabled=True, max_size=256 * MB))), num_nodes=1) + + root = ignite.persistent_root + + ignite.config = ignite.config._replace(data_storage=ignite.config.data_storage._replace( + storage_path=os.path.join(root, "partitions"), + wal_path=os.path.join(root, "wal"), + wal_archive_path=os.path.join(root, "wal_archive"))) + + return ignite def __start(self, ignite): """ - Starts the node, activates the cluster and starts the load. - :return: Load application service, already putting data and recording its progress. + Starts and activates the node, then a load that records its puts and errors once a second. + :return: Load application service. """ ignite.start() @@ -267,20 +220,20 @@ def __start(self, ignite): return load @staticmethod - def __check_serves(ignite, load, duration_sec=15): + def __check_serves_load(ignite, load, seconds): """ - Checks that the node stays up and the load's puts go on without errors. + Waits the given seconds and checks that the node is alive and the puts went on without errors. """ puts, errors = progress(load) - time.sleep(duration_sec) + time.sleep(seconds) assert ignite.alive(ignite.nodes[0]), "The node stopped" puts_after, errors_after = progress(load) - assert puts_after > puts, f"No puts in {duration_sec} s" - assert errors_after == errors, f"{errors_after - errors} puts failed in {duration_sec} s" + assert puts_after > puts, f"No puts in {seconds} s" + assert errors_after == errors, f"{errors_after - errors} puts failed in {seconds} s" @staticmethod def __check_stops(ignite, timeout_sec=60): @@ -291,30 +244,45 @@ def __check_stops(ignite, timeout_sec=60): err_msg=f"The node is still up {timeout_sec} s after its volume filled up") -def fill(ignite, path): +def fill_volume(ignite, path): """ - Writes a file into the directory until its volume is full, and checks that less than 1 MB stays free. + Writes a file into the path until its volume has no space left. """ - node = ignite.nodes[0] + ignite.nodes[0].account.ssh(f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M", allow_fail=True) - node.account.ssh(f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M", allow_fail=True) - free_bytes = int(ignite.exec_command(node, f"df --output=avail -B1 {path} | tail -1").strip()) +def free_volume(ignite, path): + """ + Removes the file fill_volume wrote. + """ + ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=False) - assert free_bytes < MB, f"{free_bytes} bytes are free on the volume {path} after the fill" +def free_space(ignite, path): + """ + :return: Bytes free on the volume of the path. + """ + return int(ignite.exec_command(ignite.nodes[0], f"df --output=avail -B1 {path} | tail -1")) -def free(ignite, path): + +def check_log_grows(ignite, seconds): """ - Removes the file fill() wrote. + Checks that the node writes more than a page into its log within the given seconds. """ - ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=True) + node = ignite.nodes[0] + + def log_size(): + return int(ignite.exec_command(node, f"stat -c %s {node.log_file}")) + + start_size = log_size() + + wait_until(lambda: log_size() - start_size > PAGE, timeout_sec=seconds, + err_msg=f"The node wrote less than {PAGE} bytes into its log in {seconds} s") def check_failure(ignite, failure_type, cause): """ - Checks that the node's log records the failure handler's FailureContext of the given type, with the cause and - "No space left on device" in the lines that follow it. + Checks that the node logged a FailureContext of the type, caused by the writer and "No space left on device". """ node = ignite.nodes[0] From 110fb11115e31e119c55e58b8f91a5a8c3271af1 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 15 Oct 2026 15:58:25 +0300 Subject: [PATCH 06/20] IGNITE-29165 Mount only over own tmpfs --- .../tests/ignitetest/services/utils/ignite_aware.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index b75f27b6e62c0..822d422fad9a5 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -208,9 +208,9 @@ def __mount_tmpfs_volumes(self, node): Mounts the volumes registered by mount_tmpfs, unless they are still mounted from the node's previous start. """ for path, size_mb in self.tmpfs_volumes.items(): - node.account.ssh(f"mountpoint -q {path} || (mkdir -p {path} && sudo mount -t tmpfs " - f"-o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} {path})", - allow_fail=False) + node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -M {path} > /dev/null || (mkdir -p {path} && " + f"sudo mount -t tmpfs -o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} " + f"{path})", allow_fail=False) def __unmount_tmpfs_volumes(self, node): """ From 5fe31b19c4c7763a0d7cbb4574d2b2914c45d290 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 15 Oct 2026 16:00:30 +0300 Subject: [PATCH 07/20] IGNITE-29165 Simplify disk-full load parameters --- .../disk_full/DiskFullLoadApplication.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java index a99d825062ad6..875b17ca12231 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java @@ -22,12 +22,16 @@ import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; /** - * Thin-client load for the disk-full tests. Puts values of {@code valueSize} bytes under increasing keys, so the - * partitions and the WAL keep growing, and every {@code binaryTypeEvery}-th put stores a value of a new binary type, - * so the server keeps writing binary metadata. A failed put is counted and retried after a pause, so the load - * outlives a server node that stops. Once a second it records its counters as {@code progress-> <-}. + * Puts values under growing keys, with a new binary type every 1000 puts, so the partitions, the WAL and the binary + * metadata keep growing. Retries failed puts and records {@code progress-> <-} once a second. */ public class DiskFullLoadApplication extends IgniteAwareApplication { + /** Value size, in bytes. */ + private static final int VALUE_SIZE = 1024; + + /** Puts per new binary type. */ + private static final int BINARY_TYPE_EVERY = 1000; + /** Successful puts after which the application reports it is initialized. */ private static final int WARM_UP_PUTS = 1000; @@ -36,11 +40,9 @@ public class DiskFullLoadApplication extends IgniteAwareApplication { /** {@inheritDoc} */ @Override protected void run(JsonNode jsonNode) throws Exception { - String cacheName = jsonNode.get("cacheName").asText(); - int binaryTypeEvery = jsonNode.get("binaryTypeEvery").asInt(); - byte[] val = new byte[jsonNode.get("valueSize").asInt()]; + ClientCache cache = client.getOrCreateCache(jsonNode.get("cacheName").asText()); - ClientCache cache = client.getOrCreateCache(cacheName); + byte[] val = new byte[VALUE_SIZE]; long key = 0; long puts = 0; @@ -49,7 +51,7 @@ public class DiskFullLoadApplication extends IgniteAwareApplication { while (active()) { try { - if (key % binaryTypeEvery == 0) + if (key % BINARY_TYPE_EVERY == 0) cache.put(key, client.binary().builder("DiskFullType" + key).setField("val", val).build()); else cache.put(key, val); From 06942cdf72f48e1e5bef0844184695fac00ca008 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 15 Oct 2026 16:02:26 +0300 Subject: [PATCH 08/20] IGNITE-29165 Fix after fresh review --- .../tests/ignitetest/tests/disk_full_test.py | 106 +++++++++--------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index eab54e05099cd..937f8ed9aafdd 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -66,17 +66,18 @@ def test_node_survives_full_log_volume(self, ignite_version): load = self.__start(ignite) - check_log_grows(ignite, seconds=15) - fill_volume(ignite, ignite.log_dir) assert free_space(ignite, ignite.log_dir) == 0 - self.__check_serves_load(ignite, load, seconds=15) + check_serves_load_for(ignite, load, seconds=15) assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + check_log_write_failed(ignite) free_volume(ignite, ignite.log_dir) - check_log_grows(ignite, seconds=15) + size = log_size(ignite) + wait_until(lambda: log_size(ignite) > size, timeout_sec=15, + err_msg="The node's log did not grow after space was freed") load.stop() @@ -87,32 +88,34 @@ def test_jvm_exits_on_full_jfr_volume(self, ignite_version): JFR ends the JVM once it can't write its recording. """ ignite = self.__ignite(ignite_version) - jfr_opts = ["-XX:StartFlightRecording", f"-XX:FlightRecorderOptions=repository={ignite.jfr_dir}"] + repository = os.path.join(ignite.persistent_root, "jfr_repository") + jfr_opts = ["-XX:StartFlightRecording", f"-XX:FlightRecorderOptions=repository={repository}"] ignite.spec.jvm_opts = merge_jvm_settings(ignite.spec.jvm_opts, jfr_opts) - ignite.mount_tmpfs(ignite.jfr_dir, 32) + ignite.mount_tmpfs(repository, 32) self.__start(ignite) - fill_volume(ignite, ignite.jfr_dir) - assert free_space(ignite, ignite.jfr_dir) == 0 + fill_volume(ignite, repository) + assert free_space(ignite, repository) == 0 - self.__check_stops(ignite) + ignite.await_event("An irrecoverable error in Jfr", 60, from_the_beginning=True, log_file="console.log") + check_stops(ignite) @cluster(num_nodes=2) @ignite_versions(str(DEV_BRANCH)) def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version): """ - The default layout: logs in the work directory, next to the data. + One volume holds the logs and all the data, as in Ignite's default layout. """ ignite = self.__ignite(ignite_version) ignite.mount_tmpfs(ignite.persistent_root, 256) self.__start(ignite) - fill_volume(ignite, ignite.log_dir) - assert free_space(ignite, ignite.log_dir) == 0 + fill_volume(ignite, ignite.persistent_root) + assert free_space(ignite, ignite.persistent_root) == 0 - self.__check_stops(ignite) + check_stops(ignite) @cluster(num_nodes=2) @ignite_versions(str(DEV_BRANCH)) @@ -126,13 +129,13 @@ def test_node_survives_full_wal_volume(self, ignite_version): load = self.__start(ignite) - segments = ignite.exec_command(ignite.nodes[0], f"ls {wal}/*/*.wal | wc -l") + segments = ignite.exec_command(ignite.nodes[0], f"ls {wal}/*/*.wal 2>/dev/null | wc -l") assert int(segments) > 0, "No WAL segments on the volume" fill_volume(ignite, wal) assert free_space(ignite, wal) == 0 - self.__check_serves_load(ignite, load, seconds=15) + check_serves_load_for(ignite, load, seconds=15) load.stop() @@ -141,7 +144,7 @@ def test_node_survives_full_wal_volume(self, ignite_version): @matrix(volume=["wal_archive", "partitions", "work"]) def test_node_stops_on_full_volume(self, ignite_version, volume): """ - The work directory holds the binary metadata. + The node stops once it can't write the WAL archive, the partitions or the binary metadata (in work). """ ignite = self.__ignite(ignite_version) path = os.path.join(ignite.persistent_root, volume) @@ -152,7 +155,7 @@ def test_node_stops_on_full_volume(self, ignite_version, volume): fill_volume(ignite, path) assert free_space(ignite, path) == 0 - self.__check_stops(ignite) + check_stops(ignite) check_failure(ignite, *FAILURES[volume]) @cluster(num_nodes=2) @@ -168,11 +171,11 @@ def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): # CDC hard-links archived segments, so its directory must be on the archive's volume. ignite.config = ignite.config._replace(data_storage=data_storage._replace( cdc_wal_path=os.path.join(archive, "cdc"), default=data_storage.default._replace(cdc_enabled=True))) - ignite.mount_tmpfs(archive, 96) + ignite.mount_tmpfs(archive, 256) self.__start(ignite) - self.__check_stops(ignite, timeout_sec=120) + check_stops(ignite, timeout_sec=120) check_failure(ignite, *FAILURES["wal_archive"]) def __ignite(self, ignite_version): @@ -213,36 +216,12 @@ def __start(self, ignite): self.test_context, IgniteThinClientConfiguration(addresses=[address], version=ignite.config.version), java_class_name=self.LOAD_APP, - params={"cacheName": "disk-full", "valueSize": 1024, "binaryTypeEvery": 1000}) + params={"cacheName": "disk-full"}) load.start() return load - @staticmethod - def __check_serves_load(ignite, load, seconds): - """ - Waits the given seconds and checks that the node is alive and the puts went on without errors. - """ - puts, errors = progress(load) - - time.sleep(seconds) - - assert ignite.alive(ignite.nodes[0]), "The node stopped" - - puts_after, errors_after = progress(load) - - assert puts_after > puts, f"No puts in {seconds} s" - assert errors_after == errors, f"{errors_after - errors} puts failed in {seconds} s" - - @staticmethod - def __check_stops(ignite, timeout_sec=60): - """ - Checks that the node's process ends. - """ - wait_until(lambda: not ignite.alive(ignite.nodes[0]), timeout_sec=timeout_sec, - err_msg=f"The node is still up {timeout_sec} s after its volume filled up") - def fill_volume(ignite, path): """ @@ -265,19 +244,44 @@ def free_space(ignite, path): return int(ignite.exec_command(ignite.nodes[0], f"df --output=avail -B1 {path} | tail -1")) -def check_log_grows(ignite, seconds): +def log_size(ignite): """ - Checks that the node writes more than a page into its log within the given seconds. + :return: Bytes written into the node's log. """ node = ignite.nodes[0] - def log_size(): - return int(ignite.exec_command(node, f"stat -c %s {node.log_file}")) + return int(ignite.exec_command(node, f"stat -c %s {node.log_file}")) - start_size = log_size() - wait_until(lambda: log_size() - start_size > PAGE, timeout_sec=seconds, - err_msg=f"The node wrote less than {PAGE} bytes into its log in {seconds} s") +def check_log_write_failed(ignite): + """ + Checks that the node's log ends on a page boundary: a write the full tmpfs refused filled its last page. + """ + assert log_size(ignite) % PAGE == 0, "No write into the node's log hit the full volume" + + +def check_serves_load_for(ignite, load, seconds): + """ + Waits the given seconds and checks that the node is alive and the puts went on without errors. + """ + puts, errors = progress(load) + + time.sleep(seconds) + + assert ignite.alive(ignite.nodes[0]), "The node stopped" + + puts_after, errors_after = progress(load) + + assert puts_after > puts, f"No puts in {seconds} s" + assert errors_after == errors, f"{errors_after - errors} puts failed in {seconds} s" + + +def check_stops(ignite, timeout_sec=60): + """ + Checks that the node's process ends. + """ + wait_until(lambda: not ignite.alive(ignite.nodes[0]), timeout_sec=timeout_sec, + err_msg=f"The node is still up {timeout_sec} s after its volume filled up") def check_failure(ignite, failure_type, cause): From 05d4aa2e0a5e3daf547a906f93b0c55df88a94ef Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 12:30:13 +0300 Subject: [PATCH 09/20] IGNITE-29165 Minor fixes --- .../ignitetest/services/utils/ignite_aware.py | 4 +-- .../tests/ignitetest/tests/disk_full_test.py | 26 ++++++++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 822d422fad9a5..75c04159cb6d4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -209,14 +209,14 @@ def __mount_tmpfs_volumes(self, node): """ for path, size_mb in self.tmpfs_volumes.items(): node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -M {path} > /dev/null || (mkdir -p {path} && " - f"sudo mount -t tmpfs -o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} " + f"mount -t tmpfs -o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} " f"{path})", allow_fail=False) def __unmount_tmpfs_volumes(self, node): """ Unmounts every volume mount_tmpfs made on the node, also one left by an aborted run. """ - node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET | xargs -r -n1 sudo umount -l", + node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET | xargs -r -n1 umount -l", allow_fail=False) def init_persistent(self, node): diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index 937f8ed9aafdd..96575e75595bc 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -41,6 +41,10 @@ FILLER = "filler" +# A filled volume can open up again before it is measured: the node deletes files on it, archived WAL +# segments above the archive limit being the case that happens under load. Refill that many times. +FILL_ATTEMPTS = 5 + # Failure type and failed writer the node logs when the volume fills. FAILURES = { "wal_archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), @@ -67,7 +71,6 @@ def test_node_survives_full_log_volume(self, ignite_version): load = self.__start(ignite) fill_volume(ignite, ignite.log_dir) - assert free_space(ignite, ignite.log_dir) == 0 check_serves_load_for(ignite, load, seconds=15) assert ControlUtility(ignite).cluster_state().state == "ACTIVE" @@ -96,7 +99,6 @@ def test_jvm_exits_on_full_jfr_volume(self, ignite_version): self.__start(ignite) fill_volume(ignite, repository) - assert free_space(ignite, repository) == 0 ignite.await_event("An irrecoverable error in Jfr", 60, from_the_beginning=True, log_file="console.log") check_stops(ignite) @@ -113,7 +115,6 @@ def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version) self.__start(ignite) fill_volume(ignite, ignite.persistent_root) - assert free_space(ignite, ignite.persistent_root) == 0 check_stops(ignite) @@ -133,7 +134,6 @@ def test_node_survives_full_wal_volume(self, ignite_version): assert int(segments) > 0, "No WAL segments on the volume" fill_volume(ignite, wal) - assert free_space(ignite, wal) == 0 check_serves_load_for(ignite, load, seconds=15) @@ -153,7 +153,6 @@ def test_node_stops_on_full_volume(self, ignite_version, volume): self.__start(ignite) fill_volume(ignite, path) - assert free_space(ignite, path) == 0 check_stops(ignite) check_failure(ignite, *FAILURES[volume]) @@ -225,9 +224,22 @@ def __start(self, ignite): def fill_volume(ignite, path): """ - Writes a file into the path until its volume has no space left. + Writes a file into the path until its volume has no space left, and fails if it never got full. """ - ignite.nodes[0].account.ssh(f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M", allow_fail=True) + free = -1 + + for _ in range(FILL_ATTEMPTS): + # dd exits with 1 once the volume refuses the write, which is the point here, so its exit + # code is not the evidence that the volume is full. And the node frees space on it as it + # deletes files, so a volume that was full a moment ago may not be when it is measured. + ignite.nodes[0].account.ssh(f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M", allow_fail=True) + + free = free_space(ignite, path) + + if free == 0: + break + + assert free == 0, f"{path} still has {free} bytes free after {FILL_ATTEMPTS} fill attempts" def free_volume(ignite, path): From 8d849b109a2341aae80541e0ffce8dd5b64c8cf8 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 13:33:31 +0300 Subject: [PATCH 10/20] IGNITE-29165 Restore sudo for tmpfs mounts --- .../ducktests/tests/ignitetest/services/utils/ignite_aware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 75c04159cb6d4..822d422fad9a5 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -209,14 +209,14 @@ def __mount_tmpfs_volumes(self, node): """ for path, size_mb in self.tmpfs_volumes.items(): node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -M {path} > /dev/null || (mkdir -p {path} && " - f"mount -t tmpfs -o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} " + f"sudo mount -t tmpfs -o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} " f"{path})", allow_fail=False) def __unmount_tmpfs_volumes(self, node): """ Unmounts every volume mount_tmpfs made on the node, also one left by an aborted run. """ - node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET | xargs -r -n1 umount -l", + node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET | xargs -r -n1 sudo umount -l", allow_fail=False) def init_persistent(self, node): From a251cae0e7b19310e18699b26d6470b07086e4ba Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 14:59:11 +0300 Subject: [PATCH 11/20] IGNITE-29165 Take dd's ENOSPC as proof of a full volume --- .../tests/ignitetest/tests/disk_full_test.py | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index 96575e75595bc..1ced3c335f1bd 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -41,10 +41,6 @@ FILLER = "filler" -# A filled volume can open up again before it is measured: the node deletes files on it, archived WAL -# segments above the archive limit being the case that happens under load. Refill that many times. -FILL_ATTEMPTS = 5 - # Failure type and failed writer the node logs when the volume fills. FAILURES = { "wal_archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), @@ -224,22 +220,11 @@ def __start(self, ignite): def fill_volume(ignite, path): """ - Writes a file into the path until its volume has no space left, and fails if it never got full. + Writes a file into the path until its volume refuses the write. """ - free = -1 - - for _ in range(FILL_ATTEMPTS): - # dd exits with 1 once the volume refuses the write, which is the point here, so its exit - # code is not the evidence that the volume is full. And the node frees space on it as it - # deletes files, so a volume that was full a moment ago may not be when it is measured. - ignite.nodes[0].account.ssh(f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M", allow_fail=True) - - free = free_space(ignite, path) - - if free == 0: - break + out = ignite.exec_command(ignite.nodes[0], f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M 2>&1") - assert free == 0, f"{path} still has {free} bytes free after {FILL_ATTEMPTS} fill attempts" + assert "No space left on device" in out, f"{path} did not fill up: {out}" def free_volume(ignite, path): @@ -249,13 +234,6 @@ def free_volume(ignite, path): ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=False) -def free_space(ignite, path): - """ - :return: Bytes free on the volume of the path. - """ - return int(ignite.exec_command(ignite.nodes[0], f"df --output=avail -B1 {path} | tail -1")) - - def log_size(ignite): """ :return: Bytes written into the node's log. From c72aa404b3c1264e24251bb8bcde6115af6d8627 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 15:47:18 +0300 Subject: [PATCH 12/20] IGNITE-29165 Add node spec writing its output into a file --- .../tests/checks/utils/check_ignite_spec.py | 20 ++++++++++++++-- .../ignitetest/services/utils/ignite_spec.py | 24 +++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/modules/ducktests/tests/checks/utils/check_ignite_spec.py b/modules/ducktests/tests/checks/utils/check_ignite_spec.py index 1e2cc4d3920a1..41ff5909202a7 100644 --- a/modules/ducktests/tests/checks/utils/check_ignite_spec.py +++ b/modules/ducktests/tests/checks/utils/check_ignite_spec.py @@ -17,13 +17,14 @@ Checks Spec class that describes config and command line to start Ignite-aware service. """ import os -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest from ignitetest.services.utils import IgniteServiceType from ignitetest.services.utils.gc_params import CLIENT_ROLE, SERVER_ROLE -from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec, IgniteNodeSpec, service_role +from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec, IgniteNodeSpec, IgniteNodeSpecFileOutput, \ + service_role from ignitetest.services.utils.jvm_utils import GC_PROFILES, GC_G1, MultipleGcSelectedError from ignitetest.utils.ignite_test import JFR_ENABLED @@ -287,3 +288,18 @@ def check_heap_override__from_jvm_opts__still_wins(service): assert "-Xms4G" in spec.jvm_opts assert "-Xmx1G" not in spec.jvm_opts assert "-Xms1G" not in spec.jvm_opts + + +""" +Checks the shell redirection of a node's output. +""" + + +def check_node_spec_file_output__appends_stdout_and_stderr_to_console_log(service): + service.log_dir = "/logs" + + with patch.object(IgniteNodeSpecFileOutput, "envs", return_value={}): + cmd = IgniteNodeSpecFileOutput(service).command(None) + + assert cmd.endswith(f" >> {os.path.join(service.log_dir, 'console.log')} 2>&1 &") + assert "tee" not in cmd diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index ae7036df48c2e..891c56619fce1 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -373,6 +373,12 @@ def _jvm_opts(self): opts = ["-J%s" % o for o in self.jvm_opts] return " ".join(opts) + def _output_redirect(self): + """ + :return: shell redirection of the service's stdout and stderr: through tee, appended to console.log. + """ + return "2>&1 | tee -a %s" % os.path.join(self.service.log_dir, "console.log") + def _runcmd(self, cmd): self.service.logger.debug(cmd) proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -388,16 +394,26 @@ class IgniteNodeSpec(IgniteSpec): """ def command(self, node): - cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ + cmd = "%s %s %s %s %s &" % \ (envs_to_exports(self.envs()), self.service.script("ignite.sh"), self._jvm_opts(), self.config_file_path(), - os.path.join(self.service.log_dir, "console.log")) + self._output_redirect()) return cmd +class IgniteNodeSpecFileOutput(IgniteNodeSpec): + """ + Spec to run ignite node with stdout and stderr appended to console.log by the shell, as nohup does, so the JVM + writes them itself instead of a pipe to tee. + """ + + def _output_redirect(self): + return ">> %s 2>&1" % os.path.join(self.service.log_dir, "console.log") + + class IgniteApplicationSpec(IgniteSpec): """ Spec to run ignite application @@ -422,12 +438,12 @@ def command(self, node): str(base64.b64encode(json.dumps(self.service.params).encode('utf-8')), 'utf-8') ] - cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ + cmd = "%s %s %s %s %s &" % \ (envs_to_exports(self.envs()), self.service.script("ignite.sh"), self._jvm_opts(), ",".join(args), - os.path.join(self.service.log_dir, "console.log")) + self._output_redirect()) return cmd From 18ba8b2eb3d20c637117182a5f045dd0cf93e9a5 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 15:49:08 +0300 Subject: [PATCH 13/20] IGNITE-29165 Run disk-full tests on demand, output into a file --- .../tests/ignitetest/tests/disk_full_test.py | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index 1ced3c335f1bd..a0d98f62bab37 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -15,6 +15,9 @@ """ Fills one volume of a persistent server node under load and checks whether the node survives. + +The tests reproduce what Ignite does today and guard no behavior, so they run only with +--globals '{"disk_full_tests": true}'. """ import os import time @@ -28,9 +31,10 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteThinClientConfiguration, \ DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_spec import IgniteNodeSpecFileOutput from ignitetest.services.utils.jvm_utils import merge_jvm_settings from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration -from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils import cluster, ignite_versions, ignore_if from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion @@ -48,6 +52,8 @@ "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore") } +reproducer = ignore_if(lambda _, globals: not globals.get("disk_full_tests")) + class DiskFullTest(IgniteTest): """ @@ -56,6 +62,7 @@ class DiskFullTest(IgniteTest): LOAD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication" @cluster(num_nodes=2) + @reproducer @ignite_versions(str(DEV_BRANCH)) def test_node_survives_full_log_volume(self, ignite_version): """ @@ -70,17 +77,21 @@ def test_node_survives_full_log_volume(self, ignite_version): check_serves_load_for(ignite, load, seconds=15) assert ControlUtility(ignite).cluster_state().state == "ACTIVE" - check_log_write_failed(ignite) + + logs = [ignite.nodes[0].log_file, os.path.join(ignite.log_dir, "console.log")] + + for log in logs: + check_write_failed(ignite, log) free_volume(ignite, ignite.log_dir) - size = log_size(ignite) - wait_until(lambda: log_size(ignite) > size, timeout_sec=15, - err_msg="The node's log did not grow after space was freed") + for log in logs: + check_grows(ignite, log) load.stop() @cluster(num_nodes=2) + @reproducer @ignite_versions(str(DEV_BRANCH)) def test_jvm_exits_on_full_jfr_volume(self, ignite_version): """ @@ -100,6 +111,7 @@ def test_jvm_exits_on_full_jfr_volume(self, ignite_version): check_stops(ignite) @cluster(num_nodes=2) + @reproducer @ignite_versions(str(DEV_BRANCH)) def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version): """ @@ -115,6 +127,7 @@ def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version) check_stops(ignite) @cluster(num_nodes=2) + @reproducer @ignite_versions(str(DEV_BRANCH)) def test_node_survives_full_wal_volume(self, ignite_version): """ @@ -136,6 +149,7 @@ def test_node_survives_full_wal_volume(self, ignite_version): load.stop() @cluster(num_nodes=2) + @reproducer @ignite_versions(str(DEV_BRANCH)) @matrix(volume=["wal_archive", "partitions", "work"]) def test_node_stops_on_full_volume(self, ignite_version, volume): @@ -154,6 +168,7 @@ def test_node_stops_on_full_volume(self, ignite_version, volume): check_failure(ignite, *FAILURES[volume]) @cluster(num_nodes=2) + @reproducer @ignite_versions(str(DEV_BRANCH)) def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): """ @@ -175,7 +190,8 @@ def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): def __ignite(self, ignite_version): """ - :return: Persistent server node with the partitions, the WAL and its archive in directories of their own. + :return: Persistent server node that writes its stdout and stderr into console.log itself, with the + partitions, the WAL and its archive in directories of their own. """ ignite = IgniteService(self.test_context, IgniteConfiguration( version=IgniteVersion(ignite_version), @@ -187,6 +203,8 @@ def __ignite(self, ignite_version): max_wal_archive_size=32 * MB, default=DataRegionConfiguration(persistence_enabled=True, max_size=256 * MB))), num_nodes=1) + ignite.spec = ignite.spec.rebuild_as(IgniteNodeSpecFileOutput) + root = ignite.persistent_root ignite.config = ignite.config._replace(data_storage=ignite.config.data_storage._replace( @@ -234,20 +252,27 @@ def free_volume(ignite, path): ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=False) -def log_size(ignite): +def file_size(ignite, path): """ - :return: Bytes written into the node's log. + :return: Bytes written into the file on the node. """ - node = ignite.nodes[0] + return int(ignite.exec_command(ignite.nodes[0], f"stat -c %s {path}")) - return int(ignite.exec_command(node, f"stat -c %s {node.log_file}")) +def check_write_failed(ignite, path): + """ + Checks that the file ends on a page boundary: a write the full tmpfs refused filled its last page. + """ + assert file_size(ignite, path) % PAGE == 0, f"No write into {path} hit the full volume" -def check_log_write_failed(ignite): + +def check_grows(ignite, path): """ - Checks that the node's log ends on a page boundary: a write the full tmpfs refused filled its last page. + Checks that the file grows within 15 s. """ - assert log_size(ignite) % PAGE == 0, "No write into the node's log hit the full volume" + size = file_size(ignite, path) + + wait_until(lambda: file_size(ignite, path) > size, timeout_sec=15, err_msg=f"{path} did not grow") def check_serves_load_for(ignite, load, seconds): From f79f80f0218d1194777f29cabc886fc8cde5c697 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 15:56:08 +0300 Subject: [PATCH 14/20] IGNITE-29165 Add reproducer mark for on-demand tests --- .../tests/checks/utils/check_parametrized.py | 30 +++++++++++++++- .../tests/ignitetest/tests/disk_full_test.py | 16 ++++----- .../tests/ignitetest/utils/__init__.py | 4 +-- .../ducktests/tests/ignitetest/utils/_mark.py | 34 +++++++++++++++++++ 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/modules/ducktests/tests/checks/utils/check_parametrized.py b/modules/ducktests/tests/checks/utils/check_parametrized.py index 7c814fec45780..3bc84d161fd53 100644 --- a/modules/ducktests/tests/checks/utils/check_parametrized.py +++ b/modules/ducktests/tests/checks/utils/check_parametrized.py @@ -24,7 +24,7 @@ from ducktape.mark import parametrized, parametrize, matrix, ignore from ducktape.mark.mark_expander import MarkedFunctionExpander -from ignitetest.utils import ignite_versions, ignore_if +from ignitetest.utils import ignite_versions, ignore_if, reproducer from ignitetest.utils._mark import IgniteVersionParametrize from ignitetest.utils.version import IgniteVersion, V_2_8_0, V_2_8_1, V_2_7_6, DEV_BRANCH @@ -267,3 +267,31 @@ def function_3(ignite_version): assert next(filter(lambda x: x.injected_args['ver'] == str(V_2_8_0), context_list)).ignore assert not next(filter(lambda x: x.injected_args['ver'] == str(DEV_BRANCH), context_list)).ignore + + +class CheckReproducer: + """ + Checks @reproducer. + """ + @pytest.mark.parametrize(['global_args', 'ignored'], [({}, True), ({'my_tests': False}, True), + ({'my_tests': True}, False)]) + def check_ignored_unless_global_is_set(self, global_args, ignored): + """ + Check that a test is ignored unless its global is set, with and without @ignite_versions. + """ + @reproducer('my_tests') + @ignite_versions(str(DEV_BRANCH), str(V_2_8_0)) + def versioned(ignite_version): + return IgniteVersion(ignite_version) + + @reproducer('my_tests') + def plain(): + pass + + sess_ctx = mock_session_ctx(global_args=global_args) + + context_list = expand_function(func=versioned, sess_ctx=sess_ctx) + context_list += MarkedFunctionExpander(session_context=sess_ctx, function=plain).expand() + + assert len(context_list) == 3 + assert all(ctx.ignore == ignored for ctx in context_list) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index a0d98f62bab37..a22e89454337b 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -34,7 +34,7 @@ from ignitetest.services.utils.ignite_spec import IgniteNodeSpecFileOutput from ignitetest.services.utils.jvm_utils import merge_jvm_settings from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration -from ignitetest.utils import cluster, ignite_versions, ignore_if +from ignitetest.utils import cluster, ignite_versions, reproducer from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion @@ -52,8 +52,6 @@ "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore") } -reproducer = ignore_if(lambda _, globals: not globals.get("disk_full_tests")) - class DiskFullTest(IgniteTest): """ @@ -62,7 +60,7 @@ class DiskFullTest(IgniteTest): LOAD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication" @cluster(num_nodes=2) - @reproducer + @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) def test_node_survives_full_log_volume(self, ignite_version): """ @@ -91,7 +89,7 @@ def test_node_survives_full_log_volume(self, ignite_version): load.stop() @cluster(num_nodes=2) - @reproducer + @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) def test_jvm_exits_on_full_jfr_volume(self, ignite_version): """ @@ -111,7 +109,7 @@ def test_jvm_exits_on_full_jfr_volume(self, ignite_version): check_stops(ignite) @cluster(num_nodes=2) - @reproducer + @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version): """ @@ -127,7 +125,7 @@ def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version) check_stops(ignite) @cluster(num_nodes=2) - @reproducer + @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) def test_node_survives_full_wal_volume(self, ignite_version): """ @@ -149,7 +147,7 @@ def test_node_survives_full_wal_volume(self, ignite_version): load.stop() @cluster(num_nodes=2) - @reproducer + @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) @matrix(volume=["wal_archive", "partitions", "work"]) def test_node_stops_on_full_volume(self, ignite_version, volume): @@ -168,7 +166,7 @@ def test_node_stops_on_full_volume(self, ignite_version, volume): check_failure(ignite, *FAILURES[volume]) @cluster(num_nodes=2) - @reproducer + @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): """ diff --git a/modules/ducktests/tests/ignitetest/utils/__init__.py b/modules/ducktests/tests/ignitetest/utils/__init__.py index 6d6da0d14ed12..2cc208c44baf0 100644 --- a/modules/ducktests/tests/ignitetest/utils/__init__.py +++ b/modules/ducktests/tests/ignitetest/utils/__init__.py @@ -17,6 +17,6 @@ This module contains convenient utils for test. """ -from ._mark import ignore_if, ignite_versions, cluster +from ._mark import ignore_if, ignite_versions, cluster, reproducer -__all__ = ['ignore_if', 'ignite_versions', 'cluster'] +__all__ = ['ignore_if', 'ignite_versions', 'cluster', 'reproducer'] diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py index 9d0a06df4edc2..5964312fcc3c2 100644 --- a/modules/ducktests/tests/ignitetest/utils/_mark.py +++ b/modules/ducktests/tests/ignitetest/utils/_mark.py @@ -51,6 +51,26 @@ def __eq__(self, other): return super().__eq__(other) and self.condition == other.condition +class Reproducer(Ignore): + """ + Ignore test unless the global that enables it is set. + """ + def __init__(self, global_name): + super().__init__() + self.global_name = global_name + + def apply(self, seed_context, context_list): + assert len(context_list) > 0, "reproducer decorator is not being applied to any test cases" + + for ctx in context_list: + ctx.ignore = ctx.ignore or not ctx.globals.get(self.global_name) + + return context_list + + def __eq__(self, other): + return super().__eq__(other) and self.global_name == other.global_name + + class IgniteVersionParametrize(Mark): """ Parametrize function with ignite_version @@ -219,6 +239,20 @@ def ignorer(func): return ignorer +def reproducer(global_name): + """ + Mark decorated test method as IGNORE unless the global is set, e.g. --globals '{"": true}'. + For tests that reproduce a behavior on demand rather than guard it on every run. + + :param global_name: name of the global that enables the test. + """ + def ignorer(func): + Mark.mark(func, Reproducer(global_name)) + return func + + return ignorer + + def cluster(**kwargs): """Test method decorator used to provide hints about how the test will use the given cluster. From 3c26f9edf2018461cffa6e65bed30bfc1d423d08 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 19 Oct 2026 16:17:04 +0300 Subject: [PATCH 15/20] IGNITE-29165 Add System.out flood reproducer --- .../disk_full/StdoutFloodApplication.java | 49 +++++++++++++++ .../tests/disk_full/StdoutFloodService.java | 26 ++++++++ .../disk_full/StdoutFloodServiceImpl.java | 62 +++++++++++++++++++ .../tests/ignitetest/tests/disk_full_test.py | 50 +++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java new file mode 100644 index 0000000000000..ee0fa6aff881a --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Deploys {@link StdoutFloodServiceImpl} on every server node and records {@code lines-><-}, the lines the + * service wrote, once a second. + */ +public class StdoutFloodApplication extends IgniteAwareApplication { + /** Service name. */ + private static final String SVC_NAME = "stdout-flood"; + + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + ignite.services(ignite.cluster().forServers()).deployNodeSingleton(SVC_NAME, new StdoutFloodServiceImpl()); + + StdoutFloodService svc = ignite.services().serviceProxy(SVC_NAME, StdoutFloodService.class, false); + + recordResult("lines", svc.lines()); + + markInitialized(); + + while (active()) { + Thread.sleep(1000); + + recordResult("lines", svc.lines()); + } + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java new file mode 100644 index 0000000000000..e3b277cbd9a63 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +/** + * Writes lines to {@code System.out} without end, as user code left with debug output does. + */ +public interface StdoutFloodService { + /** @return Lines written so far. */ + public long lines(); +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java new file mode 100644 index 0000000000000..3904e8a840984 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.resources.ServiceContextResource; +import org.apache.ignite.services.Service; +import org.apache.ignite.services.ServiceContext; + +/** + * Writes 1 KB lines to {@code System.out} at about 10 MB/s until cancelled. + */ +public class StdoutFloodServiceImpl implements Service, StdoutFloodService { + /** Serial version UID. */ + private static final long serialVersionUID = 0L; + + /** Line, 1 KB with its line separator. */ + private static final String LINE = "x".repeat(1023); + + /** Lines written per millisecond. */ + private static final int LINES_PER_MS = 10; + + /** Service context. */ + @ServiceContextResource + private ServiceContext ctx; + + /** Lines written. */ + private final AtomicLong lines = new AtomicLong(); + + /** {@inheritDoc} */ + @Override public void execute() throws Exception { + while (!ctx.isCancelled()) { + for (int i = 0; i < LINES_PER_MS; i++) { + System.out.println(LINE); + + lines.incrementAndGet(); + } + + Thread.sleep(1); + } + } + + /** {@inheritDoc} */ + @Override public long lines() { + return lines.get(); + } +} diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index a22e89454337b..6867e41932fae 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -31,6 +31,7 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteThinClientConfiguration, \ DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.services.utils.ignite_spec import IgniteNodeSpecFileOutput from ignitetest.services.utils.jvm_utils import merge_jvm_settings from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration @@ -59,6 +60,8 @@ class DiskFullTest(IgniteTest): """ LOAD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication" + FLOOD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.StdoutFloodApplication" + @cluster(num_nodes=2) @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) @@ -88,6 +91,39 @@ def test_node_survives_full_log_volume(self, ignite_version): load.stop() + @cluster(num_nodes=3) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_survives_log_volume_filled_by_stdout(self, ignite_version): + """ + A service on the node writes to System.out until its output fills the log volume. + """ + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.log_dir, 32) + + load = self.__start(ignite) + + flood = IgniteApplicationService( + self.test_context, + IgniteConfiguration(version=ignite.config.version, client_mode=True, + discovery_spi=from_ignite_cluster(ignite)), + java_class_name=self.FLOOD_APP) + + flood.start() + + wait_until(lambda: free_space(ignite, ignite.log_dir) == 0, timeout_sec=60, + err_msg="System.out did not fill the log volume") + + lines = printed_lines(flood) + + check_serves_load_for(ignite, load, seconds=15) + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + assert printed_lines(flood) > lines, "The service stopped writing to System.out" + check_write_failed(ignite, os.path.join(ignite.log_dir, "console.log")) + + flood.stop() + load.stop() + @cluster(num_nodes=2) @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) @@ -250,6 +286,20 @@ def free_volume(ignite, path): ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=False) +def free_space(ignite, path): + """ + :return: Bytes free on the volume of the path. + """ + return int(ignite.exec_command(ignite.nodes[0], f"df --output=avail -B1 {path} | tail -1")) + + +def printed_lines(flood): + """ + :return: Lines the service wrote to System.out, as the flood application recorded last. + """ + return int(flood.extract_results("lines")[-1]) + + def file_size(ignite, path): """ :return: Bytes written into the file on the node. From 0e16ce4b53c712591086228e96d2dec9b2dc342d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 20 Oct 2026 13:41:01 +0300 Subject: [PATCH 16/20] IGNITE-29165 Cancel the flood service and check the log after a remount --- .../disk_full/StdoutFloodApplication.java | 5 +- .../disk_full/StdoutFloodServiceImpl.java | 2 +- .../tests/ignitetest/tests/disk_full_test.py | 83 ++++++++++++++----- 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java index ee0fa6aff881a..44c7d25048646 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java @@ -22,7 +22,8 @@ /** * Deploys {@link StdoutFloodServiceImpl} on every server node and records {@code lines-><-}, the lines the - * service wrote, once a second. + * service wrote, once a second. Cancels the service on stop, so the servers stop writing to their + * {@code System.out} once {@code markFinished} is logged. */ public class StdoutFloodApplication extends IgniteAwareApplication { /** Service name. */ @@ -44,6 +45,8 @@ public class StdoutFloodApplication extends IgniteAwareApplication { recordResult("lines", svc.lines()); } + ignite.services().cancelAsync(SVC_NAME).get(); + markFinished(); } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java index 3904e8a840984..95c0e4308a06a 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java @@ -32,7 +32,7 @@ public class StdoutFloodServiceImpl implements Service, StdoutFloodService { /** Line, 1 KB with its line separator. */ private static final String LINE = "x".repeat(1023); - /** Lines written per millisecond. */ + /** Lines written once a millisecond. */ private static final int LINES_PER_MS = 10; /** Service context. */ diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index 6867e41932fae..92c87de6e240c 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -72,7 +72,10 @@ def test_node_survives_full_log_volume(self, ignite_version): ignite = self.__ignite(ignite_version) ignite.mount_tmpfs(ignite.log_dir, 32) - load = self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) fill_volume(ignite, ignite.log_dir) @@ -101,15 +104,12 @@ def test_node_survives_log_volume_filled_by_stdout(self, ignite_version): ignite = self.__ignite(ignite_version) ignite.mount_tmpfs(ignite.log_dir, 32) - load = self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() - flood = IgniteApplicationService( - self.test_context, - IgniteConfiguration(version=ignite.config.version, client_mode=True, - discovery_spi=from_ignite_cluster(ignite)), - java_class_name=self.FLOOD_APP) + load = self.__start_load_client(ignite) - flood.start() + flood = self.__start_flood_service(ignite) wait_until(lambda: free_space(ignite, ignite.log_dir) == 0, timeout_sec=60, err_msg="System.out did not fill the log volume") @@ -119,9 +119,17 @@ def test_node_survives_log_volume_filled_by_stdout(self, ignite_version): check_serves_load_for(ignite, load, seconds=15) assert ControlUtility(ignite).cluster_state().state == "ACTIVE" assert printed_lines(flood) > lines, "The service stopped writing to System.out" - check_write_failed(ignite, os.path.join(ignite.log_dir, "console.log")) + + console_log = os.path.join(ignite.log_dir, "console.log") + + check_write_failed(ignite, console_log) flood.stop() + + enlarge_volume(ignite, ignite.log_dir, 128) + + check_grows(ignite, ignite.nodes[0].log_file) + load.stop() @cluster(num_nodes=2) @@ -137,13 +145,18 @@ def test_jvm_exits_on_full_jfr_volume(self, ignite_version): ignite.spec.jvm_opts = merge_jvm_settings(ignite.spec.jvm_opts, jfr_opts) ignite.mount_tmpfs(repository, 32) - self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) fill_volume(ignite, repository) ignite.await_event("An irrecoverable error in Jfr", 60, from_the_beginning=True, log_file="console.log") check_stops(ignite) + load.stop() + @cluster(num_nodes=2) @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) @@ -154,12 +167,17 @@ def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version) ignite = self.__ignite(ignite_version) ignite.mount_tmpfs(ignite.persistent_root, 256) - self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) fill_volume(ignite, ignite.persistent_root) check_stops(ignite) + load.stop() + @cluster(num_nodes=2) @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) @@ -171,7 +189,10 @@ def test_node_survives_full_wal_volume(self, ignite_version): wal = os.path.join(ignite.persistent_root, "wal") ignite.mount_tmpfs(wal, 64) - load = self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) segments = ignite.exec_command(ignite.nodes[0], f"ls {wal}/*/*.wal 2>/dev/null | wc -l") assert int(segments) > 0, "No WAL segments on the volume" @@ -194,13 +215,18 @@ def test_node_stops_on_full_volume(self, ignite_version, volume): path = os.path.join(ignite.persistent_root, volume) ignite.mount_tmpfs(path, 64) - self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) fill_volume(ignite, path) check_stops(ignite) check_failure(ignite, *FAILURES[volume]) + load.stop() + @cluster(num_nodes=2) @reproducer("disk_full_tests") @ignite_versions(str(DEV_BRANCH)) @@ -217,11 +243,16 @@ def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): cdc_wal_path=os.path.join(archive, "cdc"), default=data_storage.default._replace(cdc_enabled=True))) ignite.mount_tmpfs(archive, 256) - self.__start(ignite) + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) check_stops(ignite, timeout_sec=120) check_failure(ignite, *FAILURES["wal_archive"]) + load.stop() + def __ignite(self, ignite_version): """ :return: Persistent server node that writes its stdout and stderr into console.log itself, with the @@ -248,15 +279,11 @@ def __ignite(self, ignite_version): return ignite - def __start(self, ignite): + def __start_load_client(self, ignite): """ Starts and activates the node, then a load that records its puts and errors once a second. :return: Load application service. """ - ignite.start() - - ControlUtility(ignite).activate() - address = f"{ignite.nodes[0].account.hostname}:{ignite.config.client_connector_configuration.port}" load = IgniteApplicationService( @@ -269,6 +296,17 @@ def __start(self, ignite): return load + def __start_flood_service(self, ignite): + flood = IgniteApplicationService( + self.test_context, + IgniteConfiguration(version=ignite.config.version, client_mode=True, + discovery_spi=from_ignite_cluster(ignite)), + java_class_name=self.FLOOD_APP) + + flood.start() + + return flood + def fill_volume(ignite, path): """ @@ -286,6 +324,13 @@ def free_volume(ignite, path): ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=False) +def enlarge_volume(ignite, path, size_mb): + """ + Grows the tmpfs mounted over the path, keeping its files and what they hold, so writes there go on. + """ + ignite.nodes[0].account.ssh(f"sudo mount -o remount,size={size_mb}m {path}", allow_fail=False) + + def free_space(ignite, path): """ :return: Bytes free on the volume of the path. From 2e38cbfb69b48801dbba26cc76fcfcf4c76fea0d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 21 Oct 2026 13:10:22 +0300 Subject: [PATCH 17/20] IGNITE-29165 Add node restart on a full log volume, check gc.log --- .../tests/ignitetest/tests/disk_full_test.py | 60 +++++++++++++++++-- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index 92c87de6e240c..91e07db33a4d4 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -82,14 +82,53 @@ def test_node_survives_full_log_volume(self, ignite_version): check_serves_load_for(ignite, load, seconds=15) assert ControlUtility(ignite).cluster_state().state == "ACTIVE" - logs = [ignite.nodes[0].log_file, os.path.join(ignite.log_dir, "console.log")] - - for log in logs: + for log in log_files(ignite): check_write_failed(ignite, log) free_volume(ignite, ignite.log_dir) - for log in logs: + for log in log_files(ignite): + check_grows(ignite, log) + + load.stop() + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_restarts_on_full_log_volume(self, ignite_version): + """ + The node restarts while its log volume is full. + """ + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.log_dir, 32) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + fill_volume(ignite, ignite.log_dir) + + ignite.stop() + + # Once the load's puts fail, its count of puts stands still until the node is back. + _, errors = progress(load) + wait_until(lambda: progress(load)[1] > errors, timeout_sec=30, err_msg="The load still reaches the node") + + puts, _ = progress(load) + + # start() awaits a line in the node's new log, which the full volume refuses. + ignite.start_async(clean=False) + + wait_until(lambda: progress(load)[0] > puts, timeout_sec=60, err_msg="No puts after the restart") + + check_serves_load_for(ignite, load, seconds=15) + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + assert file_size(ignite, ignite.nodes[0].log_file) == 0, "The restarted node wrote its log" + + free_volume(ignite, ignite.log_dir) + + for log in log_files(ignite): check_grows(ignite, log) load.stop() @@ -281,7 +320,7 @@ def __ignite(self, ignite_version): def __start_load_client(self, ignite): """ - Starts and activates the node, then a load that records its puts and errors once a second. + Starts a load that records its puts and errors once a second. :return: Load application service. """ address = f"{ignite.nodes[0].account.hostname}:{ignite.config.client_connector_configuration.port}" @@ -297,6 +336,10 @@ def __start_load_client(self, ignite): return load def __start_flood_service(self, ignite): + """ + Starts the application that deploys the System.out flood service on the server. + :return: Flood application service. + """ flood = IgniteApplicationService( self.test_context, IgniteConfiguration(version=ignite.config.version, client_mode=True, @@ -308,6 +351,13 @@ def __start_flood_service(self, ignite): return flood +def log_files(ignite): + """ + :return: The node's log4j2 log, console.log and gc.log, all on the log volume. + """ + return [ignite.nodes[0].log_file] + [os.path.join(ignite.log_dir, log) for log in ("console.log", "gc.log")] + + def fill_volume(ignite, path): """ Writes a file into the path until its volume refuses the write. From 9349409e4fa9d1fdc48878880bd5ae5e1b42cf01 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 21 Oct 2026 13:32:03 +0300 Subject: [PATCH 18/20] IGNITE-29165 Clarify the wait for a failed put --- modules/ducktests/tests/ignitetest/tests/disk_full_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index 91e07db33a4d4..b13d1c50676bb 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -111,7 +111,7 @@ def test_node_restarts_on_full_log_volume(self, ignite_version): ignite.stop() - # Once the load's puts fail, its count of puts stands still until the node is back. + # Waits for a failed put, so the count of puts read next is final until the node is back. _, errors = progress(load) wait_until(lambda: progress(load)[1] > errors, timeout_sec=30, err_msg="The load still reaches the node") From ba1fc52c43dec3b4fd9a4d43f5169728b22f36b2 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 21 Oct 2026 13:43:17 +0300 Subject: [PATCH 19/20] IGNITE-29165 Simplify the flood application's cancel and javadocs --- .../ducktest/tests/disk_full/StdoutFloodApplication.java | 7 +++---- .../ducktest/tests/disk_full/StdoutFloodServiceImpl.java | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java index 44c7d25048646..1272054346e39 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java @@ -21,9 +21,8 @@ import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; /** - * Deploys {@link StdoutFloodServiceImpl} on every server node and records {@code lines-><-}, the lines the - * service wrote, once a second. Cancels the service on stop, so the servers stop writing to their - * {@code System.out} once {@code markFinished} is logged. + * Deploys {@link StdoutFloodServiceImpl} on every server node, records {@code lines-><-}, the lines the service + * wrote, once a second, and cancels the service on stop. */ public class StdoutFloodApplication extends IgniteAwareApplication { /** Service name. */ @@ -45,7 +44,7 @@ public class StdoutFloodApplication extends IgniteAwareApplication { recordResult("lines", svc.lines()); } - ignite.services().cancelAsync(SVC_NAME).get(); + ignite.services().cancel(SVC_NAME); markFinished(); } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java index 95c0e4308a06a..3904e8a840984 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java @@ -32,7 +32,7 @@ public class StdoutFloodServiceImpl implements Service, StdoutFloodService { /** Line, 1 KB with its line separator. */ private static final String LINE = "x".repeat(1023); - /** Lines written once a millisecond. */ + /** Lines written per millisecond. */ private static final int LINES_PER_MS = 10; /** Service context. */ From 03a0383a627943fbef7e7d3059f49dd51a1d26ed Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 21 Oct 2026 13:45:03 +0300 Subject: [PATCH 20/20] IGNITE-29165 Check every log in the System.out test --- .../ducktests/tests/ignitetest/tests/disk_full_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py index b13d1c50676bb..100b567e63b42 100644 --- a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -159,15 +159,15 @@ def test_node_survives_log_volume_filled_by_stdout(self, ignite_version): assert ControlUtility(ignite).cluster_state().state == "ACTIVE" assert printed_lines(flood) > lines, "The service stopped writing to System.out" - console_log = os.path.join(ignite.log_dir, "console.log") - - check_write_failed(ignite, console_log) + for log in log_files(ignite): + check_write_failed(ignite, log) flood.stop() enlarge_volume(ignite, ignite.log_dir, 128) - check_grows(ignite, ignite.nodes[0].log_file) + for log in log_files(ignite): + check_grows(ignite, log) load.stop()