Skip to content

Commit 73a6dc6

Browse files
authored
Allow retries with type: dynamo router (#4474)
Retry was rejected because workers got the router's internal IP once, via DSTACK_ROUTER_INTERNAL_IP, and a retried router could come back at a different address. Since #4402, Dynamo services run etcd in a dedicated group referenced via replica IP interpolation (#4177), and #4473 removed the variable, so nothing depends on the router's address anymore.
1 parent a4bdeb7 commit 73a6dc6

2 files changed

Lines changed: 0 additions & 120 deletions

File tree

‎src/dstack/_internal/core/models/runs.py‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
)
4343
from dstack._internal.core.models.repos import AnyRunRepoData
4444
from dstack._internal.core.models.resources import Memory, ResourcesSpec
45-
from dstack._internal.core.models.routers import RouterType
4645
from dstack._internal.core.models.unix import UnixUser
4746
from dstack._internal.core.models.volumes import MountPoint
4847
from dstack._internal.utils import common as common_utils
@@ -629,31 +628,6 @@ def _merged_profile(self) -> Self:
629628
self.merged_profile = merged_profile
630629
return self
631630

632-
@model_validator(mode="after")
633-
def _validate_dynamo_no_retry(self) -> Self:
634-
"""Reject `retry` for services with a Dynamo router replica group.
635-
Dynamo workers cache the router's internal IP at provisioning time. A
636-
retry would produce a new router and likely a new internal_ip, leaving workers bound
637-
to a router that no longer exists.
638-
"""
639-
merged_profile = self.merged_profile
640-
cfg = self.configuration
641-
if merged_profile is None or merged_profile.retry is None:
642-
return self
643-
if not isinstance(cfg, ServiceConfiguration):
644-
return self
645-
for g in cfg.replica_groups:
646-
if g.router is not None and g.router.type == RouterType.DYNAMO:
647-
raise ValueError(
648-
"Retry cannot be configured for services with a Dynamo "
649-
"router replica group. The router's address must remain "
650-
"stable for the life of the run; allowing retry would "
651-
"leave workers bound to a router that no longer exists. "
652-
"Remove `retry` from the profile/configuration and "
653-
"re-apply."
654-
)
655-
return self
656-
657631

658632
class ServiceModelSpec(CoreModel):
659633
name: str

‎src/tests/_internal/core/models/test_runs.py‎

Lines changed: 0 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -117,100 +117,6 @@ def test_run_get_error_returns_none_for_specific_reasons():
117117
assert reason in no_error_reasons
118118

119119

120-
def _service_run_spec_dict(router_type=None, retry=None, top_level_extras=None):
121-
"""Build a minimal RunSpec dict for a service.
122-
123-
`router_type`: None | "smg" | "dynamo" — controls whether/how the
124-
second replica group has a router field.
125-
`retry`: optional dict passed as `profile.retry`.
126-
`top_level_extras`: optional dict merged into the service configuration.
127-
"""
128-
replicas = [{"name": "worker", "commands": ["echo hi"], "count": 1}]
129-
if router_type is not None:
130-
replicas.append(
131-
{
132-
"name": "router",
133-
"router": {"type": router_type},
134-
"commands": ["echo router"],
135-
"count": 1,
136-
}
137-
)
138-
configuration = {
139-
"type": "service",
140-
"port": 8000,
141-
"replicas": replicas,
142-
}
143-
if top_level_extras:
144-
configuration.update(top_level_extras)
145-
profile = {"name": "default"}
146-
if retry is not None:
147-
profile["retry"] = retry
148-
return {
149-
"run_name": "test-run",
150-
"repo_id": "test-repo",
151-
"configuration_path": "dstack.yaml",
152-
"configuration": configuration,
153-
"profile": profile,
154-
"ssh_key_pub": "ssh-rsa AAAA...",
155-
"repo_data": {"repo_type": "virtual"},
156-
}
157-
158-
159-
class TestDynamoNoRetryValidator:
160-
def test_dynamo_router_with_retry_at_profile_level_is_rejected(self):
161-
spec = _service_run_spec_dict(
162-
router_type="dynamo",
163-
retry={"on_events": ["error"]},
164-
)
165-
with pytest.raises(ValidationError, match="Dynamo"):
166-
RunSpec.model_validate(spec)
167-
168-
def test_dynamo_router_with_retry_in_configuration_is_rejected(self):
169-
# retry can also be specified at configuration level; _merged_profile
170-
# folds it into merged_profile.retry, so the validator should still
171-
# catch it.
172-
spec = _service_run_spec_dict(
173-
router_type="dynamo",
174-
top_level_extras={"retry": {"on_events": ["error"]}},
175-
)
176-
with pytest.raises(ValidationError, match="Dynamo"):
177-
RunSpec.model_validate(spec)
178-
179-
def test_dynamo_router_without_retry_is_accepted(self):
180-
spec = _service_run_spec_dict(router_type="dynamo", retry=None)
181-
# Should not raise:
182-
RunSpec.model_validate(spec)
183-
184-
def test_smg_router_with_retry_is_accepted(self):
185-
spec = _service_run_spec_dict(
186-
router_type="smg",
187-
retry={"on_events": ["error"]},
188-
)
189-
# SMG services are unaffected by the validator.
190-
RunSpec.model_validate(spec)
191-
192-
def test_service_without_router_with_retry_is_accepted(self):
193-
spec = _service_run_spec_dict(router_type=None, retry={"on_events": ["error"]})
194-
RunSpec.model_validate(spec)
195-
196-
def test_non_service_run_with_retry_is_accepted(self):
197-
# Validator is service-only. A task or dev-environment with retry
198-
# shouldn't be flagged regardless of the rest of the spec.
199-
spec = {
200-
"run_name": "test-run",
201-
"repo_id": "test-repo",
202-
"configuration_path": "dstack.yaml",
203-
"configuration": {
204-
"type": "task",
205-
"commands": ["echo hi"],
206-
},
207-
"profile": {"name": "default", "retry": {"on_events": ["error"]}},
208-
"ssh_key_pub": "ssh-rsa AAAA...",
209-
"repo_data": {"repo_type": "virtual"},
210-
}
211-
RunSpec.model_validate(spec)
212-
213-
214120
class TestRunSpec:
215121
@pytest.mark.parametrize(
216122
"spec",

0 commit comments

Comments
 (0)