Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion contributing/ORPHAN-CLEANUP.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ The pipeline processes each row at most once per cleanup interval (`DSTACK_SERVE

Processing a row:

- STEP 1: Check whether the backend may have new orphans: it was never cleaned up, the cleanup mode changed, or the project had instance or job activity since the last clean cleanup (`clean_at`). If not, the row is only marked processed. Checking activity here rather than in the fetch query keeps fetching cheap regardless of project sizes, at the cost of a small query and row update per enrolled backend per interval.
- STEP 1: Check whether the backend may have new orphans: it was never cleaned up, the cleanup mode changed, or the project had instance activity or job activity in its fleets since the last clean cleanup (`clean_at`). If not, the row is only marked processed. Checking activity here rather than in the fetch query keeps fetching cheap regardless of project sizes, at the cost of a small query and row update per enrolled backend per interval.
- STEP 2: List the backend's resource groups (`list_resource_groups()`). A resource group is one provisioning attempt as found in the provider: the owner resource (`primary_id`, the instance) and internal resources created with it.
- STEP 3: Load the project's `ProviderRefModel` records and their instances (including soft-deleted ones) by `provider_ref`, in one statement. An instance is committed with its record, so it can't be missed while the record is seen.
- STEP 4: Classify each group:
Expand Down
2 changes: 1 addition & 1 deletion mkdocs/docs/concepts/exports.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ for running tasks, dev environments, and services. Imported resources appear wit
Currently, [fleets](fleets.md) and [gateways](gateways.md) can be exported.

??? info "Limitations"
- Importer projects can only use the existing instances of imported [backend fleets](fleets.md#backend-fleets). New instances are not provisioned for importers' runs.
- [Container-based](backends.md#container-based) backends of the exporter project cannot be used for fleet importers' runs.
- Runs that mount [network volumes](volumes.md#network-volumes) cannot use imported backend fleets.

??? info "Required role"
Expand Down
13 changes: 11 additions & 2 deletions src/dstack/_internal/core/backends/base/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand All @@ -161,6 +162,11 @@ def run_job(
If required to wait to get the IP address or SSH port, return partially filled `JobProvisioningData`
and implement `update_provisioning_data()`.

`project_name`, `project_ssh_public_key`, and `project_ssh_private_key` belong to the
project that will own the provisioned resources -- the project of the job's fleet. It
differs from `run.project_name` if the fleet is imported from another project, so name
and tag the resources using `project_name`, not `run.project_name`.

`extra_authorized_keys` are the public keys to authorize on the job container in addition
to `project_ssh_public_key`, as decided by the caller -- typically the user key, or
nothing if the server does not let the user connect to the container directly. The project
Expand Down Expand Up @@ -432,6 +438,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand All @@ -447,7 +454,7 @@ def run_job(
submits the keys to the shim later, see `Compute.run_job()`.
"""
instance_config = InstanceConfiguration(
project_name=run.project_name,
project_name=project_name,
instance_name=get_job_instance_name(run, job),
user=run.user,
ssh_keys=[SSHKey(public=project_ssh_public_key.strip())],
Expand Down Expand Up @@ -530,6 +537,7 @@ def run_jobs(
run: Run,
job_configurations: List[JobConfiguration],
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
placement_group: Optional[PlacementGroup],
Expand Down Expand Up @@ -849,6 +857,7 @@ def generate_unique_instance_name(
def generate_unique_instance_name_for_job(
run: Run,
job: Job,
project_name: str,
max_length: int = _DEFAULT_MAX_RESOURCE_NAME_LEN,
with_provider_ref: bool = False,
) -> str:
Expand All @@ -860,7 +869,7 @@ def generate_unique_instance_name_for_job(
"""
return generate_unique_backend_name(
resource_name=get_job_instance_name(run, job),
project_name=run.project_name,
project_name=project_name,
max_length=max_length,
provider_ref=get_job_provider_ref(job) if with_provider_ref else None,
)
Expand Down
3 changes: 2 additions & 1 deletion src/dstack/_internal/core/backends/daytona/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand All @@ -120,7 +121,7 @@ def run_job(
if not image_name or re.search(r"\s", image_name):
raise ComputeError("Invalid Daytona container image name")
instance_name = generate_unique_instance_name_for_job(
run, job, max_length=MAX_INSTANCE_NAME_LEN
run, job, project_name, max_length=MAX_INSTANCE_NAME_LEN
)
commands = get_docker_commands(
build_authorized_keys(project_ssh_public_key, extra_authorized_keys)
Expand Down
9 changes: 5 additions & 4 deletions src/dstack/_internal/core/backends/kubernetes/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: list[Volume],
Expand All @@ -259,13 +260,13 @@ def run_job(
# There is one jump pod per project that is used as an ssh proxy jump to connect
# to all job pods of the same project.
# The service is created here and configured later in update_provisioning_data()
jump_pod_name = f"dstack-{run.project_name}-ssh-jump-pod"
jump_pod_name = f"dstack-{project_name}-ssh-jump-pod"
jump_pod_service_name = get_pod_service_name(jump_pod_name)
_create_jump_pod_service_if_not_exists(
api=api,
apps_api=client.AppsV1Api(cluster.api_client),
namespace=namespace,
project_name=run.project_name,
project_name=project_name,
jump_pod_name=jump_pod_name,
jump_pod_service_name=jump_pod_service_name,
jump_pod_port=cluster.proxy_jump.port,
Expand All @@ -277,13 +278,13 @@ def run_job(
)

pod_name = generate_unique_instance_name_for_job(
run, job, max_length=LABEL_VALUE_MAX_LENGTH
run, job, project_name, max_length=LABEL_VALUE_MAX_LENGTH
)

base_labels = build_base_labels(
component="job",
unique_name=pod_name,
project=run.project_name,
project=project_name,
name=job.job_spec.job_name,
user=run.user,
provider_ref=get_job_provider_ref(job),
Expand Down
3 changes: 2 additions & 1 deletion src/dstack/_internal/core/backends/lium/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand All @@ -117,7 +118,7 @@ def run_job(
) -> JobProvisioningData:
volume_commands = _get_volume_commands(run, job, volumes)
instance_name = generate_unique_instance_name_for_job(
run, job, max_length=MAX_INSTANCE_NAME_LEN
run, job, project_name, max_length=MAX_INSTANCE_NAME_LEN
)
authorized_keys = build_authorized_keys(project_ssh_public_key, extra_authorized_keys)
# CLI keys also need forwarding access through Lium's root SSH service.
Expand Down
6 changes: 4 additions & 2 deletions src/dstack/_internal/core/backends/runpod/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand All @@ -137,7 +138,7 @@ def run_job(
extra_authorized_keys: list[str],
) -> JobProvisioningData:
instance_config = InstanceConfiguration(
project_name=run.project_name,
project_name=project_name,
instance_name=get_job_instance_name(run, job),
ssh_keys=[],
user=run.user,
Expand Down Expand Up @@ -277,6 +278,7 @@ def run_jobs(
run: Run,
job_configurations: List[JobConfiguration],
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
placement_group: Optional[PlacementGroup],
Expand All @@ -290,7 +292,7 @@ def run_jobs(
master_job_volumes = master_job_configuration.volumes
all_volumes_names = set(v.name for jc in job_configurations for v in jc.volumes)
instance_config = InstanceConfiguration(
project_name=run.project_name,
project_name=project_name,
instance_name=get_job_instance_name(run, master_job),
ssh_keys=[],
user=run.user,
Expand Down
7 changes: 6 additions & 1 deletion src/dstack/_internal/core/backends/slurm/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: list[Volume],
Expand All @@ -149,6 +150,7 @@ def run_job(
run=run,
job=job,
instance_offer=instance_offer,
project_name=project_name,
project_ssh_public_key=project_ssh_public_key,
extra_authorized_keys=extra_authorized_keys,
requirements=requirements,
Expand All @@ -161,6 +163,7 @@ def run_jobs(
run: Run,
job_configurations: list[JobConfiguration],
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
placement_group: Optional[PlacementGroup],
Expand All @@ -172,6 +175,7 @@ def run_jobs(
run=run,
job=master_job,
instance_offer=instance_offer,
project_name=project_name,
project_ssh_public_key=project_ssh_public_key,
extra_authorized_keys=extra_authorized_keys,
requirements=requirements,
Expand All @@ -196,6 +200,7 @@ def _run_slurm_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
extra_authorized_keys: list[str],
requirements: Requirements,
Expand All @@ -216,7 +221,7 @@ def _run_slurm_job(
assert run.run_spec.run_name is not None
slurm_job_name = generate_unique_backend_name(
resource_name=run.run_spec.run_name,
project_name=run.project_name,
project_name=project_name,
max_length=SLURM_JOB_NAME_MAX_LENGTH,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ class {{ backend_name }}Compute(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand Down
3 changes: 2 additions & 1 deletion src/dstack/_internal/core/backends/vastai/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ def run_job(
run: Run,
job: Job,
instance_offer: InstanceOfferWithAvailability,
project_name: str,
project_ssh_public_key: str,
project_ssh_private_key: str,
volumes: List[Volume],
Expand All @@ -126,7 +127,7 @@ def run_job(
extra_authorized_keys: list[str],
) -> JobProvisioningData:
instance_name = generate_unique_instance_name_for_job(
run, job, max_length=MAX_INSTANCE_NAME_LEN
run, job, project_name, max_length=MAX_INSTANCE_NAME_LEN
)
commands = get_docker_commands(
build_authorized_keys(project_ssh_public_key, extra_authorized_keys)
Expand Down
Loading
Loading