Skip to content

Commit 1f64ff3

Browse files
un-defclaude
andauthored
Drain gateway replica connections on unregister (#4495)
Unregistering a replica closed its SSH tunnel before reloading Nginx, so requests already in flight through the tunnel were cut, and POST requests routed to the replica before the reload got 502. Reload Nginx first, then close the tunnel in the background: after a 5s grace period for Nginx workers that haven't picked up the reload, once nothing is connected to its socket (per /proc/net/unix), and after 5 minutes at most. Part-of: #4493 Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent beda581 commit 1f64ff3

7 files changed

Lines changed: 251 additions & 16 deletions

File tree

‎contributing/PROXY.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ The in-server proxy opens an SSH connection when it needs to forward the first r
6464

6565
The gateway opens SSH connections when each replica is registered.
6666

67+
When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica's container independently, so it can still cut requests that outlive the container.
68+
6769
### Communication with the `dstack` server
6870

6971
The in-server proxy is part of the `dstack` server, so no network communication takes place between them.

‎src/dstack/_internal/proxy/gateway/services/registry.py‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -122,11 +122,13 @@ async def unregister_service(
122122

123123
logger.debug("Unregistering service %s", service.fmt())
124124

125+
await nginx.unregister(service)
126+
# Only after the reload, so that Nginx stops routing requests to the replicas
125127
await stop_replica_connections(
126128
ids=(r.id for r in service.replicas),
127129
service_conn_pool=service_conn_pool,
130+
drain=True,
128131
)
129-
await nginx.unregister(service)
130132
await repo.delete_models_by_run(project_name, run_name)
131133
await repo.delete_service(project_name, run_name)
132134

@@ -344,18 +346,16 @@ async def apply_service(
344346
nginx: Nginx,
345347
service_conn_pool: ServiceConnectionPool,
346348
) -> dict[models.Replica, BaseException]:
349+
removed_replica_ids = []
347350
if old_service is not None:
348351
if service.domain != old_service.domain:
349352
raise UnexpectedProxyError(
350353
f"Did not expect service {service.fmt()}"
351354
f" domain name to change ({old_service.domain} -> {service.domain})"
352355
)
353-
await stop_replica_connections(
354-
ids=(
355-
replica.id for replica in old_service.replicas if replica not in service.replicas
356-
),
357-
service_conn_pool=service_conn_pool,
358-
)
356+
removed_replica_ids = [
357+
replica.id for replica in old_service.replicas if replica not in service.replicas
358+
]
359359
replica_conns, replica_failures = await get_or_add_replica_connections(
360360
service, repo, service_conn_pool
361361
)
@@ -370,6 +370,10 @@ async def apply_service(
370370
]
371371
service_config = await get_nginx_service_config(service, replica_configs)
372372
await nginx.register(service_config, (await repo.get_config()).acme_settings)
373+
# Only after the reload, so that Nginx stops routing requests to the removed replicas
374+
await stop_replica_connections(
375+
ids=removed_replica_ids, service_conn_pool=service_conn_pool, drain=True
376+
)
373377
return replica_failures
374378

375379

@@ -402,10 +406,17 @@ async def get_or_add_replica_connections(
402406

403407

404408
async def stop_replica_connections(
405-
ids: Iterable[str], service_conn_pool: ServiceConnectionPool
409+
ids: Iterable[str], service_conn_pool: ServiceConnectionPool, drain: bool = False
406410
) -> None:
407-
tasks = map(service_conn_pool.remove, ids)
408-
results = await asyncio.gather(*tasks, return_exceptions=True)
411+
"""
412+
With `drain`, return right away and close each connection once requests already sent
413+
through it finish. See `ServiceConnectionPool.remove`.
414+
"""
415+
ids = list(ids)
416+
results = await asyncio.gather(
417+
*(service_conn_pool.remove(replica_id, drain=drain) for replica_id in ids),
418+
return_exceptions=True,
419+
)
409420
for replica_id, exc in zip(ids, results):
410421
if isinstance(exc, Exception):
411422
logger.error("Error stopping connection to replica %s: %s", replica_id, exc)

‎src/dstack/_internal/proxy/gateway/testing/common.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ class Mocks:
1111
run_certbot: AnyMock
1212
open_conn: AnyMock
1313
close_conn: AnyMock
14+
wait_until_idle: AnyMock

‎src/dstack/_internal/proxy/lib/services/service_connection.py‎

Lines changed: 91 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from dstack._internal.proxy.lib.errors import UnexpectedProxyError
1919
from dstack._internal.proxy.lib.models import Project, Replica, Service
2020
from dstack._internal.proxy.lib.repo import BaseProxyRepo
21-
from dstack._internal.utils.common import get_or_error
21+
from dstack._internal.utils.common import get_or_error, run_async
2222
from dstack._internal.utils.logging import get_logger
2323
from dstack._internal.utils.path import FileContent
2424

@@ -27,6 +27,13 @@
2727
# Bound SSH startup work during a shared outage, without limiting service requests.
2828
MAX_CONCURRENT_TUNNEL_RECONNECTS = 8
2929
MAX_TUNNEL_RECONNECT_DELAY = 30
30+
# A drained connection stays open for at least this long, since Nginx workers may still route
31+
# requests to a removed replica for a short while after a reload.
32+
REPLICA_DRAIN_GRACE_PERIOD = 5
33+
REPLICA_DRAIN_POLL_INTERVAL = 1
34+
# Upper bound for draining, e.g. if a WebSocket connection keeps the replica busy.
35+
# Matches the default `stop_duration`, the time a job is given to stop gracefully.
36+
REPLICA_DRAIN_TIMEOUT = 300
3037

3138

3239
class ServiceClient(httpx.AsyncClient):
@@ -202,6 +209,7 @@ class ServiceConnectionPool:
202209
def __init__(self) -> None:
203210
# TODO(#2238): remove connections to stopped replicas in-server
204211
self.connections: Dict[str, ServiceConnection] = {}
212+
self._draining: set[asyncio.Task] = set()
205213
self._reconnect_semaphore = asyncio.Semaphore(MAX_CONCURRENT_TUNNEL_RECONNECTS)
206214

207215
async def get(self, replica_id: str) -> Optional[ServiceConnection]:
@@ -239,12 +247,30 @@ async def get_or_add(
239247
raise
240248
return connection
241249

242-
async def remove(self, replica_id: str) -> None:
250+
async def remove(self, replica_id: str, drain: bool = False) -> None:
251+
"""
252+
Remove the replica's connection from the pool and close it.
253+
254+
With `drain`, return right away and close the connection in the background once nothing
255+
is connected to its socket, so that requests already sent to the replica can finish.
256+
The caller must stop routing requests to the socket first, e.g. by reloading Nginx.
257+
"""
243258
connection = self.connections.pop(replica_id, None)
244-
if connection is not None:
259+
if connection is None:
260+
return
261+
if not drain:
245262
await connection.close()
263+
return
264+
task = asyncio.create_task(_close_after_drain(connection))
265+
# The event loop only keeps weak references to tasks
266+
self._draining.add(task)
267+
task.add_done_callback(self._draining.discard)
246268

247269
async def remove_all(self) -> None:
270+
draining = list(self._draining)
271+
for task in draining:
272+
task.cancel() # closes the connection without waiting for the drain
273+
await asyncio.gather(*draining, return_exceptions=True)
248274
replica_ids = list(self.connections)
249275
results = await asyncio.gather(
250276
*(self.remove(replica_id) for replica_id in replica_ids), return_exceptions=True
@@ -280,3 +306,65 @@ async def get_service_replica_client(
280306
)
281307
connection = await service_conn_pool.get_or_add(project, service, replica)
282308
return await connection.client()
309+
310+
311+
async def _close_after_drain(connection: ServiceConnection) -> None:
312+
replica_id = connection.replica.id
313+
try:
314+
if await _wait_until_idle(connection.app_socket_path):
315+
logger.debug("Connection to replica %s is drained, closing", replica_id)
316+
else:
317+
logger.warning(
318+
"Connection to replica %s is still in use after %ss, closing",
319+
replica_id,
320+
REPLICA_DRAIN_TIMEOUT,
321+
)
322+
finally:
323+
try:
324+
await connection.close()
325+
except Exception:
326+
logger.exception("Error closing drained connection to replica %s", replica_id)
327+
328+
329+
async def _wait_until_idle(socket_path: Path) -> bool:
330+
"""
331+
Wait until nothing is connected to the Unix socket, for at most `REPLICA_DRAIN_TIMEOUT`.
332+
333+
Returns whether the socket became idle in time.
334+
"""
335+
await asyncio.sleep(REPLICA_DRAIN_GRACE_PERIOD)
336+
waited = REPLICA_DRAIN_GRACE_PERIOD
337+
# `None` means the connections cannot be counted, so wait for the timeout
338+
while await _count_socket_connections(socket_path) != 0:
339+
if waited >= REPLICA_DRAIN_TIMEOUT:
340+
return False
341+
await asyncio.sleep(REPLICA_DRAIN_POLL_INTERVAL)
342+
waited += REPLICA_DRAIN_POLL_INTERVAL
343+
return True
344+
345+
346+
# `SS_CONNECTED`, the state of a connected socket in `/proc/net/unix`
347+
_SS_CONNECTED = "03"
348+
349+
350+
async def _count_socket_connections(socket_path: Path) -> Optional[int]:
351+
"""
352+
Count the connections accepted on a listening Unix socket, or return `None` if they cannot
353+
be counted.
354+
355+
Accepted connections are listed in `/proc/net/unix` with the listening socket's path
356+
and the connected state. Linux-only, like the gateway. Not `psutil.net_connections()`:
357+
it reports no state for Unix sockets, so it cannot tell connections from the listener.
358+
"""
359+
try:
360+
table = await run_async(Path("/proc/net/unix").read_text)
361+
except OSError:
362+
return None
363+
path = str(socket_path)
364+
count = 0
365+
# Columns: Num RefCount Protocol Flags Type St Inode [Path]
366+
for line in table.splitlines()[1:]:
367+
fields = line.split(maxsplit=7)
368+
if len(fields) == 8 and fields[7] == path and fields[5] == _SS_CONNECTED:
369+
count += 1
370+
return count

‎src/tests/_internal/proxy/gateway/conftest.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,15 @@ def system_mocks() -> Generator[Mocks, None, None]:
1616
patch(f"{nginx}.Nginx.run_certbot") as run_certbot,
1717
patch(f"{connection}.ServiceConnection.open") as open_conn,
1818
patch(f"{connection}.ServiceConnection.close") as close_conn,
19+
patch(f"{connection}._wait_until_idle") as wait_until_idle,
1920
):
2021
sudo.return_value = []
22+
# Drains finish right away unless a test holds them
23+
wait_until_idle.return_value = True
2124
yield Mocks(
2225
reload_nginx=reload_nginx,
2326
run_certbot=run_certbot,
2427
open_conn=open_conn,
2528
close_conn=close_conn,
29+
wait_until_idle=wait_until_idle,
2630
)

‎src/tests/_internal/proxy/gateway/routers/test_registry.py‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import asyncio
12
import re
23
import uuid
34
from datetime import datetime, timezone
@@ -95,6 +96,28 @@ def sample_model_options(name: str = "test-model") -> dict:
9596
}
9697

9798

99+
def hold_drains(mocks: Mocks) -> asyncio.Event:
100+
"""Keep the connections of removed replicas draining until the returned event is set."""
101+
drained = asyncio.Event()
102+
103+
async def wait_until_idle(socket_path: Path) -> bool:
104+
await drained.wait()
105+
return True
106+
107+
mocks.wait_until_idle.side_effect = wait_until_idle
108+
return drained
109+
110+
111+
async def wait_for_closed_connections(mocks: Mocks, count: int) -> None:
112+
"""Wait for drained connections, which are closed in the background."""
113+
114+
async def closed() -> None:
115+
while mocks.close_conn.await_count < count:
116+
await asyncio.sleep(0)
117+
118+
await asyncio.wait_for(closed(), timeout=5)
119+
120+
98121
@pytest.mark.asyncio
99122
class TestUpdateReplicaProxyJumpKeys:
100123
async def test_saves_key_without_opening_a_missing_connection(self, tmp_path, system_mocks):
@@ -656,6 +679,7 @@ async def test_unregister_not_registered_error(
656679
assert system_mocks.reload_nginx.call_count == 0
657680

658681
async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mocks) -> None:
682+
drained = hold_drains(system_mocks)
659683
client = make_client(tmp_path)
660684
# register service
661685
resp = await client.post(
@@ -676,7 +700,10 @@ async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mock
676700
assert resp.status_code == 200
677701
assert not (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
678702
assert system_mocks.reload_nginx.call_count == 4
679-
assert system_mocks.close_conn.call_count == 2
703+
# Requests already sent to the replicas can still finish
704+
system_mocks.close_conn.assert_not_awaited()
705+
drained.set()
706+
await wait_for_closed_connections(system_mocks, 2)
680707

681708
async def test_unregister_with_model(self, tmp_path: Path, system_mocks: Mocks) -> None:
682709
repo = GatewayProxyRepo()
@@ -698,6 +725,7 @@ async def test_unregister_with_model(self, tmp_path: Path, system_mocks: Mocks)
698725
@pytest.mark.asyncio
699726
class TestUnregisterReplica:
700727
async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
728+
drained = hold_drains(system_mocks)
701729
client = make_client(tmp_path)
702730
# register service
703731
resp = await client.post(
@@ -725,7 +753,11 @@ async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
725753
assert "replica xxx-xxx" in conf
726754
assert "replica yyy-yyy" not in conf
727755
assert system_mocks.reload_nginx.call_count == 4
728-
assert system_mocks.close_conn.call_count == 1
756+
# Requests already sent to the replica can still finish
757+
system_mocks.close_conn.assert_not_awaited()
758+
drained.set()
759+
await wait_for_closed_connections(system_mocks, 1)
760+
system_mocks.wait_until_idle.assert_awaited_once()
729761

730762
async def test_unregister_no_replica_error(self, tmp_path: Path, system_mocks: Mocks) -> None:
731763
client = make_client(tmp_path)

0 commit comments

Comments
 (0)