|
18 | 18 | from dstack._internal.proxy.lib.errors import UnexpectedProxyError |
19 | 19 | from dstack._internal.proxy.lib.models import Project, Replica, Service |
20 | 20 | from dstack._internal.proxy.lib.repo import BaseProxyRepo |
21 | | -from dstack._internal.utils.common import get_or_error |
| 21 | +from dstack._internal.utils.common import get_or_error, run_async |
22 | 22 | from dstack._internal.utils.logging import get_logger |
23 | 23 | from dstack._internal.utils.path import FileContent |
24 | 24 |
|
|
27 | 27 | # Bound SSH startup work during a shared outage, without limiting service requests. |
28 | 28 | MAX_CONCURRENT_TUNNEL_RECONNECTS = 8 |
29 | 29 | MAX_TUNNEL_RECONNECT_DELAY = 30 |
| 30 | +# A drained connection stays open for at least this long, since Nginx workers may still route |
| 31 | +# requests to a removed replica for a short while after a reload. |
| 32 | +REPLICA_DRAIN_GRACE_PERIOD = 5 |
| 33 | +REPLICA_DRAIN_POLL_INTERVAL = 1 |
| 34 | +# Upper bound for draining, e.g. if a WebSocket connection keeps the replica busy. |
| 35 | +# Matches the default `stop_duration`, the time a job is given to stop gracefully. |
| 36 | +REPLICA_DRAIN_TIMEOUT = 300 |
30 | 37 |
|
31 | 38 |
|
32 | 39 | class ServiceClient(httpx.AsyncClient): |
@@ -202,6 +209,7 @@ class ServiceConnectionPool: |
202 | 209 | def __init__(self) -> None: |
203 | 210 | # TODO(#2238): remove connections to stopped replicas in-server |
204 | 211 | self.connections: Dict[str, ServiceConnection] = {} |
| 212 | + self._draining: set[asyncio.Task] = set() |
205 | 213 | self._reconnect_semaphore = asyncio.Semaphore(MAX_CONCURRENT_TUNNEL_RECONNECTS) |
206 | 214 |
|
207 | 215 | async def get(self, replica_id: str) -> Optional[ServiceConnection]: |
@@ -239,12 +247,30 @@ async def get_or_add( |
239 | 247 | raise |
240 | 248 | return connection |
241 | 249 |
|
242 | | - async def remove(self, replica_id: str) -> None: |
| 250 | + async def remove(self, replica_id: str, drain: bool = False) -> None: |
| 251 | + """ |
| 252 | + Remove the replica's connection from the pool and close it. |
| 253 | +
|
| 254 | + With `drain`, return right away and close the connection in the background once nothing |
| 255 | + is connected to its socket, so that requests already sent to the replica can finish. |
| 256 | + The caller must stop routing requests to the socket first, e.g. by reloading Nginx. |
| 257 | + """ |
243 | 258 | connection = self.connections.pop(replica_id, None) |
244 | | - if connection is not None: |
| 259 | + if connection is None: |
| 260 | + return |
| 261 | + if not drain: |
245 | 262 | await connection.close() |
| 263 | + return |
| 264 | + task = asyncio.create_task(_close_after_drain(connection)) |
| 265 | + # The event loop only keeps weak references to tasks |
| 266 | + self._draining.add(task) |
| 267 | + task.add_done_callback(self._draining.discard) |
246 | 268 |
|
247 | 269 | async def remove_all(self) -> None: |
| 270 | + draining = list(self._draining) |
| 271 | + for task in draining: |
| 272 | + task.cancel() # closes the connection without waiting for the drain |
| 273 | + await asyncio.gather(*draining, return_exceptions=True) |
248 | 274 | replica_ids = list(self.connections) |
249 | 275 | results = await asyncio.gather( |
250 | 276 | *(self.remove(replica_id) for replica_id in replica_ids), return_exceptions=True |
@@ -280,3 +306,65 @@ async def get_service_replica_client( |
280 | 306 | ) |
281 | 307 | connection = await service_conn_pool.get_or_add(project, service, replica) |
282 | 308 | return await connection.client() |
| 309 | + |
| 310 | + |
| 311 | +async def _close_after_drain(connection: ServiceConnection) -> None: |
| 312 | + replica_id = connection.replica.id |
| 313 | + try: |
| 314 | + if await _wait_until_idle(connection.app_socket_path): |
| 315 | + logger.debug("Connection to replica %s is drained, closing", replica_id) |
| 316 | + else: |
| 317 | + logger.warning( |
| 318 | + "Connection to replica %s is still in use after %ss, closing", |
| 319 | + replica_id, |
| 320 | + REPLICA_DRAIN_TIMEOUT, |
| 321 | + ) |
| 322 | + finally: |
| 323 | + try: |
| 324 | + await connection.close() |
| 325 | + except Exception: |
| 326 | + logger.exception("Error closing drained connection to replica %s", replica_id) |
| 327 | + |
| 328 | + |
| 329 | +async def _wait_until_idle(socket_path: Path) -> bool: |
| 330 | + """ |
| 331 | + Wait until nothing is connected to the Unix socket, for at most `REPLICA_DRAIN_TIMEOUT`. |
| 332 | +
|
| 333 | + Returns whether the socket became idle in time. |
| 334 | + """ |
| 335 | + await asyncio.sleep(REPLICA_DRAIN_GRACE_PERIOD) |
| 336 | + waited = REPLICA_DRAIN_GRACE_PERIOD |
| 337 | + # `None` means the connections cannot be counted, so wait for the timeout |
| 338 | + while await _count_socket_connections(socket_path) != 0: |
| 339 | + if waited >= REPLICA_DRAIN_TIMEOUT: |
| 340 | + return False |
| 341 | + await asyncio.sleep(REPLICA_DRAIN_POLL_INTERVAL) |
| 342 | + waited += REPLICA_DRAIN_POLL_INTERVAL |
| 343 | + return True |
| 344 | + |
| 345 | + |
| 346 | +# `SS_CONNECTED`, the state of a connected socket in `/proc/net/unix` |
| 347 | +_SS_CONNECTED = "03" |
| 348 | + |
| 349 | + |
| 350 | +async def _count_socket_connections(socket_path: Path) -> Optional[int]: |
| 351 | + """ |
| 352 | + Count the connections accepted on a listening Unix socket, or return `None` if they cannot |
| 353 | + be counted. |
| 354 | +
|
| 355 | + Accepted connections are listed in `/proc/net/unix` with the listening socket's path |
| 356 | + and the connected state. Linux-only, like the gateway. Not `psutil.net_connections()`: |
| 357 | + it reports no state for Unix sockets, so it cannot tell connections from the listener. |
| 358 | + """ |
| 359 | + try: |
| 360 | + table = await run_async(Path("/proc/net/unix").read_text) |
| 361 | + except OSError: |
| 362 | + return None |
| 363 | + path = str(socket_path) |
| 364 | + count = 0 |
| 365 | + # Columns: Num RefCount Protocol Flags Type St Inode [Path] |
| 366 | + for line in table.splitlines()[1:]: |
| 367 | + fields = line.split(maxsplit=7) |
| 368 | + if len(fields) == 8 and fields[7] == path and fields[5] == _SS_CONNECTED: |
| 369 | + count += 1 |
| 370 | + return count |
0 commit comments