fix(agent-gateway): add retry strategy

src/sap_cloud_sdk/agentgateway/_lob.py

@@ -8,6 +8,7 @@
 import asyncio
 import logging
 import os
+import time
 import uuid
 
 import httpx
@@ -62,6 +63,9 @@
 
 _DESTINATION_INSTANCE = "default"
 
+_FETCH_AUTH_MAX_ATTEMPTS = 3
+_FETCH_AUTH_BACKOFF_BASE = 1.0  # seconds; actual delays: 1s, 2s
+
 
 def _system_scope_key(tenant_subdomain: str) -> str:
     """Build the cache scope key for tenant-scoped system auth."""
@@ -101,6 +105,11 @@ def _fetch_auth_token(
     destination service (e.g. strips the "Bearer " prefix from "Bearer <jwt>"),
     and the gateway URL from the destination's URL property.
 
+    Retries up to _FETCH_AUTH_MAX_ATTEMPTS times with exponential backoff on
+    transient failures (OAuth2 token endpoint blips, unparseable destination
+    responses). MCPServerNotFoundError is not retried — it indicates a genuine
+    missing resource.
+
     Args:
         dest_name: Destination name.
         tenant_subdomain: Tenant subdomain for multi-tenant lookup.
@@ -110,35 +119,56 @@ def _fetch_auth_token(
         Tuple of (raw_jwt, gateway_url).
 
     Raises:
-        MCPServerNotFoundError: If no auth token is returned.
+        MCPServerNotFoundError: If no auth token is returned (not retried).
+        Exception: Last exception raised after all retry attempts are exhausted.
     """
-    client = create_destination_client(
-        instance=_DESTINATION_INSTANCE,
-        _telemetry_source=Module.AGENTGATEWAY,
-    )
-    dest = client.get_destination(
-        dest_name,
-        level=ConsumptionLevel.PROVIDER_SUBACCOUNT,
-        options=options,
-        tenant=tenant_subdomain,
-    )
-
-    if not dest or not isinstance(dest, Destination) or not dest.auth_tokens:
-        raise MCPServerNotFoundError(
-            f"No auth token returned for destination '{dest_name}'"
-        )
+    last_exc: Exception | None = None
+    for attempt in range(_FETCH_AUTH_MAX_ATTEMPTS):
+        try:
+            client = create_destination_client(
+                instance=_DESTINATION_INSTANCE,
+                _telemetry_source=Module.AGENTGATEWAY,
+            )
+            dest = client.get_destination(
+                dest_name,
+                level=ConsumptionLevel.PROVIDER_SUBACCOUNT,
+                options=options,
+                tenant=tenant_subdomain,
+            )
 
-    auth_token = dest.auth_tokens[0]
-    header_value = auth_token.http_header.get("value") or ""
-    if not header_value:
-        raise MCPServerNotFoundError(f"Empty auth header for destination '{dest_name}'")
+            if not dest or not isinstance(dest, Destination) or not dest.auth_tokens:
+                raise MCPServerNotFoundError(
+                    f"No auth token returned for destination '{dest_name}'"
+                )
 
-    # Strip "Bearer " prefix — AuthResult.access_token is always a raw JWT
-    raw_token = header_value.removeprefix("Bearer ").strip()
+            auth_token = dest.auth_tokens[0]
+            header_value = auth_token.http_header.get("value") or ""
+            if not header_value:
+                raise MCPServerNotFoundError(
+                    f"Empty auth header for destination '{dest_name}'"
+                )
 
-    gateway_url = (dest.url or "").rstrip("/")
+            # Strip "Bearer " prefix — AuthResult.access_token is always a raw JWT
+            raw_token = header_value.removeprefix("Bearer ").strip()
+            gateway_url = (dest.url or "").rstrip("/")
+            return raw_token, gateway_url
+
+        except MCPServerNotFoundError:
+            raise
+        except Exception as exc:
+            last_exc = exc
+            if attempt < _FETCH_AUTH_MAX_ATTEMPTS - 1:
+                delay = _FETCH_AUTH_BACKOFF_BASE * (2**attempt)
+                logger.warning(
+                    "Auth token fetch failed (attempt %d/%d): %s — retrying in %.1fs",
+                    attempt + 1,
+                    _FETCH_AUTH_MAX_ATTEMPTS,
+                    exc,
+                    delay,
+                )
+                time.sleep(delay)
 
-    return raw_token, gateway_url
+    raise last_exc  # type: ignore[misc]
 
 
 def get_ias_client_id_lob() -> str:

tests/agentgateway/unit/test_lob.py

@@ -195,6 +195,109 @@ def test_passes_options_to_destination(self):
             )
 
 
+# ============================================================
+# Test: _fetch_auth_token retry behaviour
+# ============================================================
+
+
+class TestFetchAuthTokenRetry:
+    """Tests for retry logic in _fetch_auth_token."""
+
+    def _make_dest(self, token="Bearer jwt", url="https://agw.example.com"):
+        dest = MagicMock(spec=Destination)
+        dest.auth_tokens = [MagicMock()]
+        dest.auth_tokens[0].http_header = {"value": token}
+        dest.url = url
+        return dest
+
+    def test_succeeds_on_first_attempt(self):
+        """Return result immediately when the first attempt succeeds."""
+        with patch(
+            "sap_cloud_sdk.agentgateway._lob.create_destination_client"
+        ) as mock_client, patch("sap_cloud_sdk.agentgateway._lob.time.sleep") as mock_sleep:
+            mock_client.return_value.get_destination.return_value = self._make_dest()
+
+            result = _fetch_auth_token("dest", "tenant")
+
+            assert result == ("jwt", "https://agw.example.com")
+            mock_sleep.assert_not_called()
+
+    def test_retries_on_transient_error_then_succeeds(self):
+        """Retry after a transient RuntimeError and return result on second attempt."""
+        dest = self._make_dest()
+        with patch(
+            "sap_cloud_sdk.agentgateway._lob.create_destination_client"
+        ) as mock_client, patch("sap_cloud_sdk.agentgateway._lob.time.sleep") as mock_sleep:
+            mock_client.return_value.get_destination.side_effect = [
+                RuntimeError("Failed to obtain OAuth2 token: (missing_token)"),
+                dest,
+            ]
+
+            result = _fetch_auth_token("dest", "tenant")
+
+            assert result == ("jwt", "https://agw.example.com")
+            mock_sleep.assert_called_once_with(1.0)
+
+    def test_retries_on_destination_parse_error_then_succeeds(self):
+        """Retry after a destination parse error and return result on third attempt."""
+        dest = self._make_dest()
+        with patch(
+            "sap_cloud_sdk.agentgateway._lob.create_destination_client"
+        ) as mock_client, patch("sap_cloud_sdk.agentgateway._lob.time.sleep") as mock_sleep:
+            mock_client.return_value.get_destination.side_effect = [
+                Exception("failed to parse consume destination response: ..."),
+                Exception("invalid JSON in list fragments response: ..."),
+                dest,
+            ]
+
+            result = _fetch_auth_token("dest", "tenant")
+
+            assert result == ("jwt", "https://agw.example.com")
+            assert mock_sleep.call_count == 2
+            mock_sleep.assert_any_call(1.0)
+            mock_sleep.assert_any_call(2.0)
+
+    def test_raises_after_all_attempts_exhausted(self):
+        """Re-raise the last exception after all retry attempts fail."""
+        err = RuntimeError("Failed to obtain OAuth2 token: (missing_token)")
+        with patch(
+            "sap_cloud_sdk.agentgateway._lob.create_destination_client"
+        ) as mock_client, patch("sap_cloud_sdk.agentgateway._lob.time.sleep"):
+            mock_client.return_value.get_destination.side_effect = err
+
+            with pytest.raises(RuntimeError, match="Failed to obtain OAuth2 token"):
+                _fetch_auth_token("dest", "tenant")
+
+            assert mock_client.return_value.get_destination.call_count == 3
+
+    def test_mcp_server_not_found_not_retried(self):
+        """MCPServerNotFoundError is not retried — it is a genuine missing resource."""
+        with patch(
+            "sap_cloud_sdk.agentgateway._lob.create_destination_client"
+        ) as mock_client, patch("sap_cloud_sdk.agentgateway._lob.time.sleep") as mock_sleep:
+            mock_client.return_value.get_destination.return_value = None
+
+            with pytest.raises(MCPServerNotFoundError):
+                _fetch_auth_token("dest", "tenant")
+
+            assert mock_client.return_value.get_destination.call_count == 1
+            mock_sleep.assert_not_called()
+
+    def test_backoff_delays_are_exponential(self):
+        """Delays follow base * 2^attempt: 1.0s then 2.0s."""
+        err = Exception("transient")
+        dest = self._make_dest()
+        with patch(
+            "sap_cloud_sdk.agentgateway._lob.create_destination_client"
+        ) as mock_client, patch("sap_cloud_sdk.agentgateway._lob.time.sleep") as mock_sleep:
+            mock_client.return_value.get_destination.side_effect = [err, err, dest]
+
+            _fetch_auth_token("dest", "tenant")
+
+            delays = [call.args[0] for call in mock_sleep.call_args_list]
+            assert delays == [1.0, 2.0]
+
+
 # ============================================================
 # Test: list_mcp_fragments
 # ============================================================