Merge pull request #493 from JoaoFula/implement-OLS-3749

OLS-3749 - add troubleshooting tests to product e2e

Author: openshift-ci[bot]

.tekton/integration-tests/pipelines/agentic-product-e2e-pipeline.yaml

@@ -181,10 +181,14 @@ spec:
                 value: "/var/run/credentials/vertex/token"
               - name: VERTEX_PROJECT_ID
                 value: "openshift-lightspeed-tests"
+              - name: VERTEX_REGION
+                value: "global"
               - name: OPENAI_PROVIDER_KEY_PATH
                 value: "/var/run/credentials/openai/token"
               - name: ARTIFACT_DIR
                 value: "/workspace/konflux-artifacts"
+              - name: E2E_SCENARIO_TAGS
+                value: "alert"
             image: registry.redhat.io/openshift4/ose-cli:latest
             script: |
               set -euo pipefail

.tekton/integration-tests/scripts/install-operator.sh

@@ -60,23 +60,6 @@ fi
 echo "Deploying operator..."
 make deploy IMG="${IMG}" OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE}" SANDBOX_MODE="${SANDBOX_MODE}"
 
-# Grant cluster-admin to operator SA (same as quickstart — covers escalation + SCC).
-echo "Granting cluster-admin to operator SA..."
-oc apply -f - <<EOF
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: lightspeed-agentic-operator-admin
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: controller-manager
-  namespace: ${OPERATOR_NAMESPACE}
-EOF
-
 # Grant cluster-reader to agent SA (required by execution RBAC discovery).
 echo "Granting cluster-reader to agent SA..."
 oc apply -f - <<EOF

README.md

@@ -139,6 +139,43 @@ make api-lint       # Kube API linter on api/ (golangci-lint custom + plugin; se
 
 **`make test-e2e`** runs **`go test -tags=e2e ./test/e2e/...`** against a live cluster with the operator running. Prerequisites: **`make run TEMPLATE_NAME=lightspeed-agent-mock`** (or deployed operator with **`--template-name=lightspeed-agent-mock`**) and mock agent SandboxTemplate applied (**`kubectl apply -k test/agent/sandboxtemplate`**). See `test/e2e/` package doc for details.
 
+### Product E2E (real providers)
+
+Product E2E runs the troubleshooting scenarios against a real LLM provider. It deploys (or reuses) the operator, clones `rhobs/troubleshooting-scenarios`, and runs the `product_e2e`-tagged tests. For example, to run Claude scenarios:
+
+```bash
+IMG=quay.io/.../lightspeed-agentic-operator@sha256:<digest> \
+VERTEX_PROVIDER_KEY_PATH=/path/to/service-account.json \
+VERTEX_PROJECT_ID=my-gcp-project \
+ARTIFACT_DIR="$PWD/artifacts" \
+bash scripts/e2e-cluster.sh claude
+```
+
+`claude` and `gemini` require `VERTEX_PROVIDER_KEY_PATH` and `VERTEX_PROJECT_ID`; `openai` requires `OPENAI_PROVIDER_KEY_PATH`. The runner accepts one or more providers (`claude`, `gemini`, and/or `openai`).
+
+| Variable | Purpose |
+|---|---|
+| `E2E_SCENARIO_TAGS=alert` | Run scenarios carrying the `alert` tag instead of the default `core` tag. Multiple tags are an **AND** filter: `core,alert` requires both. |
+| `E2E_SKIP_SCENARIOS=pending_pvc_alert` | Skip comma-separated scenario directory names. |
+| `E2E_SCENARIO_TIMEOUT=20m` | Per-scenario deadline (default: `20m`). |
+| `E2E_OTEL_ENABLED=false` | Disable the persistent OTEL/Postgres collector used for product-E2E artifacts. It is enabled by default. |
+| `E2E_OTEL_IMAGE=quay.io/.../collector:tag` | Override the OTEL collector image. |
+| `ARTIFACT_DIR=/path/to/artifacts` | Directory where test and diagnostic artifacts are written. |
+
+With `ARTIFACT_DIR` set, artifacts are arranged per provider:
+
+```text
+artifacts/<provider>/
+├── product-e2e-output.log       # Go test output
+├── operator-logs.txt             # controller-manager logs
+├── pods.yaml                     # remaining operator-namespace pods
+├── podlogs/                      # best-effort direct sandbox-pod log backup
+└── runs/<run-name>/
+    └── otel-sandbox.json         # persisted OTEL collector records for the run
+```
+
+The runner configures the collector with its Postgres backend, because the default `nop` pipeline drops records and does not expose the admin API used to export per-run records. Direct pod-log capture is best effort: sandbox pods may be removed immediately after completion, so persisted OTEL records are the primary diagnostic source.
+
 For noisy debugging: **`go test ./controller/agenticrun/... -v`**, **`go test ./api/... -v`**, **`go test ./cli/... -v`**.
 
 ### API lint (Kube API linter)

go.mod

@@ -15,6 +15,7 @@ require (
 	go.uber.org/zap v1.28.0
 	gomodules.xyz/jsonpatch/v2 v2.5.0
 	google.golang.org/grpc v1.83.2
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.35.3
 	k8s.io/apiextensions-apiserver v0.35.3
 	k8s.io/apimachinery v0.35.3
@@ -65,15 +66,18 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/gnostic-models v0.7.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/moby/spdystream v0.5.1 // indirect
 	github.com/moby/term v0.5.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.24.1 // indirect

go.sum

@@ -2,6 +2,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -87,6 +89,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
@@ -97,10 +101,18 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
 github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=
+github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
+github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y=
+github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -113,6 +125,8 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
 github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
 github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
@@ -130,6 +144,8 @@ github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3
 github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
 github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
 github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
@@ -218,11 +234,15 @@ google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsok
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
 gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ=
 k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4=
 k8s.io/apiextensions-apiserver v0.35.3 h1:2fQUhEO7P17sijylbdwt0nBdXP0TvHrHj0KeqHD8FiU=

hack/quickstart/deploy-otel.sh

@@ -7,9 +7,10 @@
 # With --postgres, deploys a Postgres instance and routes agentic logs
 # to it for the console audit UI.
 #
-# The Service uses service-ca for TLS. The generated cert secret
-# (lightspeed-otel-collector-cert) is referenced in the agentic
-# configuration ConfigMap as otel-ca-secret.
+# The Service uses service-ca for TLS. Its generated serving-cert secret
+# (lightspeed-otel-collector-cert) is mounted by the collector only. Clients
+# must instead trust the service CA via lightspeed-agentic-otel-ca (key:
+# otel-ca.crt), which deploy-configmap.sh references as otel-ca-secret.
 #
 # Usage:
 #   bash hack/quickstart/deploy-otel.sh

scripts/e2e-cluster.sh

@@ -22,6 +22,9 @@
 #   E2E_MODEL                — override default model for the provider
 #   ARTIFACT_DIR             — directory for test artifacts
 #   KONFLUX_COMPONENT_NAME   — component name for SNAPSHOT parsing
+#   E2E_OTEL_ENABLED         — deploy persistent OTEL/Postgres collector for
+#                              sandbox-log artifacts (default: true)
+#   E2E_OTEL_IMAGE           — override the OTEL collector image
 
 set -euo pipefail
 
@@ -34,6 +37,19 @@ source "$SCRIPT_DIR/e2e-lib.sh"
 PROVIDERS="${*:-claude gemini openai}"
 NAMESPACE="${OPERATOR_NAMESPACE:-openshift-lightspeed}"
 export OPERATOR_NAMESPACE="$NAMESPACE"
+SCENARIOS_TMPDIR=""
+
+# shellcheck disable=SC2329 # Invoked indirectly by the EXIT trap.
+_cleanup_on_exit() {
+    log_info "Running cleanup..."
+    cleanup_e2e_otel "$NAMESPACE"
+    cleanup_operator
+    if [[ -n "${SCENARIOS_TMPDIR:-}" && -d "$SCENARIOS_TMPDIR" ]]; then
+        log_info "Removing scenarios clone: $SCENARIOS_TMPDIR"
+        rm -rf "$SCENARIOS_TMPDIR"
+    fi
+}
+trap _cleanup_on_exit EXIT
 
 log_info "=== e2e-cluster.sh ==="
 log_info "Providers: $PROVIDERS"
@@ -45,6 +61,13 @@ parse_snapshot
 cd "$REPO_ROOT"
 
 deploy_operator
+ensure_e2e_otel "$NAMESPACE"
+
+# Clone troubleshooting scenarios from rhobs for product_e2e tests.
+SCENARIOS_TMPDIR="$(mktemp -d)"
+log_info "Cloning rhobs/troubleshooting-scenarios → $SCENARIOS_TMPDIR"
+git clone --depth 1 https://github.com/rhobs/troubleshooting-scenarios.git "$SCENARIOS_TMPDIR"
+export E2E_SCENARIOS_DIR="$SCENARIOS_TMPDIR"
 
 declare -A results
 overall_rc=0
@@ -70,40 +93,38 @@ run_provider() {
 
     log_info "--- Running e2e for provider=$provider model=$model ---"
 
+    local e2e_env=(
+        E2E_PROVIDER="$provider"
+        E2E_MODEL="$model"
+        E2E_PROVIDER_KEY_PATH="$key_path"
+        E2E_POLL_TIMEOUT="${E2E_POLL_TIMEOUT:-20m}"
+        VERTEX_PROJECT_ID="${VERTEX_PROJECT_ID:-}"
+        VERTEX_REGION="${VERTEX_REGION:-global}"
+        TEST_NAMESPACE="$NAMESPACE"
+    )
+
     local test_rc=0
 
-    if [[ -n "${ARTIFACT_DIR:-}" ]]; then
-        mkdir -p "$ARTIFACT_DIR/$provider"
-        E2E_PROVIDER="$provider" \
-        E2E_MODEL="$model" \
-        E2E_PROVIDER_KEY_PATH="$key_path" \
-        E2E_POLL_TIMEOUT="${E2E_POLL_TIMEOUT:-20m}" \
-        VERTEX_PROJECT_ID="${VERTEX_PROJECT_ID:-}" \
-        VERTEX_REGION="${VERTEX_REGION:-global}" \
-        TEST_NAMESPACE="$NAMESPACE" \
-        make test-e2e 2>&1 | tee "$ARTIFACT_DIR/$provider/test-output.log" || test_rc=$?
-    else
-        E2E_PROVIDER="$provider" \
-        E2E_MODEL="$model" \
-        E2E_PROVIDER_KEY_PATH="$key_path" \
-        E2E_POLL_TIMEOUT="${E2E_POLL_TIMEOUT:-20m}" \
-        VERTEX_PROJECT_ID="${VERTEX_PROJECT_ID:-}" \
-        VERTEX_REGION="${VERTEX_REGION:-global}" \
-        TEST_NAMESPACE="$NAMESPACE" \
-        make test-e2e || test_rc=$?
+    # Product e2e (troubleshooting scenarios against real LLMs).
+    # Mock-only tests (failure_test.go etc.) run in precommit CI via `make test-e2e`.
+    if [[ -n "${E2E_SCENARIOS_DIR:-}" ]]; then
+        log_info "--- Running product e2e for provider=$provider ---"
+        if [[ -n "${ARTIFACT_DIR:-}" ]]; then
+            mkdir -p "$ARTIFACT_DIR/$provider"
+            env "${e2e_env[@]}" E2E_SCENARIOS_DIR="$E2E_SCENARIOS_DIR" \
+                go test -tags=product_e2e ./test/e2e/... -count=1 -v -timeout 120m \
+                2>&1 | tee "$ARTIFACT_DIR/$provider/product-e2e-output.log" || test_rc=$?
+        else
+            env "${e2e_env[@]}" E2E_SCENARIOS_DIR="$E2E_SCENARIOS_DIR" \
+                go test -tags=product_e2e ./test/e2e/... -count=1 -v -timeout 120m || test_rc=$?
+        fi
     fi
 
     collect_artifacts "$provider"
 
     return "$test_rc"
 }
 
-_cleanup_on_exit() {
-    log_info "Running cleanup..."
-    cleanup_operator
-}
-trap _cleanup_on_exit EXIT
-
 for provider in $PROVIDERS; do
     set +e
     run_provider "$provider"