diff --git a/Cargo.lock b/Cargo.lock index 842eaf179d89a..be751ad8cdf82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7348,6 +7348,7 @@ dependencies = [ "tokio-metrics", "tokio-postgres", "tokio-stream", + "tokio-tungstenite", "tower 0.5.3", "tower-http 0.7.1", "tower-sessions", diff --git a/Cargo.toml b/Cargo.toml index 42b3e0badd57d..9026c32fd5e59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -311,7 +311,7 @@ aws-smithy-mocks = "0.3.0" aws-smithy-runtime-api = "1.18.0" aws-smithy-types = { version = "1.8.1", features = ["byte-stream-poll-next"] } aws-types = "1.6.0" -axum = { version = "0.8.9", features = ["ws"] } +axum = { version = "0.8.9", features = ["http2", "ws"] } axum-extra = { version = "0.12.5", features = ["typed-header"] } axum-server = { version = "0.9.0", features = ["tls-rustls"] } azure_core = { version = "1.1.0", default-features = false, features = ["hmac_rust", "reqwest", "tokio"] } @@ -396,12 +396,12 @@ http = "1.5.0" http-body-util = "0.1.3" httparse = "1.8.0" humantime = "2.4.0" -hyper = { version = "1.11.1", features = ["http1", "server"] } +hyper = { version = "1.11.1", features = ["client", "http1", "http2", "server"] } hyper-openssl = { version = "0.10.2", features = ["client-legacy", "tokio"] } # Backs the LaunchDarkly HTTPS transport, see `mz_dyncfg_launchdarkly::https_transport`. hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "native-tokio", "http1", "http2", "tls12"] } hyper-tls = "0.6.0" -hyper-util = "0.1.20" +hyper-util = { version = "0.1.20", features = ["server-auto", "tokio"] } iceberg = "0.10.1" iceberg-catalog-rest = "0.10.1" iceberg-storage-opendal = { version = "0.10.1", default-features = false, features = ["opendal-s3", "opendal-gcs", "opendal-azdls"] } @@ -570,6 +570,7 @@ tokio-openssl = "0.6.5" tokio-postgres = "0.7.15" tokio-stream = "0.1.18" tokio-test = "0.4.6" +tokio-tungstenite = "0.29.0" tokio-util = "0.7.18" toml = "1.1.6" toml_edit = { version = "0.25.15", features = ["serde"] } diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 5d82203ce05c0..aec9d9b321fa2 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -894,6 +894,7 @@ def get_default_system_parameters( "balancerd_connection_high_watermark", "balancerd_connection_low_watermark", "balancerd_pre_resolved_timeout", + "balancerd_https_enable_http2_alpn", "balancerd_log_filter", "balancerd_opentelemetry_filter", "balancerd_log_filter_defaults", diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 84f175b7d72e7..555ad1f22d883 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -3394,6 +3394,7 @@ def __init__( "balancerd_connection_high_watermark", "balancerd_connection_low_watermark", "balancerd_pre_resolved_timeout", + "balancerd_https_enable_http2_alpn", "balancerd_log_filter", "balancerd_opentelemetry_filter", "balancerd_log_filter_defaults", diff --git a/src/balancerd/src/dyncfgs.rs b/src/balancerd/src/dyncfgs.rs index d7176989bdcb7..28db9b7ba62cb 100644 --- a/src/balancerd/src/dyncfgs.rs +++ b/src/balancerd/src/dyncfgs.rs @@ -90,6 +90,27 @@ pub const PRE_RESOLVED_TIMEOUT: Config = Config::new( ParameterScope::Environment, ); +/// Whether to advertise HTTP/2 via ALPN on the HTTPS listener. +/// +/// balancerd is a byte proxy: it terminates TLS and forwards the decrypted +/// stream to environmentd. ALPN is answered during the client handshake, +/// before balancerd has connected upstream, so it cannot discover whether +/// environmentd speaks HTTP/2 in time. Enabling this while environmentd is +/// still HTTP/1.1-only makes clients negotiate h2 and send frames environmentd +/// rejects with "invalid HTTP version parsed (found HTTP2 preface)". Enable +/// only after every environmentd instance supports HTTP/2. +/// +/// NOTE: read once when the TLS context is built at startup, so a change only +/// takes effect after balancerd restarts. +pub const HTTPS_ENABLE_HTTP2_ALPN: Config = Config::new( + "balancerd_https_enable_http2_alpn", + false, + "Whether to advertise HTTP/2 via ALPN on the HTTPS listener. \ + Enable only after all environmentd instances support HTTP/2. \ + Takes effect on balancerd restart.", + ParameterScope::Environment, +); + /// Sets the filter to apply to stderr logging. pub const LOGGING_FILTER: Config<&str> = Config::new( "balancerd_log_filter", @@ -154,6 +175,7 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&CONNECTION_HIGH_WATERMARK) .add(&CONNECTION_LOW_WATERMARK) .add(&PRE_RESOLVED_TIMEOUT) + .add(&HTTPS_ENABLE_HTTP2_ALPN) .add(&LOGGING_FILTER) .add(&OPENTELEMETRY_FILTER) .add(&LOGGING_FILTER_DEFAULTS) @@ -200,6 +222,11 @@ pub(crate) fn set_defaults( PRE_RESOLVED_TIMEOUT.name(), mz_dyncfg::ConfigVal::Duration(humantime::parse_duration(v)?), ) + } else if k.as_str() == HTTPS_ENABLE_HTTP2_ALPN.name() { + config_updates.add_dynamic( + HTTPS_ENABLE_HTTP2_ALPN.name(), + mz_dyncfg::ConfigVal::Bool(bool::from_str(v)?), + ) } else { return Err(anyhow!("Invalid default config value {k}")); } diff --git a/src/balancerd/src/lib.rs b/src/balancerd/src/lib.rs index a6e59ceb50a68..ea3519ddcc0c2 100644 --- a/src/balancerd/src/lib.rs +++ b/src/balancerd/src/lib.rs @@ -30,7 +30,6 @@ use std::time::{Duration, Instant}; use anyhow::Context; use axum::{Router, routing}; use bytes::BytesMut; -use futures::TryFutureExt; use futures::stream::BoxStream; use hickory_resolver::config::LookupIpStrategy; use hickory_resolver::lookup_ip::LookupIp; @@ -40,7 +39,7 @@ use hickory_resolver::proto::rr::{Name, RData, RecordType}; use hickory_resolver::system_conf::read_system_conf; use hickory_resolver::{Resolver, TokioResolver}; use hyper::StatusCode; -use hyper_util::rt::TokioIo; +use hyper_util::rt::{TokioExecutor, TokioIo}; use launchdarkly_server_sdk as ld; use mz_build_info::{BuildInfo, build_info}; use mz_dyncfg::ConfigSet; @@ -84,9 +83,9 @@ use uuid::Uuid; use crate::codec::{BackendMessage, FramedConn}; use crate::dyncfgs::{ - CONNECTION_HIGH_WATERMARK, CONNECTION_LOW_WATERMARK, INJECT_PROXY_PROTOCOL_HEADER_HTTP, - MAX_CONNECTIONS, PRE_RESOLVED_TIMEOUT, SIGTERM_CONNECTION_WAIT, SIGTERM_LISTEN_WAIT, - has_tracing_config_update, tracing_config, + CONNECTION_HIGH_WATERMARK, CONNECTION_LOW_WATERMARK, HTTPS_ENABLE_HTTP2_ALPN, + INJECT_PROXY_PROTOCOL_HEADER_HTTP, MAX_CONNECTIONS, PRE_RESOLVED_TIMEOUT, + SIGTERM_CONNECTION_WAIT, SIGTERM_LISTEN_WAIT, has_tracing_config_update, tracing_config, }; /// Balancer build information. @@ -310,7 +309,8 @@ impl BalancerService { pub async fn serve(self) -> Result<(), anyhow::Error> { let (pgwire_tls, https_tls) = match &self.cfg.tls { Some(tls) => { - let context = tls.reloading_context(self.cfg.reload_certs)?; + let enable_http2_alpn = HTTPS_ENABLE_HTTP2_ALPN.get(&self.configs); + let context = tls.reloading_context(self.cfg.reload_certs, enable_http2_alpn)?; ( Some(ReloadingTlsConfig { context: context.clone(), @@ -509,8 +509,11 @@ impl mz_server_core::Server for InternalHttpServer { let conn = TokioIo::new(conn); Box::pin(async { - let http = hyper::server::conn::http1::Builder::new(); - http.serve_connection(conn, service).err_into().await + // Serve HTTP/1.1 or HTTP/2 (h2c via preface sniffing). + let http = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new()); + http.serve_connection(conn, service) + .await + .map_err(|e| anyhow::anyhow!(e)) }) } } @@ -1760,18 +1763,26 @@ async fn connect_upstream(addr: SocketAddr, metrics: &ServerMetrics) -> io::Resu /// Writes an HTTP error response to a client and closes the connection. /// /// The proxied connections are raw TCP streams, so the HTTP framing has to be written by hand. -/// Errors are ignored: the connection is going away regardless. -async fn send_http_error(client_stream: &mut Box, status: &str, body: &str) { - let response = format!( - "HTTP/1.1 {status}\r\n\ - Content-Type: text/plain\r\n\ - Content-Length: {}\r\n\ - Connection: close\r\n\ - \r\n\ - {body}", - body.len(), - ); - let _ = client_stream.write_all(response.as_bytes()).await; +/// Errors are ignored: the connection is going away regardless. A client that negotiated HTTP/2 +/// via ALPN cannot parse an HTTP/1.1 response, so for it the connection is only closed. +async fn send_http_error( + client_stream: &mut Box, + client_h2: bool, + status: &str, + body: &str, +) { + if !client_h2 { + let response = format!( + "HTTP/1.1 {status}\r\n\ + Content-Type: text/plain\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {body}", + body.len(), + ); + let _ = client_stream.write_all(response.as_bytes()).await; + } let _ = client_stream.shutdown().await; } @@ -1788,6 +1799,7 @@ struct HttpsBalancer { impl HttpsBalancer { /// Negotiates TLS and resolves a backend using the client's SNI hostname. + /// Also reports whether the client negotiated HTTP/2 via ALPN. async fn pre_resolve( conn: Connection, tls_context: Option, @@ -1796,35 +1808,37 @@ impl HttpsBalancer { port: u16, metrics: &ServerMetrics, fault: &ConnectionFault, - ) -> Result<(Box, ResolvedAddr, SocketAddr), anyhow::Error> { + ) -> Result<(Box, ResolvedAddr, SocketAddr, bool), anyhow::Error> { let peer_addr = conn .peer_addr() .context("fetching peer addr") .inspect_err(|_| fault.server())?; - let (client_stream, servername): (Box, Option) = match tls_context - { - Some(tls_context) => { - let mut ssl_stream = Ssl::new(&tls_context.get()) - .and_then(|ssl| SslStream::new(ssl, conn)) - .inspect_err(|_| fault.server())?; - if let Err(e) = Pin::new(&mut ssl_stream).accept().await { - metrics.tls_handshake_failures().inc(); - let _ = ssl_stream.get_mut().shutdown().await; - return Err(e.into()); + let (client_stream, servername, client_h2): (Box, Option, bool) = + match tls_context { + Some(tls_context) => { + let mut ssl_stream = Ssl::new(&tls_context.get()) + .and_then(|ssl| SslStream::new(ssl, conn)) + .inspect_err(|_| fault.server())?; + if let Err(e) = Pin::new(&mut ssl_stream).accept().await { + metrics.tls_handshake_failures().inc(); + let _ = ssl_stream.get_mut().shutdown().await; + return Err(e.into()); + } + let servername: Option = + ssl_stream.ssl().servername(NameType::HOST_NAME).map(|sn| { + match sn.split_once('.') { + Some((left, _right)) => left, + None => sn, + } + .into() + }); + debug!("Found sni servername: {servername:?} (https)"); + let client_h2 = + ssl_stream.ssl().selected_alpn_protocol() == Some(b"h2".as_slice()); + (Box::new(ssl_stream), servername, client_h2) } - let servername: Option = - ssl_stream.ssl().servername(NameType::HOST_NAME).map(|sn| { - match sn.split_once('.') { - Some((left, _right)) => left, - None => sn, - } - .into() - }); - debug!("Found sni servername: {servername:?} (https)"); - (Box::new(ssl_stream), servername) - } - _ => (Box::new(conn), None), - }; + _ => (Box::new(conn), None, false), + }; let resolved = fault .server_work(Self::resolve( resolver, @@ -1838,7 +1852,7 @@ impl HttpsBalancer { fault.server(); } })?; - Ok((client_stream, resolved, peer_addr)) + Ok((client_stream, resolved, peer_addr, client_h2)) } async fn resolve( @@ -1942,21 +1956,22 @@ impl mz_server_core::Server for HttpsBalancer { }; let pre_resolved = inner_metrics.pre_resolved_connections(); - let (mut client_stream, resolved, peer_addr) = under_pre_resolved_timeout( - pre_resolved_timeout, - &inner_metrics, - fault_ref, - Self::pre_resolve( - conn, - tls_context, - &resolver, - &resolve_template, - port, + let (mut client_stream, resolved, peer_addr, client_h2) = + under_pre_resolved_timeout( + pre_resolved_timeout, &inner_metrics, fault_ref, - ), - ) - .await?; + Self::pre_resolve( + conn, + tls_context, + &resolver, + &resolve_template, + port, + &inner_metrics, + fault_ref, + ), + ) + .await?; drop(pre_resolved); // See the matching log line in the pgwire handler. The tenant always comes from // SNI here, before any authentication. @@ -1980,6 +1995,7 @@ impl mz_server_core::Server for HttpsBalancer { fault_ref.server(); send_http_error( &mut client_stream, + client_h2, "502 Bad Gateway", "upstream server not available", ) diff --git a/src/balancerd/tests/server.rs b/src/balancerd/tests/server.rs index 13fb7e6fab9f3..7be769619a3e0 100644 --- a/src/balancerd/tests/server.rs +++ b/src/balancerd/tests/server.rs @@ -183,6 +183,8 @@ async fn test_balancer() { .add_root_certificate(ca_cert) // No pool so that connections are never re-used which can use old ssl certs. .pool_max_idle_per_host(0) + // HTTP/2 would multiplex requests over one connection, defeating the above. + .http1_only() .tls_info(true) .build() .unwrap(); @@ -210,7 +212,10 @@ async fn test_balancer() { None, None, TracingHandle::disabled(), - vec![], + vec![( + "balancerd_https_enable_http2_alpn".to_string(), + "true".to_string(), + )], ); let balancer_server = BalancerService::new(balancer_cfg).await.unwrap(); let balancer_pgwire_listen = balancer_server.pgwire.0.local_addr(); @@ -312,6 +317,42 @@ async fn test_balancer() { let resp_x509 = X509::from_der(tlsinfo.peer_certificate().unwrap()).unwrap(); let server_x509 = X509::from_pem(&std::fs::read(&server_cert).unwrap()).unwrap(); assert_eq!(resp_x509, server_x509); + assert_eq!(resp.version(), reqwest::Version::HTTP_11); + assert_contains!(resp.text().await.unwrap(), "12234"); + + // With `balancerd_https_enable_http2_alpn` set, balancerd offers h2 to + // clients that ask for it. + assert_eq!( + alpn_selected(balancer_https_listen, b"\x02h2\x08http/1.1") + .await + .as_deref(), + Some(&b"h2"[..]) + ); + assert_eq!( + alpn_selected(balancer_https_listen, b"\x08http/1.1") + .await + .as_deref(), + Some(&b"http/1.1"[..]) + ); + + // A client that offers h2 (reqwest's default) is proxied to environmentd + // over HTTP/2 end to end. + let h2_client = reqwest::Client::builder() + .add_root_certificate( + reqwest::Certificate::from_pem(&ca.cert.to_pem().unwrap()).unwrap(), + ) + .pool_max_idle_per_host(0) + .build() + .unwrap(); + let resp = h2_client + .post(&https_url) + .header("Content-Type", "application/json") + .basic_auth(frontegg_user, Some(&frontegg_password)) + .body(body) + .send() + .await + .unwrap(); + assert_eq!(resp.version(), reqwest::Version::HTTP_2); assert_contains!(resp.text().await.unwrap(), "12234"); // Generate new certs. Install only the key, reload, and make sure the old cert is still in @@ -430,6 +471,16 @@ async fn test_balancer() { }) .await .unwrap(); + + // The internal HTTP server serves h2c (HTTP/2 with prior knowledge) + // alongside HTTP/1.1. + let h2c_client = reqwest::Client::builder() + .http2_prior_knowledge() + .build() + .unwrap(); + let resp = h2c_client.get(&metrics_url).send().await.unwrap(); + assert_eq!(resp.version(), reqwest::Version::HTTP_2); + assert!(resp.status().is_success()); } } @@ -1065,3 +1116,27 @@ async fn test_startup_rejection_reaches_the_client() { the client as a bare close", ); } + +/// Returns the protocol the TLS server at `addr` selects for a client offering +/// `alpn`, in OpenSSL wire format (length-prefixed protocol names). +async fn alpn_selected(addr: SocketAddr, alpn: &'static [u8]) -> Option> { + // The handshake is blocking, and the server shares this runtime. + mz_ore::task::spawn_blocking( + || "alpn_probe", + move || { + let mut connector = SslConnector::builder(SslMethod::tls()).unwrap(); + connector.set_verify(SslVerifyMode::NONE); + connector.set_alpn_protos(alpn).unwrap(); + let stream = connector + .build() + .configure() + .unwrap() + .verify_hostname(false) + .use_server_name_indication(false) + .connect("", std::net::TcpStream::connect(addr).unwrap()) + .unwrap(); + stream.ssl().selected_alpn_protocol().map(<[u8]>::to_vec) + }, + ) + .await +} diff --git a/src/environmentd/Cargo.toml b/src/environmentd/Cargo.toml index a213284650482..8fcf14a1ab07d 100644 --- a/src/environmentd/Cargo.toml +++ b/src/environmentd/Cargo.toml @@ -145,6 +145,7 @@ serde_urlencoded.workspace = true similar-asserts.workspace = true timely.workspace = true tokio-postgres = { workspace = true, features = ["with-chrono-0_4", "with-serde_json-1"] } +tokio-tungstenite.workspace = true uuid.workspace = true [build-dependencies] diff --git a/src/environmentd/src/http.rs b/src/environmentd/src/http.rs index 6c6f250029748..994ef3619f5c5 100644 --- a/src/environmentd/src/http.rs +++ b/src/environmentd/src/http.rs @@ -68,7 +68,7 @@ use axum::extract::{ConnectInfo, DefaultBodyLimit, FromRequestParts, Query, Requ use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Redirect, Response}; use axum::{Extension, Json, Router, routing}; -use futures::future::{Shared, TryFutureExt}; +use futures::future::Shared; use headers::authorization::{Authorization, Basic, Bearer}; use headers::{HeaderMapExt, HeaderName}; use http::header::{AUTHORIZATION, CONTENT_TYPE}; @@ -76,7 +76,7 @@ use http::uri::Scheme; use http::{HeaderMap, HeaderValue, Method, StatusCode, Uri}; use hyper_openssl::SslStream; use hyper_openssl::client::legacy::MaybeHttpsStream; -use hyper_util::rt::TokioIo; +use hyper_util::rt::{TokioExecutor, TokioIo}; use mz_adapter::session::{Session as AdapterSession, SessionConfig as AdapterSessionConfig}; use mz_adapter::{AdapterError, AdapterNotice, Client, SessionClient, WebhookAppenderCache}; use mz_adapter_types::dyncfgs::OIDC_GROUP_CLAIM; @@ -321,7 +321,12 @@ impl HttpServer { base_router = base_router.merge(base_group); let mut ws_router = Router::new() - .route("/api/experimental/sql", routing::get(sql::handle_sql_ws)) + // WebSockets arrive as a GET with the `Upgrade` header on + // HTTP/1.1 and as an extended CONNECT (RFC 8441) on HTTP/2. + .route( + "/api/experimental/sql", + routing::get(sql::handle_sql_ws).connect(sql::handle_sql_ws), + ) .with_state(WsState { frontegg, oidc_rx: oidc_rx.clone(), @@ -747,11 +752,14 @@ impl Server for HttpServer { .into_make_service_with_connect_info::(); let tower_svc = make_tower_svc.call(peer_addr).await.unwrap(); let hyper_svc = hyper::service::service_fn(|req| tower_svc.clone().call(req)); - let http = hyper::server::conn::http1::Builder::new(); - http.serve_connection(conn, hyper_svc) - .with_upgrades() - .err_into() + // Serves HTTP/1.1 or HTTP/2, chosen by sniffing for the HTTP/2 + // preface (ALPN is not consulted). + let mut http = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new()); + // Advertise RFC 8441 extended CONNECT for WebSockets over HTTP/2. + http.http2().enable_connect_protocol(); + http.serve_connection_with_upgrades(conn, hyper_svc) .await + .map_err(|e| anyhow::anyhow!(e)) }) } } diff --git a/src/environmentd/src/http/console.rs b/src/environmentd/src/http/console.rs index 2b4845d95a9e4..0dbfd773a83ae 100644 --- a/src/environmentd/src/http/console.rs +++ b/src/environmentd/src/http/console.rs @@ -191,6 +191,10 @@ pub(crate) async fn handle_internal_console( let host = uri.host().unwrap().to_string(); // Preserve the request, but update the URI to point upstream. *req.uri_mut() = uri; + // The client fails an HTTP/2 request on an HTTP/1 upstream connection + // with `UserUnsupportedVersion`, and the inbound version is whatever the + // browser negotiated with us. + *req.version_mut() = http::Version::HTTP_11; // If vercel sees the request being served from a different host it tries to redirect to it's own. req.headers_mut() @@ -300,11 +304,16 @@ fn is_same_origin(req: &Request) -> bool { { return site == "same-origin" || site == "none"; } - let (Some(origin), Some(host)) = (req.headers().get(ORIGIN), req.headers().get(HOST)) else { + let Some(origin) = req.headers().get(ORIGIN) else { // No `Origin` on a POST means it was not a cross-site form submission. return true; }; - let (Ok(origin), Ok(host)) = (origin.to_str(), host.to_str()) else { + // HTTP/2 requests carry `:authority` in the URI and no `Host` header. + let host = match req.headers().get(HOST) { + Some(host) => host.to_str().ok(), + None => req.uri().authority().map(|a| a.as_str()), + }; + let (Ok(origin), Some(host)) = (origin.to_str(), host) else { return false; }; origin @@ -375,3 +384,22 @@ fn preview_build_from_cookie(headers: &HeaderMap) -> Option { } None } + +#[cfg(test)] +mod tests { + use super::*; + + #[mz_ore::test] + fn test_is_same_origin_uses_http2_authority() { + let req = |origin: &str| { + Request::builder() + .method(Method::POST) + .uri("https://mz.example.com/internal-console/") + .header(ORIGIN, origin) + .body(Body::empty()) + .unwrap() + }; + assert!(is_same_origin(&req("https://mz.example.com"))); + assert!(!is_same_origin(&req("https://evil.example.com"))); + } +} diff --git a/src/environmentd/src/http/oauth_metadata.rs b/src/environmentd/src/http/oauth_metadata.rs index 3c84a211d928d..487a0046a0b96 100644 --- a/src/environmentd/src/http/oauth_metadata.rs +++ b/src/environmentd/src/http/oauth_metadata.rs @@ -401,7 +401,8 @@ pub(crate) fn metadata_url(req: &Request, http_host_name: Option<&str>) -> Optio /// Resolves the host string to embed in published absolute URLs. /// /// Prefers the operator-configured `http_host_name`; falls back to the -/// request's `Host` header. **Never consults `X-Forwarded-*`**. See the +/// request's `Host` header, then to the URI authority, which is where HTTP/2 +/// requests carry `:authority` instead of a `Host` header. **Never consults `X-Forwarded-*`**. See the /// module-level "Host derivation" notes. /// /// The returned value is parsed through [`http::uri::Authority`] before @@ -425,7 +426,8 @@ fn resolve_host(req: &Request, http_host_name: Option<&str>) -> Option { .map(str::trim) .filter(|s| !s.is_empty()) .map(str::to_string) - })?; + }) + .or_else(|| req.uri().authority().map(|a| a.as_str().to_string()))?; // Reject `user@host` explicitly. `Authority::as_str()` keeps userinfo, // so the round-trip below would pass it through, letting an attacker // poison the published URLs via a forged `Host: bobby@evil...`. @@ -536,6 +538,18 @@ mod tests { ); } + #[mz_ore::test] + fn test_resolve_host_falls_back_to_http2_authority() { + let req = Request::builder() + .uri("https://example.com:8080/.well-known/oauth-protected-resource") + .body(Body::empty()) + .unwrap(); + assert_eq!( + resolve_host(&req, None).as_deref(), + Some("example.com:8080"), + ); + } + /// No host config and no `Host` header → `None`. #[mz_ore::test] fn test_resolve_host_returns_none_when_unavailable() { diff --git a/src/environmentd/src/lib.rs b/src/environmentd/src/lib.rs index 784813c5d00cb..c47349bb9bb91 100644 --- a/src/environmentd/src/lib.rs +++ b/src/environmentd/src/lib.rs @@ -395,7 +395,10 @@ impl Listeners { // Validate TLS configuration, if present. let tls_reloading_context = match config.tls { - Some(tls_config) => Some(tls_config.reloading_context(config.tls_reload_certs)?), + Some(tls_config) => { + let enable_http2_alpn = true; + Some(tls_config.reloading_context(config.tls_reload_certs, enable_http2_alpn)?) + } None => None, }; diff --git a/src/environmentd/tests/server.rs b/src/environmentd/tests/server.rs index 9647ae6204810..01c3bf015ecfb 100644 --- a/src/environmentd/tests/server.rs +++ b/src/environmentd/tests/server.rs @@ -15,7 +15,7 @@ use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::fmt::Write; use std::io::{Read as _, Write as _}; -use std::net::{IpAddr, Ipv4Addr}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -51,7 +51,7 @@ use mz_ore::{assert_err, assert_ok, task}; use mz_pgrepr::UInt8; use mz_repr::UNKNOWN_COLUMN_NAME; use mz_sql::session::user::{ANALYTICS_USER, HTTP_DEFAULT_USER, SYSTEM_USER}; -use openssl::ssl::{SslConnectorBuilder, SslVerifyMode}; +use openssl::ssl::{SslConnector, SslConnectorBuilder, SslMethod, SslVerifyMode}; use openssl::x509::X509; use postgres::config::SslMode; use postgres_array::Array; @@ -5622,6 +5622,8 @@ async fn test_cert_reloading() { .add_root_certificate(ca_cert) // No pool so that connections are never re-used which can use old ssl certs. .pool_max_idle_per_host(0) + // HTTP/2 would multiplex requests over one connection, defeating the above. + .http1_only() .tls_info(true) .build() .unwrap(); @@ -8767,3 +8769,301 @@ fn test_cluster_labeled_metrics_are_removed_at_zero_replicas() { ); } } + +// Test that the HTTP server negotiates HTTP/2 over TLS via ALPN and that +// HTTP/1.1-only clients can still connect. +#[mz_ore::test(tokio::test(flavor = "multi_thread", worker_threads = 1))] +#[cfg_attr(miri, ignore)] // too slow +async fn test_http2_tls() { + let ca = Ca::new_root("test ca").unwrap(); + let (server_cert, server_key) = ca + .request_cert("server", vec![IpAddr::V4(Ipv4Addr::LOCALHOST)]) + .unwrap(); + let server = test_util::TestHarness::default() + .with_tls(server_cert, server_key) + .start() + .await; + + let addr = server.http_local_addr(); + assert_eq!( + alpn_selected(addr, b"\x02h2\x08http/1.1").await.as_deref(), + Some(&b"h2"[..]) + ); + assert_eq!( + alpn_selected(addr, b"\x08http/1.1").await.as_deref(), + Some(&b"http/1.1"[..]) + ); + // libpq 17+ offers `postgresql`; the handshake must still succeed. + assert_eq!(alpn_selected(addr, b"\x0apostgresql").await, None); + + let https_url = Url::parse(&format!("https://{addr}/api/sql")).unwrap(); + let json: serde_json::Value = serde_json::from_str(r#"{ "query": "SELECT 42;" }"#).unwrap(); + let ca_cert = reqwest::Certificate::from_pem(&ca.cert.to_pem().unwrap()).unwrap(); + + // reqwest offers h2 via ALPN by default, so it is served over HTTP/2. + let client = reqwest::Client::builder() + .add_root_certificate(ca_cert.clone()) + .build() + .unwrap(); + let response = client + .post(https_url.clone()) + .json(&json) + .send() + .await + .unwrap(); + assert_eq!(response.version(), reqwest::Version::HTTP_2); + assert!(response.status().is_success()); + assert_contains!(response.text().await.unwrap(), "42"); + + // HTTP/1.1-only clients are still served. + let client = reqwest::Client::builder() + .add_root_certificate(ca_cert) + .http1_only() + .build() + .unwrap(); + let response = client.post(https_url).json(&json).send().await.unwrap(); + assert_eq!(response.version(), reqwest::Version::HTTP_11); + assert!(response.status().is_success()); + assert_contains!(response.text().await.unwrap(), "42"); +} + +/// Returns the protocol the TLS server at `addr` selects for a client offering +/// `alpn`, in OpenSSL wire format (length-prefixed protocol names). +async fn alpn_selected(addr: SocketAddr, alpn: &'static [u8]) -> Option> { + // The handshake is blocking, and the server shares this runtime. + mz_ore::task::spawn_blocking( + || "alpn_probe", + move || { + let mut connector = SslConnector::builder(SslMethod::tls()).unwrap(); + connector.set_verify(SslVerifyMode::NONE); + connector.set_alpn_protos(alpn).unwrap(); + let stream = connector + .build() + .configure() + .unwrap() + .verify_hostname(false) + .use_server_name_indication(false) + .connect("", std::net::TcpStream::connect(addr).unwrap()) + .unwrap(); + stream.ssl().selected_alpn_protocol().map(<[u8]>::to_vec) + }, + ) + .await +} + +// Test that plaintext listeners serve both HTTP/1.1 and HTTP/2 (h2c via +// prior knowledge). +#[mz_ore::test(tokio::test(flavor = "multi_thread", worker_threads = 1))] +#[cfg_attr(miri, ignore)] // too slow +async fn test_http2_cleartext() { + let server = test_util::TestHarness::default().start().await; + + let http_url = Url::parse(&format!("http://{}/api/sql", server.http_local_addr())).unwrap(); + let json: serde_json::Value = serde_json::from_str(r#"{ "query": "SELECT 42;" }"#).unwrap(); + + // HTTP/2 with prior knowledge (h2c). + let client = reqwest::Client::builder() + .http2_prior_knowledge() + .build() + .unwrap(); + let response = client + .post(http_url.clone()) + .json(&json) + .send() + .await + .unwrap(); + assert_eq!(response.version(), reqwest::Version::HTTP_2); + assert!(response.status().is_success()); + assert_contains!(response.text().await.unwrap(), "42"); + + // Plain HTTP/1.1 is unchanged. + let response = reqwest::Client::new() + .post(http_url) + .json(&json) + .send() + .await + .unwrap(); + assert_eq!(response.version(), reqwest::Version::HTTP_11); + assert!(response.status().is_success()); + assert_contains!(response.text().await.unwrap(), "42"); +} + +// Test WebSockets over HTTP/2 (RFC 8441 extended CONNECT): authentication +// works over an HTTP/2 stream, bad credentials are rejected, and the HTTP/1.1 +// WebSocket upgrade (the downgrade path) keeps working against the same +// server. +#[mz_ore::test(tokio::test(flavor = "multi_thread", worker_threads = 1))] +#[cfg_attr(miri, ignore)] // too slow +async fn test_http2_websocket_auth() { + use futures::{SinkExt, StreamExt}; + use mz_auth::password::Password; + use tokio_tungstenite::tungstenite::Message as WsMessage; + + type WsStream = + tokio_tungstenite::WebSocketStream>; + + // Reads messages until the initial ReadyForQuery, mirroring + // `test_util::auth_with_ws_impl` for async streams. + async fn read_until_ready(ws: &mut WsStream) -> Result, anyhow::Error> { + let mut msgs = Vec::new(); + loop { + let msg = ws + .next() + .await + .ok_or_else(|| anyhow::anyhow!("ws stream ended"))??; + match msg { + WsMessage::Text(text) => { + let msg: WebSocketResponse = serde_json::from_str(&text).unwrap(); + match msg { + WebSocketResponse::ReadyForQuery(_) => return Ok(msgs), + msg => msgs.push(msg), + } + } + WsMessage::Ping(_) => continue, + WsMessage::Close(frame) => anyhow::bail!("ws closed: {frame:?}"), + other => panic!("unexpected message: {other:?}"), + } + } + } + + let server = test_util::TestHarness::default() + .with_system_parameter_default("enable_password_auth".to_string(), "true".to_string()) + .with_password_auth(Password("mz_system_password".to_owned())) + .start() + .await; + + // Opens a WebSocket over an HTTP/2 stream via extended CONNECT. + let connect_ws_http2 = || async { + let tcp = tokio::net::TcpStream::connect(server.http_local_addr()) + .await + .unwrap(); + let (mut sender, conn) = + hyper::client::conn::http2::Builder::new(hyper_util::rt::TokioExecutor::new()) + .handshake::<_, http_body_util::Empty>(hyper_util::rt::TokioIo::new( + tcp, + )) + .await + .unwrap(); + // The client can only send extended CONNECT after the server's + // SETTINGS frame advertising it has arrived, so drive the connection + // until it has been processed. + let mut conn = Box::pin(conn); + let deadline = Instant::now() + Duration::from_secs(10); + while !conn.is_extended_connect_protocol_enabled() { + assert!( + Instant::now() < deadline, + "server did not advertise RFC 8441 extended CONNECT" + ); + let _ = futures::poll!(conn.as_mut()); + tokio::task::yield_now().await; + } + task::spawn(|| "h2_ws_conn", async move { + let _ = conn.await; + }); + let req = Request::builder() + .method("CONNECT") + .extension(hyper::ext::Protocol::from_static("websocket")) + .uri("/api/experimental/sql") + .header("host", server.http_local_addr().to_string()) + .header("sec-websocket-version", "13") + .body(http_body_util::Empty::::new()) + .unwrap(); + let response = sender.send_request(req).await.unwrap(); + assert_eq!(response.version(), http::Version::HTTP_2); + assert_eq!( + response.status(), + StatusCode::OK, + "headers: {:?}", + response.headers() + ); + let upgraded = hyper::upgrade::on(response).await.unwrap(); + tokio_tungstenite::WebSocketStream::from_raw_socket( + hyper_util::rt::TokioIo::new(upgraded), + tokio_tungstenite::tungstenite::protocol::Role::Client, + None, + ) + .await + }; + + let auth_json = |password: &str| { + serde_json::to_string(&WebSocketAuth::Basic { + user: "mz_system".into(), + password: Password(password.to_owned()), + options: BTreeMap::default(), + }) + .unwrap() + }; + + // WebSocket over HTTP/2 with valid credentials: authenticates and runs a + // query. + let mut ws = connect_ws_http2().await; + ws.send(WsMessage::Text(auth_json("mz_system_password").into())) + .await + .unwrap(); + read_until_ready(&mut ws).await.unwrap(); + ws.send(WsMessage::Text(r#"{"query": "SELECT 'row42'"}"#.into())) + .await + .unwrap(); + let mut saw_row = false; + loop { + match ws.next().await.unwrap().unwrap() { + WsMessage::Text(text) => match serde_json::from_str(&text).unwrap() { + WebSocketResponse::Row(row) => { + assert_eq!(row, vec![serde_json::json!("row42")]); + saw_row = true; + } + WebSocketResponse::ReadyForQuery(_) => break, + _ => {} + }, + WsMessage::Ping(_) => continue, + other => panic!("unexpected message: {other:?}"), + } + } + assert!(saw_row, "expected a row from SELECT over HTTP/2 WebSocket"); + + // WebSocket over HTTP/2 with bad credentials: the server closes the + // socket without revealing detail. + let mut ws = connect_ws_http2().await; + ws.send(WsMessage::Text(auth_json("wrong_password").into())) + .await + .unwrap(); + let err = read_until_ready(&mut ws).await.unwrap_err(); + assert_contains!(err.to_string(), "unauthorized"); + + // Downgrade: the same server still serves HTTP/1.1 WebSocket upgrades, + // with the same authentication behavior. + let (mut ws, _resp) = tungstenite::connect(server.ws_addr()).unwrap(); + test_util::auth_with_ws_impl( + &mut ws, + Message::Text(auth_json("mz_system_password").into()), + ) + .unwrap(); + ws.send(Message::Text(r#"{"query": "SELECT 'row42'"}"#.into())) + .unwrap(); + let mut saw_row = false; + loop { + match ws.read().unwrap() { + Message::Text(text) => match serde_json::from_str(&text).unwrap() { + WebSocketResponse::Row(row) => { + assert_eq!(row, vec![serde_json::json!("row42")]); + saw_row = true; + } + WebSocketResponse::ReadyForQuery(_) => break, + _ => {} + }, + Message::Ping(_) => continue, + other => panic!("unexpected message: {other:?}"), + } + } + assert!( + saw_row, + "expected a row from SELECT over HTTP/1.1 WebSocket" + ); + + // HTTP/1.1 with bad credentials is also still rejected. + let (mut ws, _resp) = tungstenite::connect(server.ws_addr()).unwrap(); + let err = + test_util::auth_with_ws_impl(&mut ws, Message::Text(auth_json("wrong_password").into())) + .unwrap_err(); + assert_contains!(format!("{err:?}"), "unauthorized"); +} diff --git a/src/server-core/src/lib.rs b/src/server-core/src/lib.rs index 2b52653e8cb42..8c612db2963bd 100644 --- a/src/server-core/src/lib.rs +++ b/src/server-core/src/lib.rs @@ -29,7 +29,7 @@ use mz_ore::error::ErrorExt; use mz_ore::netio::AsyncReady; use mz_ore::option::OptionExt; use mz_ore::task::JoinSetExt; -use openssl::ssl::{SslAcceptor, SslContext, SslFiletype, SslMethod}; +use openssl::ssl::{AlpnError, SslAcceptor, SslContext, SslFiletype, SslMethod, select_next_proto}; use proxy_header::{ParseConfig, ProxiedAddress, ProxyHeader}; use schemars::JsonSchema; use scopeguard::ScopeGuard; @@ -503,7 +503,10 @@ pub struct TlsCertConfig { impl TlsCertConfig { /// Returns the SSL context to use in TlsConfigs. - pub fn load_context(&self) -> Result { + /// + /// ALPN offers `h2, http/1.1` when `enable_http2_alpn` is set and only + /// `http/1.1` otherwise. + pub fn load_context(&self, enable_http2_alpn: bool) -> Result { // Mozilla publishes three presets: old, intermediate, and modern. They // recommend the intermediate preset for general purpose servers, which // is what we use, as it is compatible with nearly every client released @@ -511,6 +514,17 @@ impl TlsCertConfig { // ciphers. We once tried to use the modern preset, but it was // incompatible with Fivetran, and presumably other JDBC-based tools. let mut builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls())?; + // This context is shared with pgwire listeners. Clients offering no + // ALPN, or no HTTP protocol (libpq 17+ offers `postgresql`), get + // `NOACK`, which completes the handshake without ALPN. + let alpn_list: &[u8] = if enable_http2_alpn { + b"\x02h2\x08http/1.1" + } else { + b"\x08http/1.1" + }; + builder.set_alpn_select_callback(move |_ssl, client_protos| { + select_next_proto(alpn_list, client_protos).ok_or(AlpnError::NOACK) + }); builder.set_certificate_chain_file(&self.cert)?; builder.set_private_key_file(&self.key, SslFiletype::PEM)?; Ok(builder.build().into_context()) @@ -524,13 +538,14 @@ impl TlsCertConfig { pub fn reloading_context( &self, mut ticker: ReloadTrigger, + enable_http2_alpn: bool, ) -> Result { - let context = Arc::new(RwLock::new(self.load_context()?)); + let context = Arc::new(RwLock::new(self.load_context(enable_http2_alpn)?)); let updater_context = Arc::clone(&context); let config = self.clone(); mz_ore::task::spawn(|| "TlsCertConfig reloading_context", async move { while let Some(chan) = ticker.next().await { - let result = match config.load_context() { + let result = match config.load_context(enable_http2_alpn) { Ok(ctx) => { *updater_context.write().expect("poisoned") = ctx; Ok(()) diff --git a/test/balancerd/mzcompose.py b/test/balancerd/mzcompose.py index 13d1b5d833590..e5970b98c84b4 100644 --- a/test/balancerd/mzcompose.py +++ b/test/balancerd/mzcompose.py @@ -163,7 +163,7 @@ def app_password(email: str) -> str: "--tls-cert=/secrets/balancerd.crt", "--internal-tls", "--tls-mode=require", - "--default-config=balancerd_inject_proxy_protocol_header_http=true", + "--default-config=balancerd_inject_proxy_protocol_header_http=true,balancerd_https_enable_http2_alpn=true", # Nonsensical but we don't need cancellations here "--cancellation-resolver-dir=/secrets/", ], @@ -346,7 +346,7 @@ def workflow_plaintext(c: Composition) -> None: "--https-resolver-template=materialized:6876", "--tls-key=/secrets/balancerd.key", "--tls-cert=/secrets/balancerd.crt", - "--default-config=balancerd_inject_proxy_protocol_header_http=true", + "--default-config=balancerd_inject_proxy_protocol_header_http=true,balancerd_https_enable_http2_alpn=true", # Nonsensical but we don't need cancellations here "--cancellation-resolver-dir=/secrets/", ], @@ -721,7 +721,7 @@ def workflow_user(c: Composition) -> None: "--tls-cert=/secrets/balancerd.crt", "--internal-tls", "--tls-mode=require", - "--default-config=balancerd_inject_proxy_protocol_header_http=true", + "--default-config=balancerd_inject_proxy_protocol_header_http=true,balancerd_https_enable_http2_alpn=true", # Nonsensical but we don't need cancellations here "--cancellation-resolver-dir=/secrets/", ], @@ -907,7 +907,7 @@ def workflow_pgwire_with_sni_no_cname(c: Composition) -> None: "--tls-cert=/secrets/balancerd.crt", "--internal-tls", "--tls-mode=require", - "--default-config=balancerd_inject_proxy_protocol_header_http=true", + "--default-config=balancerd_inject_proxy_protocol_header_http=true,balancerd_https_enable_http2_alpn=true", # Nonsensical but we don't need cancellations here "--cancellation-resolver-dir=/secrets/", ],