Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ aws-smithy-mocks = "0.3.0"
aws-smithy-runtime-api = "1.18.0"
aws-smithy-types = { version = "1.8.1", features = ["byte-stream-poll-next"] }
aws-types = "1.6.0"
axum = { version = "0.8.9", features = ["ws"] }
axum = { version = "0.8.9", features = ["http2", "ws"] }
axum-extra = { version = "0.12.5", features = ["typed-header"] }
axum-server = { version = "0.9.0", features = ["tls-rustls"] }
azure_core = { version = "1.1.0", default-features = false, features = ["hmac_rust", "reqwest", "tokio"] }
Expand Down Expand Up @@ -396,12 +396,12 @@ http = "1.5.0"
http-body-util = "0.1.3"
httparse = "1.8.0"
humantime = "2.4.0"
hyper = { version = "1.11.1", features = ["http1", "server"] }
hyper = { version = "1.11.1", features = ["client", "http1", "http2", "server"] }
hyper-openssl = { version = "0.10.2", features = ["client-legacy", "tokio"] }
# Backs the LaunchDarkly HTTPS transport, see `mz_dyncfg_launchdarkly::https_transport`.
hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "native-tokio", "http1", "http2", "tls12"] }
hyper-tls = "0.6.0"
hyper-util = "0.1.20"
hyper-util = { version = "0.1.20", features = ["server-auto", "tokio"] }
iceberg = "0.10.1"
iceberg-catalog-rest = "0.10.1"
iceberg-storage-opendal = { version = "0.10.1", default-features = false, features = ["opendal-s3", "opendal-gcs", "opendal-azdls"] }
Expand Down Expand Up @@ -570,6 +570,7 @@ tokio-openssl = "0.6.5"
tokio-postgres = "0.7.15"
tokio-stream = "0.1.18"
tokio-test = "0.4.6"
tokio-tungstenite = "0.29.0"
tokio-util = "0.7.18"
toml = "1.1.6"
toml_edit = { version = "0.25.15", features = ["serde"] }
Expand Down
1 change: 1 addition & 0 deletions misc/python/materialize/mzcompose/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -894,6 +894,7 @@ def get_default_system_parameters(
"balancerd_connection_high_watermark",
"balancerd_connection_low_watermark",
"balancerd_pre_resolved_timeout",
"balancerd_https_enable_http2_alpn",
"balancerd_log_filter",
"balancerd_opentelemetry_filter",
"balancerd_log_filter_defaults",
Expand Down
1 change: 1 addition & 0 deletions misc/python/materialize/parallel_workload/action.py
Original file line number Diff line number Diff line change
Expand Up @@ -3394,6 +3394,7 @@ def __init__(
"balancerd_connection_high_watermark",
"balancerd_connection_low_watermark",
"balancerd_pre_resolved_timeout",
"balancerd_https_enable_http2_alpn",
"balancerd_log_filter",
"balancerd_opentelemetry_filter",
"balancerd_log_filter_defaults",
Expand Down
27 changes: 27 additions & 0 deletions src/balancerd/src/dyncfgs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,27 @@ pub const PRE_RESOLVED_TIMEOUT: Config<Duration> = Config::new(
ParameterScope::Environment,
);

/// Whether to advertise HTTP/2 via ALPN on the HTTPS listener.
///
/// balancerd is a byte proxy: it terminates TLS and forwards the decrypted
/// stream to environmentd. ALPN is answered during the client handshake,
/// before balancerd has connected upstream, so it cannot discover whether
/// environmentd speaks HTTP/2 in time. Enabling this while environmentd is
/// still HTTP/1.1-only makes clients negotiate h2 and send frames environmentd
/// rejects with "invalid HTTP version parsed (found HTTP2 preface)". Enable
/// only after every environmentd instance supports HTTP/2.
///
/// NOTE: read once when the TLS context is built at startup, so a change only
/// takes effect after balancerd restarts.
pub const HTTPS_ENABLE_HTTP2_ALPN: Config<bool> = Config::new(
"balancerd_https_enable_http2_alpn",
false,
"Whether to advertise HTTP/2 via ALPN on the HTTPS listener. \
Enable only after all environmentd instances support HTTP/2. \
Takes effect on balancerd restart.",
ParameterScope::Environment,
);

/// Sets the filter to apply to stderr logging.
pub const LOGGING_FILTER: Config<&str> = Config::new(
"balancerd_log_filter",
Expand Down Expand Up @@ -154,6 +175,7 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
.add(&CONNECTION_HIGH_WATERMARK)
.add(&CONNECTION_LOW_WATERMARK)
.add(&PRE_RESOLVED_TIMEOUT)
.add(&HTTPS_ENABLE_HTTP2_ALPN)
.add(&LOGGING_FILTER)
.add(&OPENTELEMETRY_FILTER)
.add(&LOGGING_FILTER_DEFAULTS)
Expand Down Expand Up @@ -200,6 +222,11 @@ pub(crate) fn set_defaults(
PRE_RESOLVED_TIMEOUT.name(),
mz_dyncfg::ConfigVal::Duration(humantime::parse_duration(v)?),
)
} else if k.as_str() == HTTPS_ENABLE_HTTP2_ALPN.name() {
config_updates.add_dynamic(
HTTPS_ENABLE_HTTP2_ALPN.name(),
mz_dyncfg::ConfigVal::Bool(bool::from_str(v)?),
)
} else {
return Err(anyhow!("Invalid default config value {k}"));
}
Expand Down
132 changes: 74 additions & 58 deletions src/balancerd/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ use std::time::{Duration, Instant};
use anyhow::Context;
use axum::{Router, routing};
use bytes::BytesMut;
use futures::TryFutureExt;
use futures::stream::BoxStream;
use hickory_resolver::config::LookupIpStrategy;
use hickory_resolver::lookup_ip::LookupIp;
Expand All @@ -40,7 +39,7 @@ use hickory_resolver::proto::rr::{Name, RData, RecordType};
use hickory_resolver::system_conf::read_system_conf;
use hickory_resolver::{Resolver, TokioResolver};
use hyper::StatusCode;
use hyper_util::rt::TokioIo;
use hyper_util::rt::{TokioExecutor, TokioIo};
use launchdarkly_server_sdk as ld;
use mz_build_info::{BuildInfo, build_info};
use mz_dyncfg::ConfigSet;
Expand Down Expand Up @@ -84,9 +83,9 @@ use uuid::Uuid;

use crate::codec::{BackendMessage, FramedConn};
use crate::dyncfgs::{
CONNECTION_HIGH_WATERMARK, CONNECTION_LOW_WATERMARK, INJECT_PROXY_PROTOCOL_HEADER_HTTP,
MAX_CONNECTIONS, PRE_RESOLVED_TIMEOUT, SIGTERM_CONNECTION_WAIT, SIGTERM_LISTEN_WAIT,
has_tracing_config_update, tracing_config,
CONNECTION_HIGH_WATERMARK, CONNECTION_LOW_WATERMARK, HTTPS_ENABLE_HTTP2_ALPN,
INJECT_PROXY_PROTOCOL_HEADER_HTTP, MAX_CONNECTIONS, PRE_RESOLVED_TIMEOUT,
SIGTERM_CONNECTION_WAIT, SIGTERM_LISTEN_WAIT, has_tracing_config_update, tracing_config,
};

/// Balancer build information.
Expand Down Expand Up @@ -310,7 +309,8 @@ impl BalancerService {
pub async fn serve(self) -> Result<(), anyhow::Error> {
let (pgwire_tls, https_tls) = match &self.cfg.tls {
Some(tls) => {
let context = tls.reloading_context(self.cfg.reload_certs)?;
let enable_http2_alpn = HTTPS_ENABLE_HTTP2_ALPN.get(&self.configs);
let context = tls.reloading_context(self.cfg.reload_certs, enable_http2_alpn)?;
(
Some(ReloadingTlsConfig {
context: context.clone(),
Expand Down Expand Up @@ -509,8 +509,11 @@ impl mz_server_core::Server for InternalHttpServer {
let conn = TokioIo::new(conn);

Box::pin(async {
let http = hyper::server::conn::http1::Builder::new();
http.serve_connection(conn, service).err_into().await
// Serve HTTP/1.1 or HTTP/2 (h2c via preface sniffing).
let http = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new());
http.serve_connection(conn, service)
.await
.map_err(|e| anyhow::anyhow!(e))
})
}
}
Expand Down Expand Up @@ -1760,18 +1763,26 @@ async fn connect_upstream(addr: SocketAddr, metrics: &ServerMetrics) -> io::Resu
/// Writes an HTTP error response to a client and closes the connection.
///
/// The proxied connections are raw TCP streams, so the HTTP framing has to be written by hand.
/// Errors are ignored: the connection is going away regardless.
async fn send_http_error(client_stream: &mut Box<dyn ClientStream>, status: &str, body: &str) {
let response = format!(
"HTTP/1.1 {status}\r\n\
Content-Type: text/plain\r\n\
Content-Length: {}\r\n\
Connection: close\r\n\
\r\n\
{body}",
body.len(),
);
let _ = client_stream.write_all(response.as_bytes()).await;
/// Errors are ignored: the connection is going away regardless. A client that negotiated HTTP/2
/// via ALPN cannot parse an HTTP/1.1 response, so for it the connection is only closed.
async fn send_http_error(
client_stream: &mut Box<dyn ClientStream>,
client_h2: bool,
status: &str,
body: &str,
) {
if !client_h2 {
let response = format!(
"HTTP/1.1 {status}\r\n\
Content-Type: text/plain\r\n\
Content-Length: {}\r\n\
Connection: close\r\n\
\r\n\
{body}",
body.len(),
);
let _ = client_stream.write_all(response.as_bytes()).await;
}
let _ = client_stream.shutdown().await;
}

Expand All @@ -1788,6 +1799,7 @@ struct HttpsBalancer {

impl HttpsBalancer {
/// Negotiates TLS and resolves a backend using the client's SNI hostname.
/// Also reports whether the client negotiated HTTP/2 via ALPN.
async fn pre_resolve(
conn: Connection,
tls_context: Option<ReloadingSslContext>,
Expand All @@ -1796,35 +1808,37 @@ impl HttpsBalancer {
port: u16,
metrics: &ServerMetrics,
fault: &ConnectionFault,
) -> Result<(Box<dyn ClientStream>, ResolvedAddr, SocketAddr), anyhow::Error> {
) -> Result<(Box<dyn ClientStream>, ResolvedAddr, SocketAddr, bool), anyhow::Error> {
let peer_addr = conn
.peer_addr()
.context("fetching peer addr")
.inspect_err(|_| fault.server())?;
let (client_stream, servername): (Box<dyn ClientStream>, Option<String>) = match tls_context
{
Some(tls_context) => {
let mut ssl_stream = Ssl::new(&tls_context.get())
.and_then(|ssl| SslStream::new(ssl, conn))
.inspect_err(|_| fault.server())?;
if let Err(e) = Pin::new(&mut ssl_stream).accept().await {
metrics.tls_handshake_failures().inc();
let _ = ssl_stream.get_mut().shutdown().await;
return Err(e.into());
let (client_stream, servername, client_h2): (Box<dyn ClientStream>, Option<String>, bool) =
match tls_context {
Some(tls_context) => {
let mut ssl_stream = Ssl::new(&tls_context.get())
.and_then(|ssl| SslStream::new(ssl, conn))
.inspect_err(|_| fault.server())?;
if let Err(e) = Pin::new(&mut ssl_stream).accept().await {
metrics.tls_handshake_failures().inc();
let _ = ssl_stream.get_mut().shutdown().await;
return Err(e.into());
}
let servername: Option<String> =
ssl_stream.ssl().servername(NameType::HOST_NAME).map(|sn| {
match sn.split_once('.') {
Some((left, _right)) => left,
None => sn,
}
.into()
});
debug!("Found sni servername: {servername:?} (https)");
let client_h2 =
ssl_stream.ssl().selected_alpn_protocol() == Some(b"h2".as_slice());
(Box::new(ssl_stream), servername, client_h2)
}
let servername: Option<String> =
ssl_stream.ssl().servername(NameType::HOST_NAME).map(|sn| {
match sn.split_once('.') {
Some((left, _right)) => left,
None => sn,
}
.into()
});
debug!("Found sni servername: {servername:?} (https)");
(Box::new(ssl_stream), servername)
}
_ => (Box::new(conn), None),
};
_ => (Box::new(conn), None, false),
};
let resolved = fault
.server_work(Self::resolve(
resolver,
Expand All @@ -1838,7 +1852,7 @@ impl HttpsBalancer {
fault.server();
}
})?;
Ok((client_stream, resolved, peer_addr))
Ok((client_stream, resolved, peer_addr, client_h2))
}

async fn resolve(
Expand Down Expand Up @@ -1942,21 +1956,22 @@ impl mz_server_core::Server for HttpsBalancer {
};
let pre_resolved = inner_metrics.pre_resolved_connections();

let (mut client_stream, resolved, peer_addr) = under_pre_resolved_timeout(
pre_resolved_timeout,
&inner_metrics,
fault_ref,
Self::pre_resolve(
conn,
tls_context,
&resolver,
&resolve_template,
port,
let (mut client_stream, resolved, peer_addr, client_h2) =
under_pre_resolved_timeout(
pre_resolved_timeout,
&inner_metrics,
fault_ref,
),
)
.await?;
Self::pre_resolve(
conn,
tls_context,
&resolver,
&resolve_template,
port,
&inner_metrics,
fault_ref,
),
)
.await?;
drop(pre_resolved);
// See the matching log line in the pgwire handler. The tenant always comes from
// SNI here, before any authentication.
Expand All @@ -1980,6 +1995,7 @@ impl mz_server_core::Server for HttpsBalancer {
fault_ref.server();
send_http_error(
&mut client_stream,
client_h2,
"502 Bad Gateway",
"upstream server not available",
)
Expand Down
Loading
Loading