From 645e4f6ab94ed4eceb9a8621e19471ed27313fb8 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Fri, 28 Apr 2023 02:13:35 +0300 Subject: [PATCH] use TLS in link proxy --- Cargo.lock | 114 ++++++++++++++++++++++++++++- Cargo.toml | 12 +-- proxy/Cargo.toml | 2 + proxy/src/auth/backend/link.rs | 4 +- proxy/src/bin/pg_sni_router.rs | 8 +- proxy/src/compute.rs | 34 +++++++-- proxy/src/console/provider/mock.rs | 4 +- proxy/src/console/provider/neon.rs | 3 +- 8 files changed, 156 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f3a83ce2d..b3705303e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1574,6 +1574,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -2361,6 +2376,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.26.2" @@ -2483,12 +2516,50 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.18.0" @@ -2816,6 +2887,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + [[package]] name = "plotters" version = "0.3.4" @@ -2847,7 +2924,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=43e6db254a97fdecbce33d8bc0890accfd74495e#43e6db254a97fdecbce33d8bc0890accfd74495e" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=27fc5729cc71a042e48eb980c8e736762138efe6#27fc5729cc71a042e48eb980c8e736762138efe6" dependencies = [ "bytes", "fallible-iterator", @@ -2857,10 +2934,21 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "postgres-native-tls" +version = "0.5.0" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=27fc5729cc71a042e48eb980c8e736762138efe6#27fc5729cc71a042e48eb980c8e736762138efe6" +dependencies = [ + "native-tls", + "tokio", + "tokio-native-tls", + "tokio-postgres", +] + [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=43e6db254a97fdecbce33d8bc0890accfd74495e#43e6db254a97fdecbce33d8bc0890accfd74495e" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=27fc5729cc71a042e48eb980c8e736762138efe6#27fc5729cc71a042e48eb980c8e736762138efe6" dependencies = [ "base64 0.20.0", "byteorder", @@ -2878,7 +2966,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=43e6db254a97fdecbce33d8bc0890accfd74495e#43e6db254a97fdecbce33d8bc0890accfd74495e" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=27fc5729cc71a042e48eb980c8e736762138efe6#27fc5729cc71a042e48eb980c8e736762138efe6" dependencies = [ "bytes", "fallible-iterator", @@ -3109,10 +3197,12 @@ dependencies = [ "itertools", "md5", "metrics", + "native-tls", "once_cell", "opentelemetry", "parking_lot", "pin-project-lite", + "postgres-native-tls", "postgres_backend", "pq_proto", "prometheus", @@ -4319,10 +4409,20 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=43e6db254a97fdecbce33d8bc0890accfd74495e#43e6db254a97fdecbce33d8bc0890accfd74495e" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=27fc5729cc71a042e48eb980c8e736762138efe6#27fc5729cc71a042e48eb980c8e736762138efe6" dependencies = [ "async-trait", "byteorder", @@ -4914,6 +5014,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index f4872433cd..bdfa07f53e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ jsonwebtoken = "8" libc = "0.2" md5 = "0.7.0" memoffset = "0.8" +native-tls = "0.2" nix = "0.26" notify = "5.0.0" num_cpus = "1.15" @@ -124,10 +125,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="27fc5729cc71a042e48eb980c8e736762138efe6" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="27fc5729cc71a042e48eb980c8e736762138efe6" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="27fc5729cc71a042e48eb980c8e736762138efe6" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="27fc5729cc71a042e48eb980c8e736762138efe6" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="27fc5729cc71a042e48eb980c8e736762138efe6" } tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" } ## Other git libraries @@ -162,7 +164,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. [patch.crates-io] -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="27fc5729cc71a042e48eb980c8e736762138efe6" } ################# Binary contents sections diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 9d702b29c3..e7a4fd236e 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -62,6 +62,8 @@ utils.workspace = true uuid.workspace = true webpki-roots.workspace = true x509-parser.workspace = true +native-tls.workspace = true +postgres-native-tls.workspace = true workspace_hack.workspace = true tokio-util.workspace = true diff --git a/proxy/src/auth/backend/link.rs b/proxy/src/auth/backend/link.rs index 7175a23dc1..ee6b781fae 100644 --- a/proxy/src/auth/backend/link.rs +++ b/proxy/src/auth/backend/link.rs @@ -9,6 +9,7 @@ use crate::{ use pq_proto::BeMessage as Be; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_postgres::config::SslMode; use tracing::{info, info_span}; #[derive(Debug, Error)] @@ -85,7 +86,8 @@ pub(super) async fn authenticate( .host(&db_info.host) .port(db_info.port) .dbname(&db_info.dbname) - .user(&db_info.user); + .user(&db_info.user) + .ssl_mode(SslMode::Require); // we need TLS connection with SNI to properly route it if let Some(password) = db_info.password { config.password(password.as_ref()); diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index 17660e5b31..3b8852ecc8 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -1,18 +1,16 @@ /// A stand-alone program that routes connections, e.g. from -/// `aaa--bbb--123.external.domain` to `aaa.bbb.123.internal.domain`. +/// `aaa--bbb--1234.external.domain` to `aaa.bbb.internal.domain:1234`. /// /// This allows connecting to pods/services running in the same Kubernetes cluster from /// the outside. Similar to an ingress controller for HTTPS. use std::{net::SocketAddr, sync::Arc}; use tokio::net::TcpListener; -// use tokio::net::TcpListener; use anyhow::{anyhow, bail, ensure, Context}; use clap::{self, Arg}; use futures::TryFutureExt; use proxy::console::messages::MetricsAuxInfo; -// use proxy::console::messages::MetricsAuxInfo; use proxy::stream::{PqStream, Stream}; use tokio::io::{AsyncRead, AsyncWrite}; @@ -106,7 +104,7 @@ async fn main() -> anyhow::Result<()> { // Start listening for incoming client connections let proxy_address: SocketAddr = args.get_one::("listen").unwrap().parse()?; - info!("Starting proxy on {proxy_address}"); + info!("Starting sni router on {proxy_address}"); let proxy_listener = TcpListener::bind(proxy_address).await?; let cancellation_token = CancellationToken::new(); @@ -240,7 +238,7 @@ async fn handle_client( .splitn(3, "--") .collect(); let port = dest[2].parse::().context("invalid port")?; - let destination = format!("{}.{}.{}:{}", dest[0], dest[1], dest_suffix, port); + let destination = format!("{}.{}.{}:{}", dest[0], dest[1], dest_suffix, port); info!("destination: {}", destination); diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index f95c00de14..07dc92c2ec 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -5,7 +5,7 @@ use pq_proto::StartupMessageParams; use std::{io, net::SocketAddr, time::Duration}; use thiserror::Error; use tokio::net::TcpStream; -use tokio_postgres::NoTls; +use tokio_postgres::tls::MakeTlsConnect; use tracing::{error, info, warn}; const COULD_NOT_CONNECT: &str = "Couldn't connect to compute node"; @@ -19,6 +19,9 @@ pub enum ConnectionError { #[error("{COULD_NOT_CONNECT}: {0}")] CouldNotConnect(#[from] io::Error), + + #[error("{COULD_NOT_CONNECT}: {0}")] + TlsError(#[from] native_tls::Error), } impl UserFacingError for ConnectionError { @@ -133,7 +136,7 @@ impl Default for ConnCfg { impl ConnCfg { /// Establish a raw TCP connection to the compute node. - async fn connect_raw(&self) -> io::Result<(SocketAddr, TcpStream)> { + async fn connect_raw(&self) -> io::Result<(SocketAddr, TcpStream, &str)> { use tokio_postgres::config::Host; // wrap TcpStream::connect with timeout @@ -186,7 +189,7 @@ impl ConnCfg { }; match connect_once(host, *port).await { - Ok(socket) => return Ok(socket), + Ok((sockaddr, stream)) => return Ok((sockaddr, stream, host)), Err(err) => { // We can't throw an error here, as there might be more hosts to try. warn!("couldn't connect to compute node at {host}:{port}: {err}"); @@ -206,7 +209,10 @@ impl ConnCfg { pub struct PostgresConnection { /// Socket connected to a compute node. - pub stream: TcpStream, + pub stream: tokio_postgres::maybe_tls_stream::MaybeTlsStream< + tokio::net::TcpStream, + postgres_native_tls::TlsStream, + >, /// PostgreSQL connection parameters. pub params: std::collections::HashMap, /// Query cancellation token. @@ -215,10 +221,22 @@ pub struct PostgresConnection { impl ConnCfg { async fn do_connect(&self) -> Result { - // TODO: establish a secure connection to the DB. - let (socket_addr, mut stream) = self.connect_raw().await?; - let (client, connection) = self.0.connect_raw(&mut stream, NoTls).await?; - info!("connected to compute node at {socket_addr}"); + let (socket_addr, stream, host) = self.connect_raw().await?; + + let tls_connector = native_tls::TlsConnector::builder() + .build() + .unwrap(); + let mut mk_tls = postgres_native_tls::MakeTlsConnector::new(tls_connector); + let tls = MakeTlsConnect::::make_tls_connect(&mut mk_tls, host)?; + + // connect_raw() will not use TLS if sslmode is "disable" + let (client, connection) = self.0.connect_raw(stream, tls).await?; + let stream = connection.stream.into_inner(); + + info!( + "connected to compute node at {host} ({socket_addr}) sslmode={:?}", + self.0.get_ssl_mode() + ); // This is very ugly but as of now there's no better way to // extract the connection parameters from tokio-postgres' connection. diff --git a/proxy/src/console/provider/mock.rs b/proxy/src/console/provider/mock.rs index eaac9c06d9..769b5cedcd 100644 --- a/proxy/src/console/provider/mock.rs +++ b/proxy/src/console/provider/mock.rs @@ -8,6 +8,7 @@ use crate::{auth::ClientCredentials, compute, error::io_error, scram, url::ApiUr use async_trait::async_trait; use futures::TryFutureExt; use thiserror::Error; +use tokio_postgres::config::SslMode; use tracing::{error, info, info_span, warn, Instrument}; #[derive(Debug, Error)] @@ -86,7 +87,8 @@ impl Api { let mut config = compute::ConnCfg::new(); config .host(self.endpoint.host_str().unwrap_or("localhost")) - .port(self.endpoint.port().unwrap_or(5432)); + .port(self.endpoint.port().unwrap_or(5432)) + .ssl_mode(SslMode::Disable); let node = NodeInfo { config, diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 3644db17f7..2ae3bc5157 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -8,6 +8,7 @@ use super::{ use crate::{auth::ClientCredentials, compute, http, scram}; use async_trait::async_trait; use futures::TryFutureExt; +use tokio_postgres::config::SslMode; use tracing::{error, info, info_span, warn, Instrument}; #[derive(Clone)] @@ -100,7 +101,7 @@ impl Api { // We'll set username and such later using the startup message. // TODO: add more type safety (in progress). let mut config = compute::ConnCfg::new(); - config.host(host).port(port); + config.host(host).port(port).ssl_mode(SslMode::Disable); // TLS is not configured on compute nodes. let node = NodeInfo { config,