From e04e5dd57d8ee5a42270d29e5597f431679ad3e5 Mon Sep 17 00:00:00 2001 From: kkmihai Date: Wed, 4 Nov 2026 06:21:03 +0200 Subject: [PATCH 001/151] feat(database): add ClickHouse data sources (#865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ClickHouse to Data Sources over its HTTP interface (HTTP/HTTPS, SSH tunnels, session settings, server-side cancellation): browse databases, tables and columns, inspect DDL, run queries and EXPLAIN plans, with ClickHouse dialect completion. Table data stays read-only and manual transactions are rejected, since ClickHouse keys are not unique and HTTP has no transactions. Results are streamed as JSONCompactEachRowWithNamesAndTypes and parsing stops at the row limit, because result_overflow_mode=break still returns a whole block; errors show only the server's "Code: …" message, including failures raised mid-stream. Fixes #862 Co-authored-by: kkMihai <59262001+kkMihai@users.noreply.github.com> --- Cargo.lock | 23 + crates/ide-db/Cargo.toml | 6 +- crates/ide-db/src/clickhouse.rs | 844 ++++++++++++++++++++ crates/ide-db/src/conn.rs | 40 +- crates/ide-db/src/lib.rs | 88 +- crates/ide-db/src/plan.rs | 19 +- crates/ide-db/src/source.rs | 41 +- crates/ide-db/src/split.rs | 19 +- crates/ide-db/src/sql.rs | 108 ++- crates/ide-db/tests/clickhouse.rs | 136 ++++ crates/ide-db/tests/sqlite.rs | 6 +- crates/ide-host/src/ops/db.rs | 3 +- e2e/clickhouse-source.spec.ts | 42 + src/bindings.ts | 2 +- src/features/database/ObjectDialogs.tsx | 24 +- src/features/database/SourceDialog.tsx | 50 +- src/features/database/TableEditor.tsx | 11 +- src/features/database/console.tsx | 29 +- src/features/database/database.test.ts | 40 +- src/features/database/export.ts | 6 +- src/features/database/model.ts | 4 + src/features/database/objects.test.ts | 48 ++ src/features/database/objects.ts | 37 +- src/features/database/sql/analyze.ts | 8 +- src/features/database/sql/complete.ts | 40 +- src/features/database/sql/dialect.ts | 86 ++ src/features/database/sql/lexer.ts | 22 +- src/features/database/sql/params.ts | 11 + src/features/database/sql/resolve.ts | 22 +- src/features/database/sql/sql.test.ts | 66 +- src/features/database/store.ts | 8 +- src/features/database/table-editor.test.tsx | 90 +++ src/features/database/treeActions.ts | 18 +- src/features/database/treeMenu.ts | 18 +- src/features/database/vendors.ts | 20 + src/platform/locales/text/zh.json | 3 + src/platform/locales/textCatalog.test.ts | 1 + src/ui/icons.tsx | 1 + src/ui/openidea/database/clickhouse.svg | 1 + 39 files changed, 1921 insertions(+), 120 deletions(-) create mode 100644 crates/ide-db/src/clickhouse.rs create mode 100644 crates/ide-db/tests/clickhouse.rs create mode 100644 e2e/clickhouse-source.spec.ts create mode 100644 src/features/database/table-editor.test.tsx create mode 100644 src/ui/openidea/database/clickhouse.svg diff --git a/Cargo.lock b/Cargo.lock index 42891043..ec56cb17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4154,8 +4154,11 @@ dependencies = [ "dirs 6.0.0", "futures", "ide-core", + "ide-http", "ide-ssh", "parking_lot", + "reqwest", + "rustls", "serde", "serde_json", "specta", @@ -4166,6 +4169,7 @@ dependencies = [ "tokio-util", "toml 0.9.12+spec-1.1.0", "tracing", + "uuid", ] [[package]] @@ -7133,6 +7137,7 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", @@ -7491,6 +7496,12 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "saa" version = "5.6.2" @@ -7795,6 +7806,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_with" version = "3.24.0" diff --git a/crates/ide-db/Cargo.toml b/crates/ide-db/Cargo.toml index 6a5cb440..8f985393 100644 --- a/crates/ide-db/Cargo.toml +++ b/crates/ide-db/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true [dependencies] ide-core.workspace = true serde.workspace = true -serde_json.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } specta.workspace = true toml.workspace = true tokio.workspace = true @@ -21,6 +21,10 @@ bb8.workspace = true tokio-util = { workspace = true, features = ["compat"] } dirs.workspace = true ide-ssh.workspace = true +ide-http.workspace = true +reqwest = { workspace = true, features = ["query"] } +rustls.workspace = true +uuid.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/ide-db/src/clickhouse.rs b/crates/ide-db/src/clickhouse.rs new file mode 100644 index 00000000..34e869dc --- /dev/null +++ b/crates/ide-db/src/clickhouse.rs @@ -0,0 +1,844 @@ +use crate::conn::{QueryResult, ResultColumn, Setup}; +use crate::source::{DataSource, DbSslMode, DbTxMode}; +use ide_core::{CancellationToken, IdeError, IdeResult, error::ErrorKind}; +use parking_lot::RwLock; +use reqwest::{Client, RequestBuilder, StatusCode, Url}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::Mutex; + +#[derive(Clone)] +pub struct ClickhousePool(Arc); + +struct State { + client: RwLock>, + endpoint: Url, + user: String, + password: String, + database: String, + settings: Vec<(String, String)>, + read_only: bool, + session_id: String, + session: Mutex, + closed: CancellationToken, +} + +/// The response format: a names line, a types line, then one JSON array per row, so rows past +/// the grid's limit are skipped without being parsed. +const FORMAT: &str = "JSONCompactEachRowWithNamesAndTypes"; + +pub(crate) fn transaction_error() -> IdeError { + IdeError::invalid("ClickHouse HTTP does not support SQL transactions; use auto-commit and run writes directly") +} + +impl ClickhousePool { + pub(crate) async fn connect(source: &DataSource, password: Option<&str>, setup: Setup) -> IdeResult { + if source.options.transaction == DbTxMode::Manual { + return Err(transaction_error()); + } + for (name, _) in &setup.params { + if reserved_setting(name) { + return Err(IdeError::invalid(format!("ClickHouse HTTP manages the {name} setting"))); + } + } + let secure = matches!(source.ssl, DbSslMode::Require | DbSslMode::VerifyCa | DbSslMode::VerifyFull); + let tls_files = [&source.ssl_ca_file, &source.ssl_cert_file, &source.ssl_key_file]; + if !secure && tls_files.iter().any(|p| !p.trim().is_empty()) { + return Err(IdeError::invalid( + "ClickHouse TLS certificate files require an SSL mode of Require, Verify CA, or Verify Full", + )); + } + let host = source.host.trim(); + if host.is_empty() || host.contains(['/', '@', '?', '#']) { + return Err(IdeError::invalid("ClickHouse host must contain only a hostname or IP address")); + } + let host = if host.contains(':') && !host.starts_with('[') { format!("[{host}]") } else { host.to_owned() }; + let scheme = if secure { "https" } else { "http" }; + let port = source.port.unwrap_or(if secure { 8443 } else { 8123 }); + let endpoint = Url::parse(&format!("{scheme}://{host}:{port}/")) + .map_err(|_| IdeError::invalid("invalid ClickHouse HTTP host or port"))?; + let timeout = setup.connect_timeout.unwrap_or(Duration::from_secs(30)); + let _ = rustls::crypto::ring::default_provider().install_default(); + let mut builder = Client::builder() + .connect_timeout(timeout) + .pool_max_idle_per_host(4) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(concat!("OpenIDEA/", env!("CARGO_PKG_VERSION"))); + let ca = source.ssl_ca_file.trim(); + if !ca.is_empty() { + let pem = read_pem(ca, "CA certificate")?; + let certificates = reqwest::Certificate::from_pem_bundle(&pem) + .map_err(|_| IdeError::invalid("invalid ClickHouse CA certificate PEM"))?; + if certificates.is_empty() { + return Err(IdeError::invalid("ClickHouse CA certificate PEM contains no certificates")); + } + builder = builder.tls_certs_only(certificates); + } + if source.ssl == DbSslMode::VerifyCa { + if ca.is_empty() { + return Err(IdeError::invalid("ClickHouse Verify CA requires a CA certificate file")); + } + builder = builder.tls_danger_accept_invalid_hostnames(true); + } + let cert = source.ssl_cert_file.trim(); + let key = source.ssl_key_file.trim(); + if cert.is_empty() != key.is_empty() { + return Err(IdeError::invalid("ClickHouse client TLS requires both certificate and key files")); + } + if !cert.is_empty() { + let mut pem = read_pem(cert, "client certificate")?; + pem.push(b'\n'); + pem.extend(read_pem(key, "client key")?); + let identity = reqwest::Identity::from_pem(&pem) + .map_err(|_| IdeError::invalid("invalid ClickHouse client certificate or key PEM"))?; + builder = builder.identity(identity); + } + let proxy = ide_core::proxy_settings::proxy(); + let client = ide_http::client::with_proxy(builder, &proxy).build().map_err(http_error)?; + let database = if source.options.default_schema.trim().is_empty() { + source.database.trim() + } else { + source.options.default_schema.trim() + }; + let pool = Self(Arc::new(State { + client: RwLock::new(Some(client)), + endpoint, + user: if source.user.is_empty() { "default".into() } else { source.user.clone() }, + password: password.unwrap_or_default().to_owned(), + database: if database.is_empty() { "default".into() } else { database.into() }, + settings: setup.params, + read_only: source.read_only, + session_id: uuid::Uuid::new_v4().to_string(), + session: Mutex::new(false), + closed: CancellationToken::new(), + })); + let cancel = CancellationToken::new(); + let initialize = async { + pool.execute("SELECT 1", 1, &cancel).await?; + for sql in setup.statements { + pool.execute(&sql, 0, &cancel).await?; + } + Ok(pool.clone()) + }; + tokio::pin!(initialize); + tokio::select! { + result = &mut initialize => { + if result.is_err() { + pool.close(); + } + result + }, + () = tokio::time::sleep(timeout) => { + cancel.cancel(); + let _ = initialize.await; + pool.close(); + Err(IdeError::new(ErrorKind::TimedOut, "timed out connecting to ClickHouse HTTP")) + } + } + } + + pub fn close(&self) { + self.0.closed.cancel(); + self.0.client.write().take(); + } + + pub fn is_closed(&self) -> bool { + self.0.closed.is_cancelled() + } + + fn client(&self) -> IdeResult { + self.0.client.read().clone().ok_or_else(|| IdeError::new(ErrorKind::Io, "ClickHouse connection is closed")) + } + + fn request(&self, client: &Client, query_id: &str, max_rows: usize, initialized: bool) -> RequestBuilder { + let mut request = client + .post(self.0.endpoint.clone()) + .basic_auth(&self.0.user, Some(&self.0.password)) + .header(reqwest::header::CONTENT_TYPE, "text/plain; charset=utf-8") + .query(&self.0.settings) + .query(&[ + ("session_id", self.0.session_id.clone()), + ("session_timeout", "3600".into()), + ("query_id", query_id.to_owned()), + ("default_format", FORMAT.into()), + ("output_format_json_quote_denormals", "1".into()), + ("max_result_rows", max_rows.saturating_add(1).to_string()), + ("result_overflow_mode", "break".into()), + ]); + if initialized { + request = request.query(&[("session_check", "1")]); + } else { + request = request.query(&[("database", &self.0.database)]); + } + if self.0.read_only { + request = request.query(&[("readonly", "1")]); + } + request + } + + pub async fn execute(&self, sql: &str, max_rows: usize, cancel: &CancellationToken) -> IdeResult { + if transaction_statement(sql) { + return Err(transaction_error()); + } + let mut session = tokio::select! { + biased; + () = cancel.cancelled() => return Err(IdeError::cancelled()), + () = self.0.closed.cancelled() => return Err(IdeError::new(ErrorKind::Io, "ClickHouse connection is closed")), + session = self.0.session.lock() => session, + }; + let client = self.client()?; + let query_id = uuid::Uuid::new_v4().to_string(); + let request = self.request(&client, &query_id, max_rows, *session).body(sql.to_owned()); + let started = Instant::now(); + let fetch = async { + let mut response = request.send().await.map_err(http_error)?; + let execution_ms = started.elapsed().as_secs_f64() * 1000.0; + let status = response.status(); + let header = |name: &str| response.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_owned); + let exception_code = header("X-ClickHouse-Exception-Code"); + let format = header("X-ClickHouse-Format"); + let affected = header("X-ClickHouse-Summary") + .and_then(|v| serde_json::from_str::(&v).ok()) + .and_then(|v| { + v.get("written_rows") + .and_then(|n| n.as_str().and_then(|n| n.parse::().ok()).or_else(|| n.as_u64())) + }); + if !status.is_success() || exception_code.as_deref().is_some_and(|code| code != "0") { + let body = response.bytes().await.map_err(http_error)?; + return Err(self.server_error(status, exception_code.as_deref(), &body)); + } + let (columns, rows, has_more) = if format.as_deref().is_some_and(|f| f != FORMAT) { + let body = response.bytes().await.map_err(http_error)?; + if !body.iter().all(u8::is_ascii_whitespace) { + return Err(IdeError::invalid( + "ClickHouse result grids need the default output format; remove the query's FORMAT clause", + )); + } + (Vec::new(), Vec::new(), false) + } else { + let mut reader = RowReader::new(max_rows); + while let Some(chunk) = response.chunk().await.map_err(http_error)? { + reader.push(&chunk)?; + } + reader.finish().map_err(|error| self.redact_error(error))? + }; + let affected = if columns.is_empty() { affected.map(|n| n as f64) } else { None }; + Ok(QueryResult { + statement: sql.to_owned(), + columns, + rows, + affected, + has_more, + elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, + execution_ms, + }) + }; + let result = tokio::select! { + biased; + () = cancel.cancelled() => None, + () = self.0.closed.cancelled() => None, + result = fetch => Some(result), + }; + match result { + Some(result) => { + if result.is_ok() { + *session = true; + } + result + } + None => { + let killed = self.kill(&client, &query_id).await; + match killed { + Ok(()) => Err(IdeError::cancelled()), + Err(error) => Err(IdeError::new( + ErrorKind::Cancelled, + format!("cancelled locally; ClickHouse server cancellation failed: {}", error.message), + )), + } + } + } + } + + async fn kill(&self, client: &Client, query_id: &str) -> IdeResult<()> { + let response = client + .post(self.0.endpoint.clone()) + .basic_auth(&self.0.user, Some(&self.0.password)) + .timeout(Duration::from_secs(3)) + .body(format!("KILL QUERY WHERE query_id = '{query_id}' SYNC")) + .send() + .await + .map_err(http_error)?; + let status = response.status(); + let code = response + .headers() + .get("X-ClickHouse-Exception-Code") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body = response.bytes().await.map_err(http_error)?; + if status.is_success() && !code.as_deref().is_some_and(|code| code != "0") && !body.starts_with(b"Code:") { + Ok(()) + } else { + Err(self.server_error(status, code.as_deref(), &body)) + } + } + + fn redact_error(&self, mut error: IdeError) -> IdeError { + if !self.0.password.is_empty() { + error.message = error.message.replace(&self.0.password, "[redacted]"); + } + error + } + + fn server_error(&self, status: StatusCode, code: Option<&str>, body: &[u8]) -> IdeError { + let message = String::from_utf8_lossy(body); + let message: String = exception_text(&message).chars().take(16_384).collect(); + let auth = matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) + || matches!(code, Some("193" | "194" | "195" | "516" | "497")); + let detail = if message.is_empty() { status.canonical_reason().unwrap_or("request failed") } else { &message }; + let detail = if detail.contains("SESSION_NOT_FOUND") || detail.contains("Session not found") { + "ClickHouse HTTP session expired; reconnect to restore session settings" + } else { + detail + }; + self.redact_error(IdeError::new( + if auth { ErrorKind::PermissionDenied } else { ErrorKind::InvalidInput }, + format!("ClickHouse HTTP {}: {detail}", status.as_u16()), + )) + } +} + +type DecodedResult = (Vec, Vec>>, bool); + +/// Decodes a streamed [`FORMAT`] body. Keeps the first `max_rows` rows; later rows only set +/// `has_more` and are read past unparsed, so a grid limit bounds memory however large the +/// block `result_overflow_mode=break` lets the server send. A line that is not a JSON array +/// starts the exception ClickHouse writes when a query fails mid-stream. +struct RowReader { + max_rows: usize, + pending: Vec, + names: Option>, + types: Option>, + rows: Vec>>, + more: bool, + exception: Option, +} + +impl RowReader { + fn new(max_rows: usize) -> Self { + Self { max_rows, pending: Vec::new(), names: None, types: None, rows: Vec::new(), more: false, exception: None } + } + + fn push(&mut self, chunk: &[u8]) -> IdeResult<()> { + self.pending.extend_from_slice(chunk); + let buffer = std::mem::take(&mut self.pending); + let mut start = 0; + while let Some(end) = buffer[start..].iter().position(|&b| b == b'\n') { + self.line(&buffer[start..start + end])?; + start += end + 1; + } + self.pending = buffer[start..].to_vec(); + Ok(()) + } + + fn line(&mut self, line: &[u8]) -> IdeResult<()> { + if let Some(exception) = &mut self.exception { + if exception.len() < 16_384 { + exception.push_str(&String::from_utf8_lossy(line)); + exception.push('\n'); + } + return Ok(()); + } + let text = line.trim_ascii(); + if text.is_empty() { + return Ok(()); + } + if text[0] != b'[' { + self.exception = Some(format!("{}\n", String::from_utf8_lossy(line))); + return Ok(()); + } + if self.names.is_some() && self.types.is_some() && self.rows.len() >= self.max_rows { + self.more = true; + return Ok(()); + } + let Ok(values) = serde_json::from_slice::>>(text) else { + self.exception = Some(format!("{}\n", String::from_utf8_lossy(line))); + return Ok(()); + }; + let protocol = |message: String| IdeError::new(ErrorKind::Protocol, message); + let header = |values: Vec>| { + values + .iter() + .map(|v| serde_json::from_str::(v.get())) + .collect::, _>>() + .map_err(|e| protocol(format!("invalid ClickHouse result header: {e}"))) + }; + let Some(names) = &self.names else { + self.names = Some(header(values)?); + return Ok(()); + }; + if self.types.is_none() { + self.types = Some(header(values)?); + return Ok(()); + } + if values.len() != names.len() { + return Err(protocol("ClickHouse result row width does not match its metadata".into())); + } + let row = values + .into_iter() + .map(|value| { + let text = value.get(); + if text == "null" { + Ok(None) + } else if text.starts_with('"') { + serde_json::from_str::(text).map(Some).map_err(|e| protocol(e.to_string())) + } else { + Ok(Some(text.to_owned())) + } + }) + .collect::>()?; + self.rows.push(row); + Ok(()) + } + + fn finish(mut self) -> IdeResult { + let rest = std::mem::take(&mut self.pending); + self.line(&rest)?; + if let Some(exception) = self.exception { + return Err(IdeError::invalid(format!("ClickHouse: {}", exception_text(&exception)))); + } + let Some(names) = self.names else { return Ok((Vec::new(), Vec::new(), false)) }; + let types = self.types.unwrap_or_default(); + if types.len() != names.len() { + return Err(IdeError::new(ErrorKind::Protocol, "ClickHouse result types do not match its columns")); + } + let columns = names.into_iter().zip(types).map(|(name, type_name)| ResultColumn { name, type_name }).collect(); + Ok((columns, self.rows, self.more)) + } +} + +/// The server's message in a failed response: rows already streamed and the +/// `__exception__` block markers ClickHouse wraps a mid-stream error in are dropped. +fn exception_text(body: &str) -> &str { + let Some(at) = body.find("Code:") else { return body.trim() }; + let text = &body[at..]; + match text.find("\n__exception__") { + // The line before the closing marker is ` `. + Some(end) => text[..end].trim_end().rsplit_once('\n').map_or(text[..end].trim(), |(message, _)| message.trim()), + None => text.trim(), + } +} + +fn http_error(error: reqwest::Error) -> IdeError { + use std::error::Error; + let kind = if error.is_timeout() { ErrorKind::TimedOut } else { ErrorKind::Io }; + let error = error.without_url(); + let mut message = error.to_string(); + let mut source = error.source(); + while let Some(error) = source { + let detail = error.to_string(); + if !message.contains(&detail) { + message.push_str(": "); + message.push_str(&detail); + } + source = error.source(); + } + IdeError::new(kind, format!("ClickHouse HTTP: {message}")) +} + +fn read_pem(path: &str, label: &str) -> IdeResult> { + std::fs::read(ide_ssh::expand(path)) + .map_err(|error| IdeError::new(ErrorKind::Io, format!("cannot read ClickHouse {label}: {error}"))) +} + +fn reserved_setting(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + name.starts_with("output_format_json_") + || matches!( + name.as_str(), + "user" + | "password" + | "query" + | "database" + | "session_id" + | "session_check" + | "session_timeout" + | "query_id" + | "default_format" + | "max_result_rows" + | "result_overflow_mode" + | "readonly" + ) +} + +fn transaction_statement(sql: &str) -> bool { + let mut sql = sql.trim_start(); + loop { + if let Some(rest) = sql.strip_prefix("--").or_else(|| sql.strip_prefix('#')) { + sql = rest.split_once('\n').map_or("", |(_, rest)| rest).trim_start(); + } else if let Some(rest) = sql.strip_prefix("/*") { + sql = rest.split_once("*/").map_or("", |(_, rest)| rest).trim_start(); + } else { + break; + } + } + let mut words = sql.split(|c: char| !c.is_ascii_alphanumeric() && c != '_').filter(|s| !s.is_empty()); + let first = words.next().unwrap_or_default(); + ["BEGIN", "COMMIT", "ROLLBACK", "SAVEPOINT", "RELEASE"].iter().any(|s| first.eq_ignore_ascii_case(s)) + || (first.eq_ignore_ascii_case("START") && words.next().is_some_and(|s| s.eq_ignore_ascii_case("TRANSACTION"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::conn::Pool; + use crate::source::{DbDriver, DbOptions, DbProperty}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + const ONE: &str = "[\"1\"]\n[\"UInt8\"]\n[1]\n"; + + async fn read_request(stream: &mut tokio::net::TcpStream) -> String { + let mut bytes = Vec::new(); + let mut buffer = [0; 4096]; + loop { + let count = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut buffer)).await.unwrap().unwrap(); + assert!(count > 0, "request ended early"); + bytes.extend_from_slice(&buffer[..count]); + if let Some(end) = bytes.windows(4).position(|s| s == b"\r\n\r\n") { + let header = String::from_utf8_lossy(&bytes[..end]); + let length: usize = header + .lines() + .find_map(|line| { + line.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse().unwrap()) + }) + .unwrap_or(0); + if bytes.len() >= end + 4 + length { + return String::from_utf8(bytes).unwrap(); + } + } + } + } + + async fn reply(stream: &mut tokio::net::TcpStream, status: &str, headers: &str, body: &str) { + let response = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n{headers}\r\n{body}", body.len()); + stream.write_all(response.as_bytes()).await.unwrap(); + } + + async fn server( + responses: Vec<(&'static str, &'static str, &'static str)>, + ) -> (u16, tokio::task::JoinHandle>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + let mut requests = Vec::new(); + for (status, headers, body) in responses { + let (mut stream, _) = + tokio::time::timeout(Duration::from_secs(5), listener.accept()).await.unwrap().unwrap(); + requests.push(read_request(&mut stream).await); + reply(&mut stream, status, headers, body).await; + } + requests + }); + (port, task) + } + + fn source(port: u16) -> DataSource { + DataSource { driver: DbDriver::Clickhouse, host: "127.0.0.1".into(), port: Some(port), ..Default::default() } + } + + fn decode(body: &[u8], max_rows: usize, chunk: usize) -> IdeResult { + let mut reader = RowReader::new(max_rows); + for part in body.chunks(chunk) { + reader.push(part)?; + } + reader.finish() + } + + #[test] + fn rows_preserve_precision_nulls_and_strings_across_chunks() { + let body = concat!( + r#"["n", "d", "v", "v", "v"]"#, + "\n", + r#"["UInt64", "Decimal(38, 9)", "Nullable(String)", "Nullable(String)", "Nullable(String)"]"#, + "\n", + r#"[18446744073709551615, 12345678901234567890.123456789, null, "null", "ᴺᵁᴸᴸ"]"#, + ); + for chunk in [1, 7, body.len()] { + let (columns, rows, more) = decode(body.as_bytes(), 1, chunk).unwrap(); + assert_eq!(columns.len(), 5); + assert_eq!(columns[1].type_name, "Decimal(38, 9)"); + assert_eq!( + rows[0], + vec![ + Some("18446744073709551615".into()), + Some("12345678901234567890.123456789".into()), + None, + Some("null".into()), + Some("ᴺᵁᴸᴸ".into()), + ] + ); + assert!(!more); + } + let (columns, rows, more) = decode(b"[\"x\"]\n[\"String\"]\n", 0, 64).unwrap(); + assert_eq!(columns[0].name, "x"); + assert!(rows.is_empty() && !more); + assert_eq!(decode(b"", 5, 64).unwrap().0.len(), 0); + assert!(decode(b"[\"x\"]\n[\"UInt8\"]\n[1, 2]\n", 1, 64).is_err()); + } + + #[test] + fn rows_past_the_limit_are_skipped_but_a_later_exception_still_fails() { + let mut body = String::from("[\"n\"]\n[\"UInt64\"]\n"); + for n in 0..1000 { + body.push_str(&format!("[{n}]\n")); + } + let (_, rows, more) = decode(body.as_bytes(), 2, 4096).unwrap(); + assert_eq!(rows, vec![vec![Some("0".into())], vec![Some("1".into())]]); + assert!(more); + body.push_str("\n__exception__\nabcdefgh\nCode: 395. DB::Exception: Value passed to 'throwIf' is non-zero\n"); + body.push_str("62 abcdefgh\n__exception__\n"); + let error = decode(body.as_bytes(), 2, 4096).unwrap_err(); + assert_eq!(error.message, "ClickHouse: Code: 395. DB::Exception: Value passed to 'throwIf' is non-zero"); + let multiline = "Code: 62. DB::Exception: Syntax error\nat line 2"; + assert_eq!(exception_text(multiline), multiline); + } + + #[tokio::test] + async fn http_requests_use_auth_settings_limits_and_a_shared_session() { + let (port, requests) = server(vec![ + ("200 OK", "X-ClickHouse-Format: JSONCompactEachRowWithNamesAndTypes\r\n", ONE), + ("200 OK", "", ""), + ( + "200 OK", + "X-ClickHouse-Format: JSONCompactEachRowWithNamesAndTypes\r\n", + "[\"n\"]\n[\"UInt64\"]\n[\"1\"]\n[\"2\"]\n[\"3\"]\n", + ), + ]) + .await; + let source = DataSource { + user: "reader".into(), + read_only: true, + options: DbOptions { default_schema: "analytics".into(), time_zone: "UTC".into(), ..Default::default() }, + properties: vec![DbProperty { name: "max_threads".into(), value: "2".into() }], + ..source(port) + }; + let pool = Pool::connect(&source, Some("mock-password"), None).await.unwrap(); + pool.execute("USE other", 0, &CancellationToken::new()).await.unwrap(); + let result = pool.execute("SELECT number FROM numbers(3)", 1, &CancellationToken::new()).await.unwrap(); + assert_eq!(result.rows, vec![vec![Some("1".into())]]); + assert!(result.has_more && result.affected.is_none()); + let requests = requests.await.unwrap(); + let target = |request: &String| { + Url::parse(&format!( + "http://localhost{}", + request.lines().next().unwrap().split_whitespace().nth(1).unwrap() + )) + .unwrap() + }; + let urls: Vec<_> = requests.iter().map(target).collect(); + let value = |url: &Url, key: &str| url.query_pairs().find(|(k, _)| k == key).map(|(_, v)| v.into_owned()); + assert_eq!(value(&urls[0], "database").as_deref(), Some("analytics")); + assert_eq!(value(&urls[1], "database"), None); + assert_eq!(value(&urls[1], "session_check").as_deref(), Some("1")); + assert_eq!(value(&urls[0], "session_id"), value(&urls[2], "session_id")); + assert_ne!(value(&urls[0], "query_id"), value(&urls[2], "query_id")); + assert_eq!(value(&urls[2], "max_result_rows").as_deref(), Some("2")); + for (request, url) in requests.iter().zip(&urls) { + assert!(request.starts_with("POST /?")); + assert!(request.contains("Basic cmVhZGVyOm1vY2stcGFzc3dvcmQ=")); + assert!(!url.as_str().contains("reader") && !url.as_str().contains("mock-password")); + assert_eq!(value(url, "readonly").as_deref(), Some("1")); + assert_eq!(value(url, "session_timezone").as_deref(), Some("UTC")); + assert_eq!(value(url, "max_threads").as_deref(), Some("2")); + } + assert!(requests[0].ends_with("SELECT 1")); + assert!(pool.transaction(&[]).await.is_err()); + let cancel = CancellationToken::new(); + cancel.cancel(); + assert_eq!(pool.execute("SELECT 1", 1, &cancel).await.unwrap_err().kind, ErrorKind::Cancelled); + pool.close().await; + assert!(pool.is_closed()); + assert!(pool.execute("SELECT 1", 1, &CancellationToken::new()).await.is_err()); + } + + #[tokio::test] + async fn sequential_queries_reuse_the_http_connection() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + for _ in 0..2 { + read_request(&mut stream).await; + let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{ONE}", ONE.len()); + stream.write_all(response.as_bytes()).await.unwrap(); + } + }); + let pool = Pool::connect(&source(port), None, None).await.unwrap(); + assert_eq!( + pool.execute("SELECT 1", 1, &CancellationToken::new()).await.unwrap().rows, + vec![vec![Some("1".into())]] + ); + task.await.unwrap(); + } + + #[tokio::test] + async fn server_errors_keep_details_and_hide_passwords() { + let (port, requests) = server(vec![( + "403 Forbidden", + "X-ClickHouse-Exception-Code: 516\r\n", + "Code: 516. DB::Exception: bad mock-password", + )]) + .await; + let error = match Pool::connect(&source(port), Some("mock-password"), None).await { + Ok(_) => panic!("expected login failure"), + Err(error) => error, + }; + assert_eq!(error.kind, ErrorKind::PermissionDenied); + assert!(error.message.contains("516") && error.message.contains("[redacted]")); + assert!(!error.message.contains("mock-password")); + requests.await.unwrap(); + let (port, requests) = server(vec![("200 OK", "", "Code: 241. DB::Exception: memory limit")]).await; + let error = match Pool::connect(&source(port), None, None).await { + Ok(_) => panic!("expected late query failure"), + Err(error) => error, + }; + assert!(error.message.contains("memory limit")); + requests.await.unwrap(); + let (port, requests) = server(vec![( + "500 Internal Server Error", + "X-ClickHouse-Exception-Code: 395\r\n", + "[\"1\"]\n[\"UInt8\"]\n[1]\n\n__exception__\ntag\nCode: 395. DB::Exception: thrown\n32 tag\n__exception__\n", + )]) + .await; + let error = match Pool::connect(&source(port), None, None).await { + Ok(_) => panic!("expected mid-stream failure"), + Err(error) => error, + }; + assert_eq!(error.message, "ClickHouse HTTP 500: Code: 395. DB::Exception: thrown"); + requests.await.unwrap(); + } + + #[tokio::test] + async fn cancellation_sends_kill_outside_the_busy_session() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let (started, received) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut health, _) = listener.accept().await.unwrap(); + read_request(&mut health).await; + reply(&mut health, "200 OK", "", ONE).await; + drop(health); + let (mut query, _) = listener.accept().await.unwrap(); + let request = read_request(&mut query).await; + started.send(()).unwrap(); + let (mut kill, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()).await.unwrap().unwrap(); + let kill_request = read_request(&mut kill).await; + reply(&mut kill, "200 OK", "", "").await; + (request, kill_request) + }); + let pool = Pool::connect(&source(port), None, None).await.unwrap(); + let cancel = CancellationToken::new(); + let query = pool.execute("SELECT sleep(30)", 1, &cancel); + let cancelled = async { + received.await.unwrap(); + cancel.cancel(); + }; + let (result, ()) = tokio::join!(query, cancelled); + assert_eq!(result.unwrap_err().kind, ErrorKind::Cancelled); + let (query, kill) = server.await.unwrap(); + let target = query.lines().next().unwrap().split_whitespace().nth(1).unwrap(); + let url = Url::parse(&format!("http://localhost{target}")).unwrap(); + let query_id = url.query_pairs().find(|(k, _)| k == "query_id").unwrap().1.into_owned(); + assert!(kill.ends_with(&format!("KILL QUERY WHERE query_id = '{query_id}' SYNC"))); + assert!(!kill.contains("session_id=")); + } + + #[tokio::test] + async fn unsupported_settings_and_transactions_fail_before_connecting() { + for statement in ["BEGIN", "-- comment\n COMMIT", "/* comment */ ROLLBACK", "START TRANSACTION", "SAVEPOINT s"] + { + assert!(transaction_statement(statement)); + } + assert!(!transaction_statement("SELECT 'BEGIN'")); + let sources = [ + DataSource { options: DbOptions { transaction: DbTxMode::Manual, ..Default::default() }, ..source(1) }, + DataSource { + properties: vec![DbProperty { name: "statement_cache_capacity".into(), value: "4".into() }], + ..source(1) + }, + DataSource { properties: vec![DbProperty { name: "password".into(), value: "x".into() }], ..source(1) }, + DataSource { ssl_ca_file: "unused.pem".into(), ..source(1) }, + DataSource { ssl: DbSslMode::VerifyCa, ..source(1) }, + DataSource { ssl: DbSslMode::Require, ssl_cert_file: "unused.pem".into(), ..source(1) }, + ]; + for source in sources { + let error = match Pool::connect(&source, None, None).await { + Ok(_) => panic!("unsupported settings were accepted"), + Err(error) => error, + }; + assert_eq!(error.kind, ErrorKind::InvalidInput); + } + } + + #[tokio::test] + #[ignore = "requires CLICKHOUSE_TEST_HOST, CLICKHOUSE_TEST_USER, and CLICKHOUSE_TEST_PASSWORD"] + async fn clickhouse_cloud_transport() { + let source = DataSource { + driver: DbDriver::Clickhouse, + host: std::env::var("CLICKHOUSE_TEST_HOST").expect("CLICKHOUSE_TEST_HOST"), + port: Some(std::env::var("CLICKHOUSE_TEST_PORT").ok().and_then(|v| v.parse().ok()).unwrap_or(8443)), + user: std::env::var("CLICKHOUSE_TEST_USER").expect("CLICKHOUSE_TEST_USER"), + ssl: DbSslMode::Require, + options: DbOptions { + time_zone: "UTC".into(), + startup_script: "SET max_threads = 2".into(), + ..Default::default() + }, + properties: vec![DbProperty { name: "max_block_size".into(), value: "50".into() }], + ..Default::default() + }; + let password = std::env::var("CLICKHOUSE_TEST_PASSWORD").expect("CLICKHOUSE_TEST_PASSWORD"); + let pool = Pool::connect(&source, Some(&password), None).await.unwrap(); + let cancel = CancellationToken::new(); + let settings = pool + .execute("SELECT getSetting('max_threads'), getSetting('max_block_size'), timezone()", 1, &cancel) + .await + .unwrap(); + assert_eq!(settings.rows, vec![vec![Some("2".into()), Some("50".into()), Some("UTC".into())]]); + let result = pool.execute("SELECT toUInt64('18446744073709551615') AS n, CAST(NULL AS Nullable(String)) AS missing, CAST('ᴺᵁᴸᴸ' AS Nullable(String)) AS text, 'a\\nb' AS lines", 1, &cancel).await.unwrap(); + assert_eq!( + result.rows, + vec![vec![Some("18446744073709551615".into()), None, Some("ᴺᵁᴸᴸ".into()), Some("a\nb".into())]] + ); + assert_eq!(result.columns[1].type_name, "Nullable(String)"); + let limited = pool.execute("SELECT number FROM numbers(100)", 2, &cancel).await.unwrap(); + assert_eq!(limited.rows.len(), 2); + assert!(limited.has_more); + let marker = format!("transport_cancel_{}", uuid::Uuid::new_v4().simple()); + let sql = format!("SELECT sleepEachRow(0.05) FROM numbers(50) /* {marker} */"); + let stop = CancellationToken::new(); + let query = pool.execute(&sql, 1, &stop); + let cancel_query = async { + tokio::time::sleep(Duration::from_millis(500)).await; + stop.cancel(); + }; + let (cancelled, ()) = tokio::join!(query, cancel_query); + let cancelled = cancelled.unwrap_err(); + assert_eq!(cancelled.kind, ErrorKind::Cancelled); + assert_eq!(cancelled.message, "cancelled"); + let active = pool.execute(&format!("SELECT count() FROM system.processes WHERE position(query, '{marker}') > 0 AND query_id != currentQueryID()"), 1, &cancel).await.unwrap(); + assert_eq!(active.rows, vec![vec![Some("0".into())]]); + let table = format!("transport_test_{}", uuid::Uuid::new_v4().simple()); + pool.execute(&format!("CREATE TEMPORARY TABLE {table} (n UInt8)"), 0, &cancel).await.unwrap(); + let continuity = async { + pool.execute(&format!("INSERT INTO {table} VALUES (7)"), 0, &cancel).await?; + pool.execute(&format!("SELECT n FROM {table}"), 1, &cancel).await + } + .await; + let cleanup = pool.execute(&format!("DROP TEMPORARY TABLE {table}"), 0, &cancel).await; + assert_eq!(continuity.unwrap().rows, vec![vec![Some("7".into())]]); + cleanup.unwrap(); + pool.close().await; + assert!(pool.is_closed()); + } +} diff --git a/crates/ide-db/src/conn.rs b/crates/ide-db/src/conn.rs index aa9fd7f4..94bd0134 100644 --- a/crates/ide-db/src/conn.rs +++ b/crates/ide-db/src/conn.rs @@ -106,6 +106,7 @@ pub enum Pool { Postgres(PgPool), Mysql(MySqlPool), SqlServer(Bb8Pool), + Clickhouse(crate::clickhouse::ClickhousePool), } /// Runs `$body` with `$p` bound to the concrete pool of each driver. @@ -115,7 +116,7 @@ macro_rules! with_pool { Pool::Sqlite($p) => $body, Pool::Postgres($p) => $body, Pool::Mysql($p) => $body, - Pool::SqlServer(_) => unreachable!(), + Pool::SqlServer(_) | Pool::Clickhouse(_) => unreachable!(), } }; } @@ -503,6 +504,9 @@ impl Setup { value.parse::().map_err(|_| IdeError::invalid(format!("{what} must be a whole number: {value}"))) }; match (source.driver, name.to_ascii_lowercase().as_str()) { + (DbDriver::Clickhouse, "statement_cache_capacity") => { + return Err(IdeError::invalid("ClickHouse HTTP does not support statement caching")); + } (_, "statement_cache_capacity") => s.statement_cache = Some(number(&name)? as usize), (_, "connect_timeout") => s.connect_timeout = Some(Duration::from_secs(number(&name)?)), (DbDriver::Postgres, "application_name") => s.application_name = Some(value.to_owned()), @@ -512,6 +516,9 @@ impl Setup { _ => s.params.push((name, value.to_owned())), } } + if source.driver == DbDriver::Clickhouse && !o.time_zone.trim().is_empty() { + s.params.push(("session_timezone".into(), o.time_zone.trim().into())); + } let script = o.startup_script.trim(); if !script.is_empty() { s.statements.extend(crate::split::split(script, source.driver).into_iter().map(|st| st.text)); @@ -527,6 +534,7 @@ impl Pool { Pool::Postgres(_) => DbDriver::Postgres, Pool::Mysql(_) => DbDriver::Mysql, Pool::SqlServer(_) => DbDriver::SqlServer, + Pool::Clickhouse(_) => DbDriver::Clickhouse, } } @@ -534,6 +542,9 @@ impl Pool { pub async fn connect(source: &DataSource, password: Option<&str>, root: Option<&Path>) -> IdeResult { const MAX: u32 = 4; let setup = Setup::of(source)?; + if source.driver == DbDriver::Clickhouse { + return crate::clickhouse::ClickhousePool::connect(source, password, setup).await.map(Pool::Clickhouse); + } let timeout = setup.connect_timeout.unwrap_or(Duration::from_secs(30)); let init = Arc::new(setup.statements); // Runs the session statements (startup script…) on every new connection. @@ -695,12 +706,14 @@ impl Pool { conn.query("SELECT 1", &[]).await.map_err(mssql_err)?.into_results().await.map_err(mssql_err)?; Ok(Pool::SqlServer(pool)) } + DbDriver::Clickhouse => unreachable!(), } } pub async fn close(&self) { match self { Pool::SqlServer(_) => {} + Pool::Clickhouse(p) => p.close(), _ => with_pool!(self, p => p.close().await), } } @@ -708,12 +721,16 @@ impl Pool { pub fn is_closed(&self) -> bool { match self { Pool::SqlServer(_) => false, + Pool::Clickhouse(p) => p.is_closed(), _ => with_pool!(self, p => p.is_closed()), } } /// Runs `sql` (no row limit beyond `max`), for introspection: cells are text (see [`Cells::Text`]). pub async fn rows(&self, sql: &str, max: usize) -> IdeResult>>> { + if let Pool::Clickhouse(p) = self { + return Ok(p.execute(sql, max, &CancellationToken::new()).await?.rows); + } let fetched: IdeResult = match self { Pool::SqlServer(p) => { let mut conn = p.get_owned().await.map_err(mssql_pool_err)?; @@ -727,6 +744,9 @@ impl Pool { /// Runs one statement and keeps at most `max_rows` rows. Cancelling stops /// the statement on the server (PostgreSQL, MySQL) and drops the connection. pub async fn execute(&self, sql: &str, max_rows: usize, cancel: &CancellationToken) -> IdeResult { + if let Pool::Clickhouse(p) = self { + return p.execute(sql, max_rows, cancel).await; + } let mut conn = Held::acquire(self).await?; let r = conn.execute(self, sql, max_rows, cancel).await; if cancel.is_cancelled() { @@ -740,6 +760,7 @@ impl Pool { /// row when flagged (table editor submits). pub async fn transaction(&self, statements: &[(String, bool)]) -> IdeResult { match self { + Pool::Clickhouse(_) => Err(crate::clickhouse::transaction_error()), Pool::SqlServer(p) => { let mut conn = p.get_owned().await.map_err(mssql_pool_err)?; let result = async { @@ -790,6 +811,7 @@ pub enum Held { Postgres(PoolConnection), Mysql(PoolConnection), SqlServer(Box>), + Clickhouse(crate::clickhouse::ClickhousePool), } impl Held { @@ -799,6 +821,12 @@ impl Held { Pool::Postgres(p) => Held::Postgres(p.acquire().await.map_err(db_err)?), Pool::Mysql(p) => Held::Mysql(p.acquire().await.map_err(db_err)?), Pool::SqlServer(p) => Held::SqlServer(Box::new(p.get_owned().await.map_err(mssql_pool_err)?)), + Pool::Clickhouse(p) => { + if p.is_closed() { + return Err(IdeError::new(ErrorKind::Io, "ClickHouse connection is closed")); + } + Held::Clickhouse(p.clone()) + } }) } @@ -806,13 +834,14 @@ impl Held { pub async fn command(&mut self, sql: &str) -> IdeResult<()> { match self { Held::SqlServer(c) => mssql_command(c, sql).await, + Held::Clickhouse(_) => Err(crate::clickhouse::transaction_error()), other => { let sql = sqlx::AssertSqlSafe(sql.to_owned()); match other { Held::Sqlite(c) => sqlx::raw_sql(sql).execute(&mut **c).await.map(drop), Held::Postgres(c) => sqlx::raw_sql(sql).execute(&mut **c).await.map(drop), Held::Mysql(c) => sqlx::raw_sql(sql).execute(&mut **c).await.map(drop), - Held::SqlServer(_) => unreachable!(), + Held::SqlServer(_) | Held::Clickhouse(_) => unreachable!(), } .map_err(db_err) } @@ -833,6 +862,7 @@ impl Held { Held::SqlServer(mut c) => { c.0.take(); } + Held::Clickhouse(_) => {} } } @@ -845,6 +875,9 @@ impl Held { max_rows: usize, cancel: &CancellationToken, ) -> IdeResult { + if let Held::Clickhouse(c) = self { + return c.execute(sql, max_rows, cancel).await; + } let started = Instant::now(); // `$id`: the query naming the server process, `$kill`: how another connection stops it. macro_rules! run { @@ -887,6 +920,7 @@ impl Held { () = cancel.cancelled() => return Err(IdeError::cancelled()), } } + Held::Clickhouse(_) => unreachable!(), }; let (columns, rows, affected, has_more, first) = fetched; let elapsed_ms = started.elapsed().as_secs_f64() * 1000.0; @@ -907,7 +941,7 @@ pub fn begin_statement(driver: DbDriver) -> &'static str { match driver { DbDriver::Mysql => "START TRANSACTION", DbDriver::SqlServer => "BEGIN TRAN", - DbDriver::Sqlite | DbDriver::Postgres => "BEGIN", + DbDriver::Sqlite | DbDriver::Postgres | DbDriver::Clickhouse => "BEGIN", } } diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index 5f7034c5..f12f6a63 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -2,6 +2,7 @@ //! execution ([`conn`]), introspection, table data paging / editing ([`sql`]) //! and statement splitting ([`split`]), for SQLite, PostgreSQL and MySQL/MariaDB. +mod clickhouse; pub mod conn; pub mod history; pub mod plan; @@ -547,12 +548,14 @@ impl DbManager { })?; let version = match source.driver { DbDriver::Sqlite => "SELECT 'SQLite ' || sqlite_version()", - DbDriver::Postgres => "SELECT version()", + DbDriver::Postgres | DbDriver::Clickhouse => "SELECT version()", DbDriver::Mysql => "SELECT CONCAT(@@version_comment, ' ', VERSION())", DbDriver::SqlServer => "SELECT @@VERSION", }; - let v = pool.rows(version, 1).await.ok().and_then(|r| r.into_iter().next()?.into_iter().next()?); + let result = pool.rows(version, 1).await; pool.close().await; + let rows = if source.driver == DbDriver::Clickhouse { result? } else { result.unwrap_or_default() }; + let v = rows.into_iter().next().and_then(|r| r.into_iter().next().flatten()); Ok(v.unwrap_or_else(|| "Connected".into())) } @@ -701,8 +704,11 @@ impl DbManager { _ => indexes.push(IndexInfo { name, unique: flag(&unique), columns: vec![col] }), } } - let fk_rows = - pool.rows(&sql::table_foreign_keys(driver, schema, table), MAX_META_ROWS).await.unwrap_or_default(); + let fk_rows = if driver == DbDriver::Clickhouse { + Vec::new() + } else { + pool.rows(&sql::table_foreign_keys(driver, schema, table), MAX_META_ROWS).await.unwrap_or_default() + }; let mut foreign_keys = group_foreign_keys(fk_rows, &[]); // SQLite keys that reference the primary key implicitly: look it up. for k in foreign_keys.iter_mut().filter(|k| k.ref_columns.is_empty()) { @@ -715,13 +721,15 @@ impl DbManager { .collect(); } foreign_keys.retain(|k| k.columns.len() == k.ref_columns.len()); - let triggers = pool - .rows(&sql::triggers(driver, schema, table), MAX_META_ROWS) - .await - .unwrap_or_else(|e| { + let trigger_rows = if driver == DbDriver::Clickhouse { + Vec::new() + } else { + pool.rows(&sql::triggers(driver, schema, table), MAX_META_ROWS).await.unwrap_or_else(|e| { tracing::warn!("triggers: {}", e.message); Vec::new() }) + }; + let triggers = trigger_rows .into_iter() .filter_map(|r| { let mut it = r.into_iter(); @@ -822,6 +830,11 @@ impl DbManager { let ddl = cells(pool.rows(&q, 1).await?, 1).into_iter().next(); ddl.map(|d| format!("{d};\n")).ok_or_else(missing) } + DbDriver::Clickhouse => { + let q = format!("SHOW CREATE TABLE {}", sql::qualified(driver, schema, name)); + let ddl = cells(pool.rows(&q, 1).await?, 0).into_iter().next(); + ddl.map(|d| format!("{};\n", d.trim_end().trim_end_matches(';'))).ok_or_else(missing) + } DbDriver::SqlServer => { let obj = format!("{}.{}", schema, name).replace('\'', "''"); let ddl = cells(pool.rows(&format!("SELECT OBJECT_DEFINITION(OBJECT_ID('{}'))", obj), 1).await?, 0) @@ -902,10 +915,14 @@ impl DbManager { } } // Foreign keys are a nicety (JOIN completion): a server that refuses the catalog query still completes. - let fk_rows = pool.rows(sql::foreign_keys(driver), MAX_META_ROWS).await.unwrap_or_else(|e| { - tracing::warn!("foreign keys: {}", e.message); + let fk_rows = if driver == DbDriver::Clickhouse { Vec::new() - }); + } else { + pool.rows(sql::foreign_keys(driver), MAX_META_ROWS).await.unwrap_or_else(|e| { + tracing::warn!("foreign keys: {}", e.message); + Vec::new() + }) + }; let foreign_keys = group_foreign_keys(fk_rows, &tables); let default_schema = match sql::current_schema(driver) { None => "main".to_owned(), @@ -980,6 +997,9 @@ impl DbManager { let Some(session) = session else { return pool.execute(statement, max_rows, cancel).await; }; + if pool.driver() == DbDriver::Clickhouse { + return Err(IdeError::invalid("ClickHouse manual transactions are unsupported")); + } let mut held = session.lock().await; if held.is_none() { let mut conn = Held::acquire(&pool).await?; @@ -998,8 +1018,11 @@ impl DbManager { /// Switches `id` between auto-commit and manual commit. Leaving manual /// mode rolls back what was not committed. - pub async fn set_manual_commit(&self, id: &str, manual: bool) { + pub async fn set_manual_commit(&self, id: &str, manual: bool) -> IdeResult<()> { if manual { + if self.source(id)?.driver == DbDriver::Clickhouse { + return Err(IdeError::invalid("ClickHouse manual transactions are unsupported")); + } self.sessions.lock().entry(id.to_owned()).or_default(); } else { let session = self.sessions.lock().remove(id); @@ -1007,10 +1030,14 @@ impl DbManager { end_session(&s).await; } } + Ok(()) } /// Commits (or rolls back) the open transaction of a manual-commit data source. pub async fn finish_transaction(&self, id: &str, commit: bool) -> IdeResult<()> { + if self.source(id).is_ok_and(|s| s.driver == DbDriver::Clickhouse) { + return Err(IdeError::invalid("ClickHouse manual transactions are unsupported")); + } let session = self.sessions.lock().get(id).cloned(); let Some(session) = session else { return Ok(()) }; let mut held = session.lock().await; @@ -1052,6 +1079,9 @@ impl DbManager { /// Applies the table editor's pending changes in one transaction. pub async fn apply_edits(&self, id: &str, schema: &str, table: &str, edits: &[DbRowEdit]) -> IdeResult { + if self.source(id)?.driver == DbDriver::Clickhouse { + return Err(IdeError::invalid("ClickHouse table editing is unsupported")); + } let pool = self.pool(id).await?; let driver = pool.driver(); let statements = edits @@ -1064,3 +1094,37 @@ impl DbManager { pool.transaction(&statements).await } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn clickhouse_rejects_manual_transactions_and_table_edits_without_connecting() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let ssh = Arc::new(SshManager::new(root.join("ssh-passwords.json"))); + let db = DbManager::new(root.clone(), root.join("passwords.json"), root.join("history.json"), ssh); + db.save_sources(vec![DataSource { + id: "clickhouse".into(), + name: "clickhouse".into(), + driver: DbDriver::Clickhouse, + host: "127.0.0.1".into(), + port: Some(1), + ..Default::default() + }]) + .await + .unwrap(); + let error = db.set_manual_commit("clickhouse", true).await.unwrap_err(); + assert!(error.message.contains("manual transactions are unsupported")); + assert!(db.sessions.lock().is_empty()); + for commit in [false, true] { + let error = db.finish_transaction("clickhouse", commit).await.unwrap_err(); + assert!(error.message.contains("manual transactions are unsupported")); + } + let error = db.apply_edits("clickhouse", "default", "events", &[]).await.unwrap_err(); + assert!(error.message.contains("table editing is unsupported")); + assert!(!db.is_connected("clickhouse")); + db.set_manual_commit("clickhouse", false).await.unwrap(); + } +} diff --git a/crates/ide-db/src/plan.rs b/crates/ide-db/src/plan.rs index af378913..32b217b6 100644 --- a/crates/ide-db/src/plan.rs +++ b/crates/ide-db/src/plan.rs @@ -35,6 +35,7 @@ pub fn explain_statement(driver: DbDriver, statement: &str) -> String { match driver { DbDriver::Sqlite => format!("EXPLAIN QUERY PLAN {s}"), DbDriver::Postgres => format!("EXPLAIN (FORMAT JSON) {s}"), + DbDriver::Clickhouse => format!("EXPLAIN json = 1 {s}"), DbDriver::Mysql => format!("EXPLAIN {s}"), DbDriver::SqlServer => format!("SET SHOWPLAN_TEXT ON; {s}"), } @@ -44,7 +45,7 @@ pub fn explain_statement(driver: DbDriver, statement: &str) -> String { pub fn parse(driver: DbDriver, result: &QueryResult) -> IdeResult> { match driver { DbDriver::Sqlite => Ok(sqlite(&result.rows)), - DbDriver::Postgres => { + DbDriver::Postgres | DbDriver::Clickhouse => { let text: String = result.rows.iter().filter_map(|r| r.first().cloned().flatten()).collect::>().join("\n"); let json: Value = serde_json::from_str(&text).map_err(|e| IdeError::invalid(format!("plan: {e}")))?; @@ -199,6 +200,7 @@ mod tests { assert_eq!(explain_statement(DbDriver::Sqlite, "select 1;"), "EXPLAIN QUERY PLAN select 1"); assert_eq!(explain_statement(DbDriver::Postgres, " select 1 "), "EXPLAIN (FORMAT JSON) select 1"); assert_eq!(explain_statement(DbDriver::Mysql, "select 1"), "EXPLAIN select 1"); + assert_eq!(explain_statement(DbDriver::Clickhouse, " select 1;"), "EXPLAIN json = 1 select 1"); assert_eq!(explain_statement(DbDriver::SqlServer, "select 1"), "SET SHOWPLAN_TEXT ON; select 1"); } @@ -254,6 +256,21 @@ mod tests { assert_eq!((plan[3].parent, plan[3].operation.as_str()), (Some(2), "Seq Scan")); } + #[test] + fn reads_clickhouse_json_plans_and_rejects_invalid_json() { + let json = r#"[{"Plan": {"Node Type": "Expression", "Node Id": "Expression_5", + "Description": "Project names", "Plans": [ + {"Node Type": "ReadFromMergeTree", "Node Id": "ReadFromMergeTree_0", + "Description": "default.events"} + ]}}]"#; + let plan = parse(DbDriver::Clickhouse, &result(&["explain"], vec![vec![Some(json)]])).unwrap(); + assert_eq!(plan.len(), 2); + assert_eq!((plan[0].parent, plan[0].operation.as_str()), (None, "Expression")); + assert_eq!((plan[1].parent, plan[1].operation.as_str()), (Some(0), "ReadFromMergeTree")); + assert!(plan[1].properties.iter().any(|p| p.name == "Description" && p.value == "default.events")); + assert!(parse(DbDriver::Clickhouse, &result(&["explain"], vec![vec![Some("invalid")]])).is_err()); + } + #[test] fn reads_mysql_rows() { let r = result( diff --git a/crates/ide-db/src/source.rs b/crates/ide-db/src/source.rs index a3590ae6..c048430d 100644 --- a/crates/ide-db/src/source.rs +++ b/crates/ide-db/src/source.rs @@ -18,6 +18,7 @@ pub enum DbDriver { Mysql, /// Microsoft SQL Server. SqlServer, + Clickhouse, } impl DbDriver { @@ -27,6 +28,7 @@ impl DbDriver { DbDriver::Postgres => Some(5432), DbDriver::Mysql => Some(3306), DbDriver::SqlServer => Some(1433), + DbDriver::Clickhouse => Some(8123), } } } @@ -253,6 +255,7 @@ pub fn scheme_driver(scheme: &str) -> Option { "mysql" | "mariadb" | "tidb" | "singlestore" | "memsql" | "oceanbase" => Some(DbDriver::Mysql), "sqlserver" | "mssql" | "mssqlserver" => Some(DbDriver::SqlServer), "sqlite" | "sqlite3" => Some(DbDriver::Sqlite), + "clickhouse" | "http" | "https" => Some(DbDriver::Clickhouse), _ => None, } } @@ -260,7 +263,7 @@ pub fn scheme_driver(scheme: &str) -> Option { /// Parses `scheme://[user[:password]@]host[:port][/database][?k=v&…]`, `jdbc:` forms /// included, and `sqlite:path`. pub fn parse_url(raw: &str) -> IdeResult { - let bad = || IdeError::invalid(format!("not a database URL: {raw}")); + let bad = || IdeError::invalid("invalid database URL"); let url = raw.trim(); let url = url.strip_prefix("jdbc:").or_else(|| url.strip_prefix("JDBC:")).unwrap_or(url); let (scheme, rest) = url.split_once(':').ok_or_else(bad)?; @@ -367,6 +370,15 @@ impl DataSource { } properties.append(&mut s.properties); s.properties = properties; + if s.driver == DbDriver::Clickhouse { + let raw_url = self.url.trim().to_ascii_lowercase(); + let url = raw_url.strip_prefix("jdbc:").unwrap_or(&raw_url); + if url.starts_with("https://") { + s.ssl = DbSslMode::VerifyFull; + } else if url.starts_with("http://") { + s.ssl = DbSslMode::Disable; + } + } Ok(s) } @@ -519,4 +531,31 @@ mod tests { assert!(DataSource { url: "mysql://h/db".into(), ..s.clone() }.resolved().is_err()); assert_eq!(DataSource { url_only: false, ..s }.resolved().unwrap().host, "localhost"); } + + #[test] + fn clickhouse_urls_resolve_http_https_and_credentials() { + assert_eq!(DbDriver::Clickhouse.default_port(), Some(8123)); + let source = DataSource { + driver: DbDriver::Clickhouse, + url_only: true, + url: "https://reader:s%40cret@db.example:8443/analytics?max_execution_time=30".into(), + ..Default::default() + }; + let resolved = source.resolved().unwrap(); + assert_eq!(resolved.host, "db.example"); + assert_eq!(resolved.port, Some(8443)); + assert_eq!(resolved.database, "analytics"); + assert_eq!(resolved.user, "reader"); + assert_eq!(resolved.ssl, DbSslMode::VerifyFull); + assert_eq!(source.url_password().as_deref(), Some("s@cret")); + assert_eq!(resolved.properties[0].name, "max_execution_time"); + let http = DataSource { url: "http://localhost/default".into(), ..source.clone() }.resolved().unwrap(); + assert_eq!(http.ssl, DbSslMode::Disable); + let jdbc = DataSource { url: "jdbc:clickhouse://localhost/analytics".into(), ..source.clone() }; + assert_eq!(jdbc.resolved().unwrap().database, "analytics"); + let secure = DataSource { url: "https://db.example/default?sslmode=disable".into(), ..source }; + assert_eq!(secure.resolved().unwrap().ssl, DbSslMode::VerifyFull); + let error = parse_url("clickhouse://reader:secret@db.example:invalid/default").unwrap_err(); + assert!(!error.message.contains("secret")); + } } diff --git a/crates/ide-db/src/split.rs b/crates/ide-db/src/split.rs index e9cdbc84..b2ee3a10 100644 --- a/crates/ide-db/src/split.rs +++ b/crates/ide-db/src/split.rs @@ -28,6 +28,7 @@ enum State { /// Every statement of `text` that contains more than whitespace and comments. pub fn split(text: &str, driver: DbDriver) -> Vec { let mysql = driver == DbDriver::Mysql; + let clickhouse = driver == DbDriver::Clickhouse; let chars: Vec<(usize, char)> = text.char_indices().collect(); // UTF-16 offset of each char index (plus the end). let mut u16_at = Vec::with_capacity(chars.len() + 1); @@ -81,7 +82,7 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { state = State::Double; has_code = true; } - '`' if mysql => { + '`' if mysql || clickhouse => { state = State::Backtick; has_code = true; } @@ -117,7 +118,7 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { _ => {} }, State::Single => { - if mysql && c == '\\' { + if (mysql || clickhouse) && c == '\\' { i += 1; } else if c == '\'' { if next == Some('\'') { @@ -128,7 +129,9 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { } } State::Double => { - if c == '"' { + if clickhouse && c == '\\' { + i += 1; + } else if c == '"' { if next == Some('"') { i += 1; } else { @@ -137,7 +140,9 @@ pub fn split(text: &str, driver: DbDriver) -> Vec { } } State::Backtick => { - if c == '`' { + if clickhouse && (c == '\\' || (c == '`' && next == Some('`'))) { + i += 1; + } else if c == '`' { state = State::Code; } } @@ -224,6 +229,12 @@ mod tests { assert_eq!(texts(sql, DbDriver::Mysql), vec!["select `a;b`, 'it\\'s;' from t # x;y", "select 2"]); } + #[test] + fn clickhouse_backticks_and_escaped_quotes_keep_statements_intact() { + let query = r#"SELECT `a\`;b`, "c\";d", 'it\'s;safe'; SELECT 2"#; + assert_eq!(texts(query, DbDriver::Clickhouse), vec![r#"SELECT `a\`;b`, "c\";d", 'it\'s;safe'"#, "SELECT 2"]); + } + #[test] fn postgres_dollar_quotes() { let sql = diff --git a/crates/ide-db/src/sql.rs b/crates/ide-db/src/sql.rs index 79d6c1ec..bd281a85 100644 --- a/crates/ide-db/src/sql.rs +++ b/crates/ide-db/src/sql.rs @@ -7,6 +7,7 @@ use specta::Type; pub fn quote_ident(driver: DbDriver, name: &str) -> String { match driver { + DbDriver::Clickhouse => format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")), DbDriver::Mysql => format!("`{}`", name.replace('`', "``")), DbDriver::SqlServer => format!("[{}]", name.replace(']', "]]")), _ => format!("\"{}\"", name.replace('"', "\"\"")), @@ -15,7 +16,7 @@ pub fn quote_ident(driver: DbDriver, name: &str) -> String { pub fn quote_literal(driver: DbDriver, value: &str) -> String { let mut s = value.replace('\'', "''"); - if driver == DbDriver::Mysql { + if matches!(driver, DbDriver::Mysql | DbDriver::Clickhouse) { s = s.replace('\\', "\\\\"); } format!("'{s}'") @@ -48,6 +49,10 @@ pub const INTERNAL_DATABASES: &[&str] = &["rdsadmin", "azure_maintenance", "azur pub fn schemas(driver: DbDriver, system: bool) -> &'static str { match driver { DbDriver::Sqlite => "SELECT name FROM pragma_database_list WHERE name <> 'temp' ORDER BY seq", + DbDriver::Clickhouse if system => "SELECT name FROM system.databases ORDER BY name", + DbDriver::Clickhouse => { + "SELECT name FROM system.databases WHERE lower(name) NOT IN ('system', 'information_schema') ORDER BY name" + } DbDriver::Postgres if system => { "SELECT schema_name FROM information_schema.schemata \ WHERE schema_name NOT LIKE 'pg\\_toast%' AND schema_name NOT LIKE 'pg\\_temp%' ORDER BY 1" @@ -73,6 +78,12 @@ pub fn schemas(driver: DbDriver, system: bool) -> &'static str { /// Rows: `name, kind` where kind is `table`, `view` or `materialized view`. pub fn tables(driver: DbDriver, schema: &str) -> String { match driver { + DbDriver::Clickhouse => format!( + "SELECT name, CASE engine WHEN 'View' THEN 'view' WHEN 'MaterializedView' THEN 'materialized view' \ + WHEN 'WindowView' THEN 'view' ELSE 'table' END FROM system.tables \ + WHERE database = {} AND is_temporary = 0 ORDER BY name", + quote_literal(driver, schema) + ), DbDriver::Sqlite => format!( "SELECT name, type FROM {}.sqlite_master WHERE type IN ('table', 'view') \ AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' ORDER BY name", @@ -101,6 +112,12 @@ pub fn tables(driver: DbDriver, schema: &str) -> String { pub fn columns(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => format!( + "SELECT name, type, CASE WHEN startsWith(type, 'Nullable(') \ + OR startsWith(type, 'LowCardinality(Nullable(') THEN 'YES' ELSE 'NO' END, \ + if(default_kind = '', NULL, default_expression), is_in_primary_key \ + FROM system.columns WHERE database = {s} AND table = {t} ORDER BY position" + ), DbDriver::Sqlite => format!( "SELECT name, type, CASE WHEN \"notnull\" = 0 THEN 'YES' ELSE 'NO' END, dflt_value, \ CASE WHEN pk > 0 THEN 1 ELSE 0 END FROM pragma_table_info({t}, {s}) ORDER BY cid" @@ -138,6 +155,12 @@ pub fn columns(driver: DbDriver, schema: &str, table: &str) -> String { pub fn indexes(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => format!( + "SELECT * FROM (SELECT 'PRIMARY KEY' AS index_name, 0 AS is_unique, primary_key AS expression \ + FROM system.tables WHERE database = {s} AND name = {t} AND primary_key != '' \ + UNION ALL SELECT name, 0, expr FROM system.data_skipping_indices \ + WHERE database = {s} AND table = {t}) ORDER BY index_name" + ), DbDriver::Sqlite => format!( "SELECT il.name, il.\"unique\", ii.name FROM pragma_index_list({t}, {s}) il \ JOIN pragma_index_info(il.name, {s}) ii ORDER BY il.name, ii.seqno" @@ -169,6 +192,15 @@ pub fn indexes(driver: DbDriver, schema: &str, table: &str) -> String { /// for SQL completion, grouped by table in column order. pub fn completion_columns(driver: DbDriver) -> &'static str { match driver { + DbDriver::Clickhouse => { + "SELECT c.database, c.table, CASE t.engine WHEN 'View' THEN 'view' \ + WHEN 'MaterializedView' THEN 'materialized view' WHEN 'WindowView' THEN 'view' ELSE 'table' END, \ + c.name, c.type, CASE WHEN startsWith(c.type, 'Nullable(') \ + OR startsWith(c.type, 'LowCardinality(Nullable(') THEN 'YES' ELSE 'NO' END, c.is_in_primary_key, 1 \ + FROM system.columns c JOIN system.tables t ON t.database = c.database AND t.name = c.table \ + WHERE lower(c.database) NOT IN ('system', 'information_schema') AND t.is_temporary = 0 \ + ORDER BY c.database, c.table, c.position LIMIT 50000" + } DbDriver::Sqlite => { "SELECT 'main', m.name, m.type, p.name, p.type, CASE WHEN p.\"notnull\" = 0 THEN 'YES' ELSE 'NO' END, \ CASE WHEN p.pk > 0 THEN 1 ELSE 0 END, \ @@ -221,6 +253,7 @@ pub fn completion_columns(driver: DbDriver) -> &'static str { /// when the key references the primary key implicitly. pub fn foreign_keys(driver: DbDriver) -> &'static str { match driver { + DbDriver::Clickhouse => "SELECT '', '', '', '', '', '', '' WHERE 0", DbDriver::Sqlite => { "SELECT 'main', m.name, f.id, f.\"from\", 'main', f.\"table\", f.\"to\" \ FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f \ @@ -258,6 +291,7 @@ pub fn foreign_keys(driver: DbDriver) -> &'static str { pub fn table_foreign_keys(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => foreign_keys(driver).to_owned(), DbDriver::Sqlite => format!( "SELECT {s}, {t}, f.id, f.\"from\", {s}, f.\"table\", f.\"to\" FROM pragma_foreign_key_list({t}, {s}) f ORDER BY f.id, f.seq" ), @@ -276,6 +310,7 @@ pub fn table_foreign_keys(driver: DbDriver, schema: &str, table: &str) -> String /// The schema unqualified names resolve to (`None`: always `main`). pub fn current_schema(driver: DbDriver) -> Option<&'static str> { match driver { + DbDriver::Clickhouse => Some("SELECT currentDatabase()"), DbDriver::Sqlite => None, DbDriver::Postgres => Some("SELECT current_schema()"), DbDriver::Mysql => Some("SELECT DATABASE()"), @@ -362,6 +397,9 @@ fn where_key(driver: DbDriver, key: &[CellValue]) -> String { /// The DML statement for `edit`. Values are written as quoted literals, which /// every supported database coerces to the column type. pub fn edit_statement(driver: DbDriver, schema: &str, table: &str, edit: &DbRowEdit) -> Result { + if driver == DbDriver::Clickhouse { + return Err("ClickHouse table editing is unsupported: primary keys do not uniquely identify rows and edits require transactions".into()); + } let target = qualified(driver, schema, table); match edit { DbRowEdit::Update { key, values } => { @@ -450,6 +488,7 @@ pub fn postgres_ddl_indexes(oid: u32) -> String { pub fn triggers(driver: DbDriver, schema: &str, table: &str) -> String { let (s, t) = (quote_literal(driver, schema), quote_literal(driver, table)); match driver { + DbDriver::Clickhouse => "SELECT '', '', '' WHERE 0".to_owned(), DbDriver::Sqlite => format!( "SELECT name, sql FROM {}.sqlite_master WHERE type = 'trigger' AND tbl_name = {t} ORDER BY name", quote_ident(driver, schema) @@ -492,7 +531,7 @@ pub fn sqlite_trigger_event(sql: &str) -> (String, String) { pub fn routines(driver: DbDriver, schema: &str) -> Option { let s = quote_literal(driver, schema); match driver { - DbDriver::Sqlite => None, + DbDriver::Sqlite | DbDriver::Clickhouse => None, // Functions of extensions are left out, as in IDEA. DbDriver::Postgres => Some(format!( "SELECT p.proname, CASE WHEN p.prokind = 'p' THEN 'procedure' ELSE 'function' END, \ @@ -625,6 +664,71 @@ mod tests { assert_eq!(quote_literal(DbDriver::Mysql, "it's \\"), "'it''s \\\\'"); } + #[test] + fn clickhouse_escapes_backslashes_and_delimiters() { + let driver = DbDriver::Clickhouse; + assert_eq!(quote_literal(driver, r"path\O'Brien"), r"'path\\O''Brien'"); + assert_eq!(quote_literal(driver, r"\' OR 1 = 1 --"), r"'\\'' OR 1 = 1 --'"); + assert_eq!(quote_ident(driver, r"a\`b"), r"`a\\\`b`"); + assert_eq!(qualified(driver, "data base", "event`log"), r"`data base`.`event\`log`"); + } + + #[test] + fn clickhouse_metadata_preserves_types_defaults_and_nonunique_keys() { + let driver = DbDriver::Clickhouse; + assert_eq!(schemas(driver, true), "SELECT name FROM system.databases ORDER BY name"); + assert!(schemas(driver, false).contains("lower(name) NOT IN ('system', 'information_schema')")); + assert_eq!(current_schema(driver), Some("SELECT currentDatabase()")); + let relations = tables(driver, "event's\\db"); + assert!(relations.contains("WHEN 'MaterializedView' THEN 'materialized view'")); + assert!(relations.contains(r"WHERE database = 'event''s\\db' AND is_temporary = 0")); + let fields = columns(driver, "default", "events"); + assert!(fields.starts_with("SELECT name, type, CASE WHEN startsWith(type, 'Nullable(')")); + assert!(fields.contains("startsWith(type, 'LowCardinality(Nullable(')")); + assert!(fields.contains("if(default_kind = '', NULL, default_expression), is_in_primary_key")); + assert!(fields.ends_with("WHERE database = 'default' AND table = 'events' ORDER BY position")); + let keys = indexes(driver, "default", "events"); + assert!(keys.contains("'PRIMARY KEY' AS index_name, 0 AS is_unique, primary_key AS expression")); + assert!(keys.contains("SELECT name, 0, expr FROM system.data_skipping_indices")); + let completion = completion_columns(driver); + assert!(completion.contains("c.is_in_primary_key, 1")); + assert!(completion.contains("ORDER BY c.database, c.table, c.position LIMIT 50000")); + } + + #[test] + fn clickhouse_has_no_foreign_key_trigger_or_schema_routine_catalogs() { + let driver = DbDriver::Clickhouse; + assert_eq!(foreign_keys(driver), "SELECT '', '', '', '', '', '', '' WHERE 0"); + assert_eq!(table_foreign_keys(driver, "default", "events"), foreign_keys(driver)); + assert_eq!(triggers(driver, "default", "events"), "SELECT '', '', '' WHERE 0"); + assert!(routines(driver, "default").is_none()); + assert!(sequences(driver, "default").is_none()); + assert!(object_ddl(driver, "default", crate::DbObjectKind::Trigger, "t").is_none()); + } + + #[test] + fn clickhouse_pages_quote_keys_and_fetch_one_extra_row() { + let sort = [SortKey { column: "event`id".into(), descending: true }]; + assert_eq!( + table_page(DbDriver::Clickhouse, "default", "events", Some("WHERE id > 3"), &sort, 50, 25), + r"SELECT * FROM `default`.`events` WHERE id > 3 ORDER BY `event\`id` DESC LIMIT 26 OFFSET 50" + ); + } + + #[test] + fn clickhouse_keys_never_enable_transactional_table_edits() { + let key = vec![cv("id", Some("1"))]; + for edit in [ + DbRowEdit::Update { key: key.clone(), values: vec![cv("name", Some("new"))] }, + DbRowEdit::Delete { key }, + DbRowEdit::Insert { values: vec![cv("name", Some("new"))] }, + DbRowEdit::Insert { values: vec![] }, + ] { + let error = edit_statement(DbDriver::Clickhouse, "default", "events", &edit).unwrap_err(); + assert!(error.contains("primary keys do not uniquely identify rows")); + } + } + #[test] fn pages() { let sort = [SortKey { column: "id".into(), descending: true }]; diff --git a/crates/ide-db/tests/clickhouse.rs b/crates/ide-db/tests/clickhouse.rs new file mode 100644 index 00000000..d19a9ff1 --- /dev/null +++ b/crates/ide-db/tests/clickhouse.rs @@ -0,0 +1,136 @@ +use ide_core::{CancellationToken, IdeError, IdeResult}; +use ide_db::{CellValue, DataSource, DbDriver, DbManager, DbRowEdit, DbSslMode, SortKey, sql}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn check(condition: bool, message: &str) -> IdeResult<()> { + if condition { Ok(()) } else { Err(IdeError::invalid(message)) } +} + +#[tokio::test] +#[ignore = "requires CLICKHOUSE_TEST_HOST, CLICKHOUSE_TEST_USER, and CLICKHOUSE_TEST_PASSWORD"] +async fn clickhouse_cloud_manager_catalogs_and_session_table() -> IdeResult<()> { + let tmp = tempfile::tempdir()?; + let root = tmp.path().to_path_buf(); + let ssh = Arc::new(ide_ssh::SshManager::new(root.join("ssh-passwords.json"))); + let db = DbManager::new(root.clone(), root.join("passwords.json"), root.join("history.json"), ssh); + let mut source = DataSource { + id: "clickhouse".into(), + name: "clickhouse".into(), + driver: DbDriver::Clickhouse, + host: std::env::var("CLICKHOUSE_TEST_HOST").expect("set CLICKHOUSE_TEST_HOST"), + port: Some( + std::env::var("CLICKHOUSE_TEST_PORT") + .map(|value| value.parse().expect("numeric CLICKHOUSE_TEST_PORT")) + .unwrap_or(8443), + ), + user: std::env::var("CLICKHOUSE_TEST_USER").expect("set CLICKHOUSE_TEST_USER"), + database: "default".into(), + ssl: DbSslMode::VerifyFull, + ..Default::default() + }; + source.introspection.show_system = true; + let password = std::env::var("CLICKHOUSE_TEST_PASSWORD").expect("set CLICKHOUSE_TEST_PASSWORD"); + check(!db.test(&source, Some(password.clone()), None).await?.is_empty(), "missing server version")?; + db.save_sources(vec![source]).await?; + db.set_password("clickhouse", Some(password), false).await?; + let cancel = CancellationToken::new(); + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let name = format!("manager_test_{}_{suffix}", std::process::id()); + let table = sql::quote_ident(DbDriver::Clickhouse, &name); + db.execute( + "clickhouse", + &format!( + "CREATE TEMPORARY TABLE {table} (id UInt64, label Nullable(String) DEFAULT NULL, \ + category LowCardinality(Nullable(String)) DEFAULT NULL, items Array(Nullable(UInt8)), \ + label_size UInt64 ALIAS length(ifNull(label, ''))) ENGINE = Memory" + ), + 0, + &cancel, + ) + .await?; + let checked: IdeResult<()> = async { + db.execute( + "clickhouse", + &format!("INSERT INTO {table} (id, label, items) VALUES (1, 'first', [1, NULL]), (1, NULL, [])"), + 0, + &cancel, + ) + .await?; + let schemas = db.schemas("clickhouse").await?; + check(schemas.iter().any(|schema| schema == "default"), "default database missing")?; + check(schemas.iter().any(|schema| schema == "system"), "system database missing")?; + let tables = db.tables("clickhouse", "system").await?; + check(tables.iter().any(|table| table.name == "one"), "system table listing missing one")?; + let details = db.table_details("clickhouse", "", &name).await?; + check(details.columns.len() == 5, "temporary table columns missing")?; + for (name, data_type, nullable) in [ + ("id", "UInt64", false), + ("label", "Nullable(String)", true), + ("category", "LowCardinality(Nullable(String))", true), + ("items", "Array(Nullable(UInt8))", false), + ] { + check( + details + .columns + .iter() + .any(|column| column.name == name && column.data_type == data_type && column.nullable == nullable), + "column type or nullability mismatch", + )?; + } + check( + details.columns.iter().any(|column| column.name == "label" && column.default.as_deref() == Some("NULL")), + "explicit default expression missing", + )?; + check(details.indexes.iter().all(|index| !index.unique), "ClickHouse index reported unique")?; + check(details.foreign_keys.is_empty() && details.triggers.is_empty(), "unsupported table objects returned")?; + let sort = [SortKey { column: "id".into(), descending: false }]; + let first = db.table_page("clickhouse", "", &name, None, &sort, 0, 1, &cancel).await?; + check(first.result.rows.len() == 1 && first.result.has_more, "first page limit failed")?; + let second = db.table_page("clickhouse", "", &name, None, &sort, 1, 1, &cancel).await?; + check(second.result.rows.len() == 1 && !second.result.has_more, "page offset failed")?; + let ddl = db.ddl("clickhouse", "", &name).await?; + check(ddl.contains(&name) && ddl.contains("Memory"), "temporary table DDL missing")?; + let completion = db.completion("clickhouse").await?; + check(completion.default_schema == "default", "completion default database mismatch")?; + check(completion.foreign_keys.is_empty(), "completion returned unsupported foreign keys")?; + if let Some(table) = completion.tables.first() { + let details = db.table_details("clickhouse", &table.schema, &table.name).await?; + check( + table.columns.iter().all(|column| { + details.columns.iter().any(|field| field.name == column.name && field.data_type == column.data_type) + }), + "completion columns differ from catalog", + )?; + } + let plan = db.explain("clickhouse", &format!("SELECT id FROM {table}"), &cancel).await?; + check(!plan.is_empty(), "query plan missing")?; + let value = r"path\O'Brien"; + let result = db + .execute("clickhouse", &format!("SELECT {}", sql::quote_literal(DbDriver::Clickhouse, value)), 1, &cancel) + .await?; + check(result.rows == vec![vec![Some(value.to_owned())]], "quoted literal changed value")?; + let edits = [DbRowEdit::Update { + key: vec![CellValue { column: "id".into(), value: Some("1".into()) }], + values: vec![CellValue { column: "label".into(), value: Some("changed".into()) }], + }]; + check( + db.apply_edits("clickhouse", "", &name, &edits) + .await + .err() + .is_some_and(|error| error.message.contains("table editing is unsupported")), + "supplied row key authorized table editing", + )?; + check(db.set_manual_commit("clickhouse", true).await.is_err(), "manual transactions were accepted")?; + for commit in [false, true] { + check(db.finish_transaction("clickhouse", commit).await.is_err(), "transaction control was accepted")?; + } + Ok(()) + } + .await; + let cleanup = db.execute("clickhouse", &format!("DROP TEMPORARY TABLE {table}"), 0, &cancel).await; + db.disconnect("clickhouse").await; + checked?; + cleanup?; + Ok(()) +} diff --git a/crates/ide-db/tests/sqlite.rs b/crates/ide-db/tests/sqlite.rs index f340ee53..b800cd15 100644 --- a/crates/ide-db/tests/sqlite.rs +++ b/crates/ide-db/tests/sqlite.rs @@ -197,7 +197,7 @@ async fn count(db: &DbManager) -> String { async fn manual_commit_keeps_a_transaction_until_commit_or_rollback() -> IdeResult<()> { let (_tmp, db) = setup().await; let c = CancellationToken::new(); - db.set_manual_commit("s1", true).await; + db.set_manual_commit("s1", true).await.unwrap(); db.execute("s1", "INSERT INTO users (name) VALUES ('dee')", 1, &c).await?; // The session sees its own change. assert_eq!(count(&db).await, "4"); @@ -208,10 +208,10 @@ async fn manual_commit_keeps_a_transaction_until_commit_or_rollback() -> IdeResu db.finish_transaction("s1", true).await?; // Leaving manual mode rolls back what is still open. db.execute("s1", "DELETE FROM users", 1, &c).await?; - db.set_manual_commit("s1", false).await; + db.set_manual_commit("s1", false).await.unwrap(); assert_eq!(count(&db).await, "4"); // Disconnecting with an open transaction does not hang. - db.set_manual_commit("s1", true).await; + db.set_manual_commit("s1", true).await.unwrap(); db.execute("s1", "DELETE FROM users", 1, &c).await?; db.disconnect("s1").await; assert_eq!(count(&db).await, "4"); diff --git a/crates/ide-host/src/ops/db.rs b/crates/ide-host/src/ops/db.rs index 63c8419a..102bf274 100644 --- a/crates/ide-host/src/ops/db.rs +++ b/crates/ide-host/src/ops/db.rs @@ -216,8 +216,7 @@ pub async fn connect(host: &Host, window: &str, id: &str) -> IdeResult<()> { /// manual mode rolls back what was not committed). pub async fn set_manual_commit(host: &Host, window: &str, id: &str, manual: bool) -> IdeResult<()> { let db = host.project(window)?.db.clone(); - db.set_manual_commit(id, manual).await; - Ok(()) + db.set_manual_commit(id, manual).await } /// Commits or rolls back the open transaction of a manual-commit data source. diff --git a/e2e/clickhouse-source.spec.ts b/e2e/clickhouse-source.spec.ts new file mode 100644 index 00000000..8a29e09e --- /dev/null +++ b/e2e/clickhouse-source.spec.ts @@ -0,0 +1,42 @@ +import { expect, test } from "./fixtures"; + +test("configures a ClickHouse HTTP source and keeps transaction controls unavailable", async ({ + workbench: page, +}, info) => { + await page.evaluate(async () => { + const storePath = "/src/features/database/store.ts"; + const bindingsPath = "/src/bindings.ts"; + const { useDatabase } = await import(storePath); + const { commands } = await import(bindingsPath); + commands.dbTest = async (source: unknown) => { + document.documentElement.dataset.testedSource = JSON.stringify(source); + return { status: "ok", data: "ClickHouse" }; + }; + useDatabase.setState({ dialog: { kind: "source", source: null } }); + }); + const dialog = page.getByRole("dialog", { name: "New Data Source", exact: true }); + await dialog.getByRole("button", { name: "Data source type", exact: true }).click(); + await page.getByRole("option", { name: "ClickHouse", exact: true }).click(); + await expect(dialog.locator("#db-port")).toHaveValue("8123"); + await expect(dialog.locator("#db-user")).toHaveValue("default"); + await expect(dialog.locator("#db-database")).toHaveValue("default"); + await expect(dialog.locator("#db-url")).toHaveValue("http://localhost:8123/default"); + await dialog.locator("#db-url").fill("https://reader@db.example:8443/analytics"); + await expect(dialog.locator("#db-host")).toHaveValue("db.example"); + await expect(dialog.locator("#db-port")).toHaveValue("8443"); + await expect(dialog.locator("#db-user")).toHaveValue("reader"); + await expect(dialog.locator("#db-database")).toHaveValue("analytics"); + await dialog.getByRole("button", { name: "Test Connection", exact: true }).click(); + await expect(dialog.getByRole("status")).toHaveText("Succeeded: ClickHouse"); + expect(await page.evaluate(() => JSON.parse(document.documentElement.dataset.testedSource!))).toMatchObject({ + driver: "clickhouse", + host: "db.example", + port: 8443, + user: "reader", + database: "analytics", + ssl: "verifyFull", + }); + await dialog.screenshot({ path: info.outputPath("clickhouse-source.png") }); + await dialog.getByRole("tab", { name: "Options", exact: true }).click(); + await expect(dialog.getByRole("button", { name: "Transaction control", exact: true })).toBeDisabled(); +}); diff --git a/src/bindings.ts b/src/bindings.ts index ab0d32e5..f8cde114 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -2405,7 +2405,7 @@ export type DbDriver = "sqlite" | "postgres" | /** MySQL and MariaDB. */ "mysql" | /** Microsoft SQL Server. */ -"sqlServer"; +"sqlServer" | "clickhouse"; /** `schema.table (columns) REFERENCES ref_schema.ref_table (ref_columns)`. */ export type DbForeignKey = { diff --git a/src/features/database/ObjectDialogs.tsx b/src/features/database/ObjectDialogs.tsx index 08fb56d4..648e7ec1 100644 --- a/src/features/database/ObjectDialogs.tsx +++ b/src/features/database/ObjectDialogs.tsx @@ -269,6 +269,7 @@ const TYPES: Record = { "bit", "varbinary(max)", ], + clickhouse: ["UInt64", "Int64", "Float64", "String", "Bool", "Date", "DateTime", "UUID", "Nullable(String)"], }; interface ModifyProps { @@ -328,7 +329,8 @@ function ModifyTableForm({ const driver = driverOf(id); const [name, setName] = useState(table ?? "table_name"); const [columns, setColumns] = useState(() => { - if (!initial) return [{ ...blankColumn(driver, 1), name: "id", primaryKey: true, notNull: true }]; + if (!initial) + return [{ ...blankColumn(driver, 1), name: "id", primaryKey: driver !== "clickhouse", notNull: true }]; const drafts = initial.map(draftOf); if (addColumn) drafts.push(blankColumn(driver, drafts.length + 1)); return drafts; @@ -448,14 +450,18 @@ function ModifyTableForm({ disabled={current.dropped || locked} onChange={(e) => patch({ dataType: e.target.value })} /> - - patch({ notNull })} - label="Not null" - /> - {!table && ( + {driver !== "clickhouse" && ( + <> + + patch({ notNull })} + label="Not null" + /> + + )} + {!table && driver !== "clickhouse" && ( <> > = [ /** Driver property names the Advanced tab suggests, and what the backend does with them. */ const PROPERTIES: Record = { + clickhouse: { + names: ["max_execution_time", "max_threads", "connect_timeout"], + hint: "ClickHouse settings sent with each HTTP request. connect_timeout is in seconds.", + }, postgres: { names: [ "statement_timeout", @@ -376,6 +380,7 @@ export function SourceDialog({ const vendor = vendorOf(s); const sqlite = s.driver === "sqlite"; const postgres = s.driver === "postgres"; + const clickhouse = s.driver === "clickhouse"; const urlOnly = !sqlite && !!s.urlOnly; const parsedUrl = urlOnly ? parseSourceUrl(s.url ?? "") : null; const urlValid = !!parsedUrl && parsedUrl.source.driver === s.driver; @@ -398,7 +403,20 @@ export function SourceDialog({ setS((x) => { const old = vendorOf(x); const port = x.port == null || x.port === old.port ? v.port : x.port; - return { ...x, driver: v.driver, vendor: v.id === v.driver ? "" : v.id, port, ssl: v.ssl ?? x.ssl }; + return { + ...x, + driver: v.driver, + vendor: v.id === v.driver ? "" : v.id, + port, + ssl: v.ssl ?? (v.driver === "clickhouse" ? "disable" : x.ssl), + ...(v.driver === "clickhouse" + ? { + user: x.user || "default", + database: x.database || "default", + options: { ...blankOptions(), ...x.options, transaction: "auto" as const }, + } + : {}), + }; }); }; @@ -614,8 +632,9 @@ export function SourceDialog({ bound && void setManualCommit(bound, v === "manual")} - disabled={!bound} - /> - {manual && ( + {supportsTransactions && ( + <> + + onChoice(e.target.value as SyncChoice)} - > - {syncChoices(entry).map((c) => ( - - ))} - - - - {entry.path} - - {entry.local ? formatSize(entry.local.size) : ""} - {formatTime(entry.local?.mtime)} - {entry.remote ? formatSize(entry.remote.size) : ""} - {formatTime(entry.remote?.mtime)} - {tx(STATUS_LABELS[entry.status])} - - ); -} - -function SyncDialog() { - const sync = useDeployment((s) => s.sync); - const root = useProject((s) => s.project?.root ?? null); - const tx = useTx(); - const txt = useTxTemplate(); - if (!sync) return null; - const entries = sync.entries; - const shown = entries?.filter((e) => !sync.hideSame || e.status !== "same") ?? []; - const steps = syncSteps(sync.choices); - const same = entries?.filter((e) => e.status === "same").length ?? 0; - const where = sync.paths.map((path) => (root ? relativePath(root, path) || "/" : path)).join(", "); - - return ( - patchSync({ hideSame })} - label={txt(`Hide identical files (${same})`)} - /> - ) - } - buttons={[ - { label: "Cancel", role: "cancel" }, - { - label: "Synchronize", - role: "default", - disabled: !entries || sync.running || steps.length === 0, - onClick: () => void runSync(steps), - }, - ]} - > -
-
- {where} - {entries && <> · {txt(`${steps.length} of ${entries.length} files will change`)}} -
- {!!sync.skipped?.length && ( -
- {txt(`Skipped: ${sync.skipped.length}`)}:{" "} - {sync.skipped.map((s) => (root ? relativePath(root, s.path) : s.path)).join(", ")} -
- )} - {sync.error &&
{sync.error}
} - {!entries && !sync.error && ( -
- Comparing local and remote files… -
- )} - {entries && ( -
- - - - - - - - - - - - - - {shown.length === 0 && ( - - - - )} - {shown.map((e) => ( - patchSync({ choices: { ...sync.choices, [e.path]: c } })} - /> - ))} - -
{tx("Action")}{tx("Path")}{tx("Local size")}{tx("Local modified")}{tx("Remote size")}{tx("Remote modified")}{tx("Status")}
- Local and remote files are identical -
-
- )} - {sync.running && ( -
- Synchronizing… -
- )} -
- - ); -} - -function SecretDialog() { +export function SecretDialog() { const prompt = useDeployment((s) => s.secret); const [secret, setSecret] = useState(""); const [save, setSave] = useState(false); diff --git a/src/features/deployment/SyncDialog.tsx b/src/features/deployment/SyncDialog.tsx new file mode 100644 index 00000000..0d100e5a --- /dev/null +++ b/src/features/deployment/SyncDialog.tsx @@ -0,0 +1,522 @@ +/** Deployment directory diff: mirrored file lists above an embedded content diff. */ +import { useEffect, useMemo, useRef, useState, type KeyboardEvent, type ReactNode } from "react"; +import { getCurrentWebviewWindow } from "@tauri-apps/api/webviewWindow"; +import clsx from "clsx"; +import type { SyncEntry, SyncStatus, SyncStep } from "@/platform/ipc"; +import { useTx, useTxTemplate } from "@/platform/i18n"; +import { useProject } from "@/platform/project"; +import { useWorkflowSettings } from "@/platform/workflowSettings"; +import { + Dialog, + DialogButtons, + Icon, + IconButton, + Input, + Spinner, + SplitGroup, + SplitHandle, + SplitPanel, + T, + fileIconName, + type DialogProps, +} from "@/ui"; +import { TitleBar } from "@/workbench/TitleBar"; +import { usePlatform } from "@/platform/platformInfo"; +import { closeSync, openSync, runSync, type SyncRunMode } from "./actions"; +import { createDeploymentComparison, type DeploymentComparison } from "./comparison"; +import { defaultSyncAction, formatSize, formatTime, syncChoices, syncSteps, type SyncChoice } from "./model"; +import { patchSync, useDeployment, type SyncDialogState } from "./store"; +import { SyncPreview } from "./SyncPreview"; +import styles from "./Deployment.module.css"; + +const STATUS: Record = { + localOnly: "Local only", + remoteOnly: "Remote only", + localNewer: "Local is newer", + remoteNewer: "Remote is newer", + different: "Contents differ", + same: "Identical", +}; +const CHOICES: Record = { + skip: ["—", "Skip"], + upload: ["→", "Upload"], + download: ["←", "Download"], + deleteRemote: ["× →", "Delete remote"], + deleteLocal: ["← ×", "Delete local"], +}; +type CompareBy = "metadata" | "content"; +const NO_VERIFIED: Record = {}; +const NO_ERRORS: Record = {}; +interface ContentScan { + comparison: DeploymentComparison; + entries: SyncEntry[]; + verified: Record; + errors: Record; + progress: number | null; +} + +function ComparisonFrame({ standalone, ...props }: DialogProps & { standalone: boolean }) { + const ownTitleBar = usePlatform((s) => s.overlayTitleBar || s.framelessTitleBar); + if (!standalone) return ; + return ( +
+ {ownTitleBar && } +
{props.children}
+
+ {props.footerStart as ReactNode} + +
+
+ ); +} + +function SyncComparison({ + sync, + standalone, + beforeSynchronize, +}: { + sync: SyncDialogState; + standalone: boolean; + beforeSynchronize?: (steps: SyncStep[]) => Promise; +}) { + const root = useProject((s) => s.project?.root ?? null); + const server = useWorkflowSettings((s) => s.deployment.servers.find((item) => item.id === sync.server)); + const tx = useTx(); + const txt = useTxTemplate(); + const [selectedPath, setSelectedPath] = useState(null); + const [filter, setFilter] = useState(""); + const [showLocal, setShowLocal] = useState(true); + const [showDifferent, setShowDifferent] = useState(true); + const [showRemote, setShowRemote] = useState(true); + const [compareBy, setCompareBy] = useState("metadata"); + const [scan, setScan] = useState(null); + const [generation, setGeneration] = useState(0); + const touched = useRef(new Set()); + const synchronizing = useRef(false); + const comparison = useMemo( + () => (server && root ? { ...createDeploymentComparison(server, root), generation, entries: sync.entries } : null), + [server, root, generation, sync.entries], + ); + const entries = sync.entries; + const activeScan = + compareBy === "content" && scan?.comparison === comparison && scan.entries === entries ? scan : null; + const verified = activeScan?.verified ?? NO_VERIFIED; + const checkErrors = activeScan?.errors ?? NO_ERRORS; + const progress = compareBy === "content" && entries && comparison ? (activeScan ? activeScan.progress : 0) : null; + const changeCompareBy = (mode: CompareBy) => { + setScan(null); + setCompareBy(mode); + }; + + useEffect(() => { + const abort = new AbortController(); + if (!entries || !comparison) return; + const suggest = (entry: SyncEntry, status: SyncStatus) => { + const current = useDeployment.getState().sync; + if (current?.entries !== entries || touched.current.has(entry.path)) return; + patchSync({ choices: { ...current.choices, [entry.path]: defaultSyncAction(status) } }); + }; + if (compareBy === "metadata") { + entries.forEach((entry) => suggest(entry, entry.status)); + return; + } + void (async () => { + await Promise.resolve(); + const verified: Record = {}; + const errors: Record = {}; + for (let i = 0; i < entries.length; i++) { + if (abort.signal.aborted) return; + const entry = entries[i]!; + try { + if (entry.local && entry.remote) { + const equal = await comparison.equal(entry, abort.signal); + if (abort.signal.aborted) return; + verified[entry.path] = equal; + suggest(entry, equal ? "same" : entry.status === "same" ? "different" : entry.status); + } + } catch (e) { + if (abort.signal.aborted) return; + errors[entry.path] = e instanceof Error ? e.message : String(e); + } + if (abort.signal.aborted) return; + setScan({ comparison, entries, verified: { ...verified }, errors: { ...errors }, progress: i + 1 }); + } + if (!abort.signal.aborted) setScan({ comparison, entries, verified, errors, progress: null }); + })(); + return () => abort.abort(); + }, [entries, comparison, compareBy]); + + const compared = useMemo( + () => + entries?.map((entry) => + verified[entry.path] === undefined + ? entry + : { + ...entry, + status: verified[entry.path] ? ("same" as const) : ("different" as const), + }, + ) ?? [], + [entries, verified], + ); + const shown = useMemo(() => { + const query = filter.trim().toLocaleLowerCase(); + return compared.filter( + (entry) => + entry.path.toLocaleLowerCase().includes(query) && + (checkErrors[entry.path] || + (entry.status === "same" + ? !sync.hideSame + : entry.status === "localOnly" + ? showLocal + : entry.status === "remoteOnly" + ? showRemote + : showDifferent)), + ); + }, [compared, filter, checkErrors, sync.hideSame, showLocal, showRemote, showDifferent]); + const selected = shown.find((entry) => entry.path === selectedPath) ?? shown[0] ?? null; + const previewEntry = entries?.find((entry) => entry.path === selected?.path) ?? null; + const steps = syncSteps(sync.choices); + const selectedSteps = steps.filter((step) => step.path === selected?.path); + const busy = sync.running || !entries || progress !== null; + const synchronize = async (mode: SyncRunMode) => { + const selectedOperations = mode === "selected" ? selectedSteps : steps; + if (busy || synchronizing.current || !selectedOperations.length) return; + synchronizing.current = true; + if (mode === "selected") selectedOperations.forEach((step) => touched.current.add(step.path)); + try { + if (beforeSynchronize) { + patchSync({ running: true }); + await beforeSynchronize(selectedOperations); + if (useDeployment.getState().sync?.paths !== sync.paths) return; + } + const report = await (mode === "selected" ? runSync(selectedOperations, mode) : runSync(selectedOperations)); + if (mode === "selected" && report?.files && !report.errors.length && !report.skipped) { + const index = shown.findIndex((entry) => entry.path === selected?.path); + const choices = useDeployment.getState().sync?.choices; + const next = [...shown.slice(index + 1), ...shown.slice(0, index)].find( + (entry) => choices?.[entry.path] && choices[entry.path] !== "skip", + ); + if (next) setSelectedPath((current) => (current === selectedPath ? next.path : current)); + } + } catch (error) { + if (useDeployment.getState().sync?.paths === sync.paths) + patchSync({ running: false, error: error instanceof Error ? error.message : String(error) }); + } finally { + synchronizing.current = false; + } + }; + const choose = (entry: SyncEntry, choice: SyncChoice) => { + if (sync.running) return; + touched.current.add(entry.path); + setSelectedPath(entry.path); + patchSync({ choices: { ...useDeployment.getState().sync?.choices, [entry.path]: choice } }); + }; + const onTableKey = (event: KeyboardEvent) => { + if ((event.target as HTMLElement).closest("select, button, input")) return; + const index = shown.findIndex((entry) => entry.path === selected?.path); + if (event.key === "ArrowDown" || event.key === "ArrowUp") { + event.preventDefault(); + const next = shown[Math.max(0, Math.min(shown.length - 1, index + (event.key === "ArrowDown" ? 1 : -1)))]; + if (next) { + setSelectedPath(next.path); + document.getElementById(`deploy-row-${shown.indexOf(next)}`)?.scrollIntoView({ block: "nearest" }); + } + } else if ( + event.key === "Enter" && + !event.altKey && + !event.ctrlKey && + !event.metaKey && + !event.shiftKey && + !event.nativeEvent.isComposing + ) { + event.preventDefault(); + void synchronize("selected"); + } else if (event.key === " " && selected && !busy) { + event.preventDefault(); + const choices = syncChoices(selected); + choose(selected, choices[(choices.indexOf(sync.choices[selected.path] ?? "skip") + 1) % choices.length]!); + } + }; + const refresh = () => { + if (!server || busy) return; + touched.current.clear(); + setGeneration((n) => n + 1); + void openSync(server, sync.paths); + }; + + return ( + void getCurrentWebviewWindow().close() : closeSync} + width="calc(100vw - 48px)" + className={styles.syncWindow} + resizable="both" + footerStart={ + {entries && txt(`${steps.length} of ${entries.length} files will change`)} + } + buttons={[ + { label: "Cancel", role: "cancel" }, + { + label: "Synchronize All", + role: "extra", + disabled: busy || steps.length === 0, + onClick: () => void synchronize("all"), + }, + ]} + > +
+
+ + + + + + + + + + void synchronize("selected")} + /> + {progress !== null && ( + <> + + + {txt(`Comparing contents: ${progress} of ${entries?.length ?? 0}`)} + + changeCompareBy("metadata")} + /> + + )} + +
+
+ + + {root ?? tx("Local files")} + + + + + {sync.serverName}: {server?.rootPath || "/"} + + +
+ {sync.error && ( +
+ {sync.error} +
+ )} + {!!sync.skipped?.length && ( +
+ {txt(`Skipped: ${sync.skipped.length}`)}: {sync.skipped.map((item) => item.path).join(", ")} +
+ )} + {!entries && !sync.error && ( +
+ + Comparing local and remote files… +
+ )} + {sync.running && ( +
+ + Synchronizing… +
+ )} + {entries && ( + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + {!shown.length && ( + + + + )} + {shown.map((entry, index) => ( + { + setSelectedPath(entry.path); + if (!(event.target as HTMLElement).closest("select, button, input")) + event.currentTarget.closest("table")?.focus({ preventScroll: true }); + }} + > + + + + + + + + + ))} + +
{tx("Local name")}{tx("Size")}{tx("Date")}↔{tx("Date")}{tx("Size")}{tx("Remote name")}
+ No matching differences +
+ {entry.local && ( + + + {entry.path} + + )} + {entry.local ? formatSize(entry.local.size) : ""}{formatTime(entry.local?.mtime)} + + {formatTime(entry.remote?.mtime)}{entry.remote ? formatSize(entry.remote.size) : ""} + {entry.remote && ( + + + {entry.path} + + )} +
+
+
+ + Space changes the selected operation. Enter synchronizes the selected file. Synchronize All applies + all chosen operations. + +
+
+ + + + {selected && checkErrors[selected.path] && ( +
+ {checkErrors[selected.path]} +
+ )} + {comparison ? ( + + ) : ( +
+ The deployment server or project is no longer available +
+ )} + + + )} +
+ + ); +} + +export function SyncDialog({ + standalone = false, + beforeSynchronize, +}: { + standalone?: boolean; + beforeSynchronize?: (steps: SyncStep[]) => Promise; +}) { + const sync = useDeployment((s) => s.sync); + return sync ? : null; +} diff --git a/src/features/deployment/SyncPreview.test.tsx b/src/features/deployment/SyncPreview.test.tsx new file mode 100644 index 00000000..6765e852 --- /dev/null +++ b/src/features/deployment/SyncPreview.test.tsx @@ -0,0 +1,72 @@ +import { act, fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import type { SyncEntry } from "@/platform/ipc"; +import { registerEditorKind } from "@/workbench/contributions"; +import type { DeploymentComparison, DeploymentPreview } from "./comparison"; +import { SyncPreview } from "./SyncPreview"; + +// The diff viewer comes from the vcs feature through its editor kind. +registerEditorKind({ + kind: "diff", + title: () => "diff", + component: ({ input }) => ( +
+ {JSON.stringify(input.data)} +
+ ), +}); + +const entry = (path: string) => ({ path, remotePath: `/srv/${path}` }) as SyncEntry; +const result = (text: string): DeploymentPreview => ({ + kind: "text", + data: { + path: "/work/file.php", + title: "file.php", + left: { type: "text", text: "local", label: "Local" }, + right: { type: "text", text, label: "Remote" }, + }, +}); + +describe("deployment diff preview", () => { + it("ignores a late response for a file that is no longer selected", async () => { + let finish!: (value: DeploymentPreview) => void; + const preview = vi + .fn() + .mockReturnValueOnce( + new Promise((resolve) => { + finish = resolve; + }), + ) + .mockResolvedValue(result("current")); + const comparison: DeploymentComparison = { preview, equal: vi.fn() }; + const mounted = render(); + const firstSignal = preview.mock.calls[0]![1] as AbortSignal; + mounted.rerender(); + expect(firstSignal.aborted).toBe(true); + expect(await screen.findByTestId("diff")).toHaveTextContent("current"); + await act(async () => finish(result("stale"))); + expect(screen.getByTestId("diff")).not.toHaveTextContent("stale"); + expect(screen.getByTestId("diff")).toHaveAttribute("data-mode", "split"); + }); + it("shows read errors and allows retrying without closing the comparison", async () => { + const preview = vi.fn().mockRejectedValueOnce(new Error("Offline")).mockResolvedValue(result("recovered")); + render(); + expect(await screen.findByRole("alert")).toHaveTextContent("Offline"); + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByTestId("diff")).toHaveTextContent("recovered"); + }); + it.each(["binary", "tooLarge"] as const)("explains why %s files have no text preview", async (kind) => { + render( + , + ); + expect( + await screen.findByText( + kind === "binary" ? "Binary files cannot be shown as text" : /too large for the text preview/, + ), + ).toBeInTheDocument(); + expect(screen.queryByTestId("diff")).not.toBeInTheDocument(); + }); +}); diff --git a/src/features/deployment/SyncPreview.tsx b/src/features/deployment/SyncPreview.tsx new file mode 100644 index 00000000..8e009928 --- /dev/null +++ b/src/features/deployment/SyncPreview.tsx @@ -0,0 +1,93 @@ +import { Suspense, useEffect, useState } from "react"; +import type { SyncEntry } from "@/platform/ipc"; +import { txTemplate } from "@/platform/i18n"; +import { Button, Spinner, T } from "@/ui"; +import { useContributions } from "@/workbench/contributions"; +import type { DeploymentComparison, DeploymentDiff, DeploymentPreview } from "./comparison"; +import styles from "./Deployment.module.css"; + +/** An input of the diff viewer (editor kind `diff`), embedded and side by side. */ +function diffInput(data: DeploymentDiff) { + return { + kind: "diff", + key: `deployment:${data.path}`, + path: data.path, + data: { ...data, embedded: true, mode: "split" }, + }; +} + +export function SyncPreview({ entry, comparison }: { entry: SyncEntry | null; comparison: DeploymentComparison }) { + const [loaded, setLoaded] = useState<{ + comparison: DeploymentComparison; + entry: SyncEntry; + attempt: number; + result: DeploymentPreview | null; + error: string | null; + } | null>(null); + const [attempt, setAttempt] = useState(0); + const current = + loaded?.comparison === comparison && loaded.entry === entry && loaded.attempt === attempt ? loaded : null; + const result = current?.result; + const error = current?.error; + const diffViewer = useContributions((s) => s.editorKinds.get("diff")); + useEffect(() => { + const abort = new AbortController(); + if (entry) + void comparison.preview(entry, abort.signal).then( + (value) => { + if (!abort.signal.aborted) setLoaded({ comparison, entry, attempt, result: value, error: null }); + }, + (e: unknown) => { + if (!abort.signal.aborted) + setLoaded({ + comparison, + entry, + attempt, + result: null, + error: txTemplate(e instanceof Error ? e.message : String(e)), + }); + }, + ); + return () => abort.abort(); + }, [entry, comparison, attempt]); + + return ( +
+ {!entry ? ( +
+ Select a file to compare its contents +
+ ) : error ? ( +
+
{error}
+ +
+ ) : !result ? ( +
+ + Loading file contents… +
+ ) : result.kind === "binary" ? ( +
+ Binary files cannot be shown as text +
+ ) : result.kind === "tooLarge" ? ( +
+ This file is too large for the text preview. Use Binary Content to compare it. +
+ ) : ( + + +
+ } + > + {diffViewer && } + + )} + + ); +} diff --git a/src/features/deployment/actions.test.tsx b/src/features/deployment/actions.test.tsx index f3a1ae48..63ab8cdb 100644 --- a/src/features/deployment/actions.test.tsx +++ b/src/features/deployment/actions.test.tsx @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { DeploymentServerValid } from "@/platform/remoteSettings"; +import type { SyncEntry } from "@/platform/ipc"; const mocks = vi.hoisted(() => ({ actions: [] as { id: string; run: (context: { arg: string[] }) => unknown }[], @@ -9,6 +10,17 @@ const mocks = vi.hoisted(() => ({ notify: vi.fn(), upload: vi.fn(), compare: vi.fn(), + sync: vi.fn(), + save: vi.fn(), + release: vi.fn(), + attachedPaths: [] as string[], + desktop: false, + openWindow: vi.fn(), +})); +vi.mock("./windowBridge", () => ({ showDeploymentComparison: mocks.openWindow })); +vi.mock("@/editor/api", () => ({ + editorsFor: (path: string) => (mocks.attachedPaths.includes(path) ? [{}] : []), + attachDocument: async () => ({ save: mocks.save, release: mocks.release }), })); vi.mock("@/platform/actions", () => ({ executeAction: vi.fn(), @@ -38,13 +50,15 @@ vi.mock("@/platform/ipc", () => ({ call: (value: unknown) => value, nextRequestId: () => 1, isCancelled: () => false, + isTauri: () => mocks.desktop, commands: { deployCoverage: async () => mocks.coverage, deployUpload: mocks.upload, deployCompare: mocks.compare, + deploySync: mocks.sync, }, })); -import { registerDeploymentActions } from "./actions"; +import { registerDeploymentActions, runSync } from "./actions"; import { useDeployment } from "./store"; async function run(id: string, paths = ["/project/a", "/project/b"]) { @@ -69,10 +83,53 @@ beforeEach(() => { mocks.choose.mockResolvedValue(null); mocks.upload.mockResolvedValue({ files: 1, skipped: 1, errors: [] }); mocks.compare.mockResolvedValue([]); + mocks.sync.mockResolvedValue({ files: 1, skipped: 0, errors: [] }); + mocks.save.mockResolvedValue(undefined); + mocks.attachedPaths = []; + mocks.desktop = false; + mocks.openWindow.mockResolvedValue(undefined); useDeployment.setState({ sync: null }); registerDeploymentActions(); }); describe("deployment eligibility", () => { + it("keeps upstream's server-specific explanation when only one server is configured", async () => { + mocks.servers = [mocks.servers[0]!]; + mocks.coverage[0]!.paths = []; + await run("deployment.uploadTo"); + expect(mocks.notify).toHaveBeenCalledWith( + expect.objectContaining({ title: "No selected paths can be deployed to one" }), + ); + expect(mocks.notify.mock.calls[0]![0].body.props.report.errors).toEqual(mocks.coverage[0]!.skipped); + }); + it("opens the selected server and paths in a native window on desktop without an in-page comparison", async () => { + mocks.desktop = true; + mocks.choose.mockResolvedValue("one"); + await run("deployment.syncWith"); + await vi.waitFor(() => expect(mocks.openWindow).toHaveBeenCalledWith("one", ["/project/a", "/project/b"])); + expect(useDeployment.getState().sync).toBeNull(); + expect(mocks.compare).not.toHaveBeenCalled(); + }); + it("saves edits made in the local diff before synchronization and stops if saving fails", async () => { + await run("deployment.syncWith"); + mocks.choose.mockResolvedValue("one"); + await run("deployment.syncWith"); + await vi.waitFor(() => expect(useDeployment.getState().sync?.entries).toEqual([])); + const sync = useDeployment.getState().sync!; + mocks.attachedPaths = ["/project/a"]; + await runSync([ + { path: "a", action: "upload" }, + { path: "b", action: "download" }, + ]); + expect(mocks.save).toHaveBeenCalledOnce(); + expect(mocks.release).toHaveBeenCalledOnce(); + expect(mocks.save.mock.invocationCallOrder[0]).toBeLessThan(mocks.sync.mock.invocationCallOrder[0]!); + useDeployment.setState({ sync }); + mocks.sync.mockClear(); + mocks.save.mockRejectedValueOnce(new Error("Cannot save local file")); + await runSync([{ path: "a", action: "upload" }]); + expect(mocks.sync).not.toHaveBeenCalled(); + expect(useDeployment.getState().sync?.error).toBe("Cannot save local file"); + }); it.each(["deployment.uploadTo", "deployment.downloadFrom", "deployment.syncWith"])( "filters %s and keeps a single option visible", async (id) => { @@ -80,17 +137,24 @@ describe("deployment eligibility", () => { expect(mocks.choose).toHaveBeenCalledWith(expect.objectContaining({ items: ["one"] })); }, ); - it("explains an empty match and links settings", async () => { - mocks.coverage[0]!.paths = []; - await run("deployment.uploadTo"); - expect(mocks.choose).not.toHaveBeenCalled(); - expect(mocks.notify).toHaveBeenCalledWith( - expect.objectContaining({ - title: "No deployment server covers the selected paths", - actions: [expect.objectContaining({ label: "Configure…" })], - }), - ); - }); + it.each(["deployment.uploadTo", "deployment.downloadFrom", "deployment.syncWith"])( + "explains the rejected paths for %s and links settings", + async (id) => { + mocks.coverage[0]!.paths = []; + await run(id); + expect(mocks.choose).not.toHaveBeenCalled(); + expect(mocks.notify).toHaveBeenCalledWith( + expect.objectContaining({ + title: "No deployment server covers the selected paths", + actions: [expect.objectContaining({ label: "Configure…" })], + }), + ); + expect(mocks.notify.mock.calls[0]![0].body.props.report.errors).toEqual([ + { path: "/project/b", message: "one: Not covered by this server's mappings" }, + ]); + expect(mocks.compare).not.toHaveBeenCalled(); + }, + ); it("explains why the default cannot deploy the named file", async () => { mocks.coverage[0]!.paths = []; await run("deployment.uploadToDefault"); @@ -104,4 +168,64 @@ describe("deployment eligibility", () => { await vi.waitFor(() => expect(mocks.compare).toHaveBeenCalledTimes(2)); expect(useDeployment.getState().sync?.skipped?.[0]?.path).toBe("/project/b"); }); + it("keeps the comparison and other choices after synchronizing a single file, then supports the next file", async () => { + mocks.choose.mockResolvedValue("one"); + await run("deployment.syncWith"); + await vi.waitFor(() => expect(useDeployment.getState().sync?.entries).toEqual([])); + const entries: SyncEntry[] = ["a", "b"].map((path) => ({ + path, + remotePath: `/remote/${path}`, + local: { size: 10, mtime: 2 }, + remote: { size: 5, mtime: 1 }, + status: "localNewer", + })); + useDeployment.setState((state) => ({ + sync: { ...state.sync!, entries, choices: { a: "upload", b: "deleteRemote" } }, + })); + const refreshed = { ...entries[0]!, remote: entries[0]!.local, status: "same" as const }; + mocks.compare.mockResolvedValueOnce([refreshed]); + await runSync([{ path: "a", action: "upload" }], "selected"); + expect(mocks.sync).toHaveBeenLastCalledWith([{ path: "a", action: "upload" }], "one", 1); + expect(mocks.compare).toHaveBeenLastCalledWith("/project/a", "one", 1); + expect(useDeployment.getState().sync).toMatchObject({ + running: false, + request: null, + choices: { a: "skip", b: "deleteRemote" }, + entries: [refreshed, entries[1]], + }); + mocks.compare.mockResolvedValueOnce([{ ...entries[1]!, remote: null, status: "localOnly" }]); + await runSync([{ path: "b", action: "deleteRemote" }], "selected"); + expect(useDeployment.getState().sync?.choices).toEqual({ a: "skip", b: "skip" }); + expect(useDeployment.getState().sync?.entries).toHaveLength(2); + }); + it("retains failed selected operations for retry and does not change pending files when refresh fails", async () => { + mocks.choose.mockResolvedValue("one"); + await run("deployment.syncWith"); + await vi.waitFor(() => expect(useDeployment.getState().sync?.entries).toEqual([])); + const entries: SyncEntry[] = ["a", "b"].map((path) => ({ + path, + remotePath: `/remote/${path}`, + local: { size: 10, mtime: 2 }, + remote: null, + status: "localOnly", + })); + useDeployment.setState((state) => ({ sync: { ...state.sync!, entries, choices: { a: "upload", b: "upload" } } })); + mocks.sync.mockResolvedValueOnce({ files: 0, skipped: 0, errors: [{ path: "a", message: "Permission denied" }] }); + mocks.compare.mockResolvedValueOnce([entries[0]]); + await runSync([{ path: "a", action: "upload" }], "selected"); + expect(useDeployment.getState().sync).toMatchObject({ + running: false, + choices: { a: "upload", b: "upload" }, + error: "a: Permission denied", + }); + mocks.compare.mockRejectedValueOnce(new Error("Connection lost")); + await runSync([{ path: "a", action: "upload" }], "selected"); + expect(useDeployment.getState().sync).toMatchObject({ + running: false, + request: null, + choices: { a: "skip", b: "upload" }, + entries, + }); + expect(useDeployment.getState().sync?.error).toContain("Could not refresh comparison: a: Connection lost"); + }); }); diff --git a/src/features/deployment/actions.tsx b/src/features/deployment/actions.tsx index 9a9dbcfe..2c558afb 100644 --- a/src/features/deployment/actions.tsx +++ b/src/features/deployment/actions.tsx @@ -1,8 +1,18 @@ /** Deployment actions: upload / download / delete / sync / open the deployed URL. */ import { executeAction, registerActions } from "@/platform/actions"; +import { attachDocument, editorsFor } from "@/editor/api"; import { getContext } from "@/platform/context"; -import { call, commands, isCancelled, nextRequestId, type SyncAction, type TransferReport } from "@/platform/ipc"; -import { txTemplate, useTxTemplate } from "@/platform/i18n"; +import { + call, + commands, + isCancelled, + isTauri, + nextRequestId, + type TransferReport, + type SyncEntry, + type SyncStep, +} from "@/platform/ipc"; +import { tx, txTemplate, useTxTemplate } from "@/platform/i18n"; import { openExternalUrl } from "@/platform/openUrl"; import { relativePath, useProject } from "@/platform/project"; import type { AutoUploadValid, DeploymentServerValid } from "@/platform/remoteSettings"; @@ -14,6 +24,7 @@ import { initialSyncChoices, resolveServer, serverLabel, summarizeReport, target import { withSsh } from "./ssh"; import { REMOTE_HOST_WINDOW, patchSync, useDeployment } from "./store"; import styles from "./Deployment.module.css"; +import { showDeploymentComparison } from "./windowBridge"; const GROUP = "tools"; const CATEGORY = "Deployment"; @@ -90,8 +101,17 @@ async function pickServer(title: string, paths: string[]): Promise coverage.some((c) => c.server === server.id && c.paths.length)); if (!servers.length) { // One server: say why, as an explicit upload to it would. - if (configured.length === 1) notifyNoCoverage(configured[0], coverage[0]?.skipped); - else notifyNoCoverage(); + if (configured.length === 1) + notifyNoCoverage(configured[0], coverage.find((c) => c.server === configured[0]!.id)?.skipped); + else { + const skipped = configured.flatMap((server) => + (coverage.find((c) => c.server === server.id)?.skipped ?? []).map((error) => ({ + ...error, + message: `${serverLabel(server)}: ${error.message}`, + })), + ); + notifyNoCoverage(undefined, skipped); + } return null; } const labels = servers.map(serverLabel); @@ -250,20 +270,86 @@ export function closeSync() { useDeployment.setState({ sync: null }); } -export async function runSync(steps: { path: string; action: SyncAction }[]) { +export type SyncRunMode = "all" | "selected"; + +export async function runSync(steps: SyncStep[], mode: SyncRunMode = "all") { const sync = useDeployment.getState().sync; const server = sync && deployment().servers.find((s) => s.id === sync.server); - if (!sync || !server) return; + if (!sync || !server || !steps.length) return; const request = nextRequestId(); - patchSync({ running: true, request }); + patchSync({ running: true, request, error: null }); try { + // The embedded diff can edit the local side. Flush those live documents + // before uploading, so the server receives the content the user reviewed. + const root = projectRoot(); + for (const step of steps) { + if (step.action !== "upload" || !root) continue; + const path = `${root.replace(/\/$/, "")}/${step.path}`; + if (!editorsFor(path).length) continue; + const document = await attachDocument(path); + try { + await document.save(); + } finally { + document.release(); + } + } + if (useDeployment.getState().sync?.request !== request) return; const report = await withSsh(server.kind === "sftp" ? server.sshConfig : "", () => call(commands.deploySync(steps, server.id, request)), ); - useDeployment.setState({ sync: null }); - report.skipped += sync.skipped.length; + if (useDeployment.getState().sync?.request !== request) return; + if (mode === "selected" && root) { + const refreshed = new Map(); + const checked = new Set(); + const refreshErrors: string[] = []; + for (const step of steps) { + if (useDeployment.getState().sync?.request !== request) return; + try { + const entries = await withSsh(server.kind === "sftp" ? server.sshConfig : "", () => + call(commands.deployCompare(`${root.replace(/\/$/, "")}/${step.path}`, server.id, request)), + ); + checked.add(step.path); + const entry = entries.find((entry) => entry.path === step.path); + if (entry) refreshed.set(step.path, entry); + } catch (error) { + if (isCancelled(error)) throw error; + refreshErrors.push(`${step.path}: ${errorText(error)}`); + } + } + const current = useDeployment.getState().sync; + if (current?.request !== request) return; + const failed = new Set(report.errors.map((error) => error.path)); + // A skipped or interrupted batch does not identify every completed file. + const completed = report.skipped === 0 && report.files === steps.length - failed.size; + const choices = { ...current.choices }; + if (completed) for (const step of steps) if (!failed.has(step.path)) choices[step.path] = "skip"; + const entries = + current.entries?.flatMap((entry) => { + if (!checked.has(entry.path)) return [entry]; + const updated = refreshed.get(entry.path); + if (updated) return [updated]; + delete choices[entry.path]; + return []; + }) ?? null; + patchSync({ + entries, + choices, + running: false, + request: null, + error: + [ + ...report.errors.map((error) => `${error.path}: ${error.message}`), + ...refreshErrors.map((error) => tx`Could not refresh comparison: ${error}`), + ].join("\n") || null, + }); + } else { + useDeployment.setState({ sync: null }); + report.skipped += sync.skipped.length; + } showReport(report, "sync", server); + return report; } catch (e) { + if (useDeployment.getState().sync?.request !== request) return; if (isCancelled(e)) patchSync({ running: false, request: null }); else patchSync({ running: false, request: null, error: errorText(e) }); } @@ -367,7 +453,15 @@ export function registerDeploymentActions(): () => void { const target = paths[0] ?? projectRoot(); if (!target) return; const targets = paths.length ? paths : [target]; - void pickServer("Sync with Deployed to", targets).then((server) => server && openSync(server, targets)); + void pickServer("Sync with Deployed to", targets).then(async (server) => { + if (!server) return; + if (!isTauri()) return openSync(server, targets); + try { + await showDeploymentComparison(server.id, targets); + } catch (error) { + showFailure("Cannot open comparison for {server}", server, error); + } + }); }, }, { diff --git a/src/features/deployment/comparison.test.ts b/src/features/deployment/comparison.test.ts new file mode 100644 index 00000000..6df84c50 --- /dev/null +++ b/src/features/deployment/comparison.test.ts @@ -0,0 +1,117 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SyncEntry } from "@/platform/ipc"; +import { DEFAULT_DEPLOYMENT_SERVER } from "@/platform/remoteSettings"; + +const mocks = vi.hoisted(() => ({ remote: vi.fn(), probe: vi.fn(), text: vi.fn(), range: vi.fn() })); +vi.mock("@/platform/ipc", () => ({ + call: (value: unknown) => value, + commands: { remoteOpen: mocks.remote, fsProbe: mocks.probe, fsReadText: mocks.text, fsReadRange: mocks.range }, +})); +vi.mock("./ssh", () => ({ withSsh: (_: unknown, run: () => unknown) => run() })); +import { createDeploymentComparison } from "./comparison"; + +const server = { ...DEFAULT_DEPLOYMENT_SERVER, id: "s1", name: "Staging", sshConfig: "ssh" }; +const entry = (patch: Partial = {}): SyncEntry => ({ + path: "src/a.php", + remotePath: "/srv/app/src/a.php", + status: "same", + local: { size: 4, mtime: 1 }, + remote: { size: 4, mtime: 1 }, + ...patch, +}); +const signal = () => new AbortController().signal; +beforeEach(() => { + vi.resetAllMocks(); + mocks.remote.mockResolvedValue("/cache/a.php"); + mocks.probe.mockResolvedValue({ size: 4, binary: false, tooLarge: false }); + mocks.text.mockResolvedValue("old\n"); + mocks.range.mockResolvedValue({ total: 4, data: "b2xkCg==" }); +}); + +describe("deployment content snapshots", () => { + it("keeps the local document live and the remote side immutable, sharing one downloaded snapshot", async () => { + const compare = createDeploymentComparison(server, "/work/app/"); + expect(await compare.preview(entry(), signal())).toMatchObject({ + kind: "text", + data: { + path: "/work/app/src/a.php", + left: { type: "file", path: "/work/app/src/a.php" }, + right: { type: "text", text: "old\n", label: "Staging: /srv/app/src/a.php" }, + }, + }); + expect(mocks.remote).toHaveBeenCalledWith({ kind: "sftp", ssh: "ssh" }, "/srv/app/src/a.php"); + expect(mocks.text).toHaveBeenCalledWith("/cache/a.php"); + await compare.equal(entry(), signal()); + expect(mocks.remote).toHaveBeenCalledTimes(1); + }); + it("represents missing sides without reading missing files", async () => { + const compare = createDeploymentComparison(server, "/work/app"); + expect(await compare.preview(entry({ remote: null }), signal())).toMatchObject({ + data: { right: { type: "text", text: "", label: expect.stringContaining("does not exist") } }, + }); + expect(mocks.remote).not.toHaveBeenCalled(); + expect(mocks.text).not.toHaveBeenCalled(); + mocks.probe.mockClear(); + expect(await compare.preview(entry({ local: null }), signal())).toMatchObject({ + data: { left: { type: "text", text: "" }, right: { text: "old\n" } }, + }); + expect(mocks.probe).toHaveBeenCalledExactlyOnceWith("/cache/a.php"); + }); + it("does not decode binaries or download oversized text previews", async () => { + const compare = createDeploymentComparison(server, "/work/app"); + expect(await compare.preview(entry({ local: { size: 20_000_000, mtime: 1 } }), signal())).toEqual({ + kind: "tooLarge", + }); + expect(mocks.remote).not.toHaveBeenCalled(); + mocks.probe.mockResolvedValue({ size: 4, binary: true }); + expect(await compare.preview(entry(), signal())).toEqual({ kind: "binary" }); + expect(mocks.text).not.toHaveBeenCalled(); + }); + it("compares bytes even when size and timestamps match", async () => { + const compare = createDeploymentComparison(server, "/work/app"); + mocks.range.mockImplementation(async (path: string) => ({ + total: 4, + data: path.startsWith("/work") ? "bmV3Cg==" : "b2xkCg==", + })); + expect(await compare.equal(entry(), signal())).toBe(false); + mocks.range.mockResolvedValue({ total: 4, data: "b2xkCg==" }); + expect(await compare.equal(entry({ remote: { size: 4, mtime: 99 } }), signal())).toBe(true); + }); + it("uses bounded reads and stops at the first differing block", async () => { + const compare = createDeploymentComparison(server, "/work/app"); + const large = entry({ local: { size: 150_000, mtime: 1 }, remote: { size: 150_000, mtime: 1 } }); + mocks.range.mockImplementation(async (path: string, offset: number) => ({ + total: 150_000, + data: offset >= 65_536 ? path : "same", + })); + expect(await compare.equal(large, signal())).toBe(false); + expect(mocks.range).toHaveBeenCalledTimes(4); + expect(mocks.range.mock.calls.every(([, , len]) => len === 65_536)).toBe(true); + }); + it("detects changed sizes during comparison and handles empty files", async () => { + const compare = createDeploymentComparison(server, "/work/app"); + mocks.range.mockResolvedValue({ total: 5, data: "changed" }); + await expect(compare.equal(entry(), signal())).rejects.toThrow("changed during comparison"); + mocks.range.mockResolvedValue({ total: 0, data: "" }); + expect(await compare.equal(entry({ local: { size: 0, mtime: 1 }, remote: { size: 0, mtime: 1 } }), signal())).toBe( + true, + ); + mocks.remote.mockClear(); + expect(await compare.equal(entry({ remote: { size: 8, mtime: 1 } }), signal())).toBe(false); + expect(mocks.remote).not.toHaveBeenCalled(); + }); + it("does not continue after selection is cancelled and retries failed downloads", async () => { + const compare = createDeploymentComparison(server, "/work/app"); + const abort = new AbortController(); + mocks.remote.mockImplementationOnce(async () => { + abort.abort(); + return "/cache/a.php"; + }); + await expect(compare.preview(entry(), abort.signal)).rejects.toMatchObject({ name: "AbortError" }); + expect(mocks.text).not.toHaveBeenCalled(); + const retry = createDeploymentComparison(server, "/work/app"); + mocks.remote.mockRejectedValueOnce(new Error("Offline")); + await expect(retry.preview(entry(), signal())).rejects.toThrow("Offline"); + await expect(retry.preview(entry(), signal())).resolves.toMatchObject({ kind: "text" }); + }); +}); diff --git a/src/features/deployment/comparison.ts b/src/features/deployment/comparison.ts new file mode 100644 index 00000000..67b71ce0 --- /dev/null +++ b/src/features/deployment/comparison.ts @@ -0,0 +1,91 @@ +/** Read-only remote snapshots for the deployment comparison; project files are never downloaded over. */ +import { call, commands, type SyncEntry } from "@/platform/ipc"; +import { tx } from "@/platform/i18n"; +import { deploymentConnection, type DeploymentServerValid } from "@/platform/remoteSettings"; +import { withSsh } from "./ssh"; + +const PREVIEW_LIMIT = 8 * 1024 * 1024; +const BLOCK_SIZE = 64 * 1024; + +/** The data of a `diff` editor input (the diff viewer's): the local file, editable, next to the server's text. */ +export interface DeploymentDiff { + path: string; + title: string; + left: { type: "file"; path: string; label: string } | { type: "text"; text: string; label: string }; + right: { type: "text"; text: string; label: string }; +} + +export type DeploymentPreview = { kind: "text"; data: DeploymentDiff } | { kind: "binary" } | { kind: "tooLarge" }; + +export function createDeploymentComparison(server: DeploymentServerValid, root: string) { + // Only the downloaded path is cached, never a live remote document. The diff + // receives immutable text, so its merge controls cannot save to the server. + const snapshots = new Map>(); + const localPath = (entry: SyncEntry) => `${root.replace(/\/$/, "")}/${entry.path}`; + const remoteSnapshot = (entry: SyncEntry) => { + let pending = snapshots.get(entry.remotePath); + if (!pending) { + pending = withSsh(server.kind === "sftp" ? server.sshConfig : "", () => + call(commands.remoteOpen(deploymentConnection(server), entry.remotePath)), + ); + snapshots.set(entry.remotePath, pending); + void pending.catch(() => snapshots.delete(entry.remotePath)); + } + return pending; + }; + + return { + async preview(entry: SyncEntry, signal: AbortSignal): Promise { + signal.throwIfAborted(); + if (Math.max(entry.local?.size ?? 0, entry.remote?.size ?? 0) > PREVIEW_LIMIT) return { kind: "tooLarge" }; + const remote = entry.remote ? await remoteSnapshot(entry) : null; + signal.throwIfAborted(); + const probes = await Promise.all( + [entry.local ? localPath(entry) : null, remote].map((path) => (path ? call(commands.fsProbe(path)) : null)), + ); + signal.throwIfAborted(); + if (probes.some((p) => p?.binary)) return { kind: "binary" }; + if (probes.some((p) => p?.tooLarge || (p?.size ?? 0) > PREVIEW_LIMIT)) return { kind: "tooLarge" }; + const text = remote ? await call(commands.fsReadText(remote)) : ""; + signal.throwIfAborted(); + return { + kind: "text", + data: { + path: localPath(entry), + title: entry.path, + left: entry.local + ? { type: "file", path: localPath(entry), label: tx`Local: ${entry.path}` } + : { type: "text", text: "", label: tx("Local: (does not exist)") }, + right: { + type: "text", + text, + label: `${server.name || server.id}: ${entry.remote ? entry.remotePath : tx("(does not exist)")}`, + }, + }, + }; + }, + + /** Exact byte comparison, including binary files and line endings, in bounded reads. */ + async equal(entry: SyncEntry, signal: AbortSignal): Promise { + signal.throwIfAborted(); + if (!entry.local || !entry.remote || entry.local.size !== entry.remote.size) return false; + const remote = await remoteSnapshot(entry); + signal.throwIfAborted(); + // Read once for empty files too, to detect files changed since the listing. + for (let offset = 0; offset === 0 || offset < entry.local.size; offset += BLOCK_SIZE) { + signal.throwIfAborted(); + const [a, b] = await Promise.all( + [localPath(entry), remote].map((path) => call(commands.fsReadRange(path, offset, BLOCK_SIZE))), + ); + signal.throwIfAborted(); + if (a!.total !== entry.local.size || b!.total !== entry.remote.size) + throw new Error("A file changed during comparison. Refresh and compare again."); + if (a!.data !== b!.data) return false; + if (entry.local.size === 0) break; + } + return true; + }, + }; +} + +export type DeploymentComparison = ReturnType; diff --git a/src/features/deployment/index.ts b/src/features/deployment/index.ts index 31411439..e45ef8f3 100644 --- a/src/features/deployment/index.ts +++ b/src/features/deployment/index.ts @@ -15,6 +15,9 @@ import { autoUploadOn, registerDeploymentActions } from "./actions"; import { DeploymentDialogs } from "./Dialogs"; import { transferMenuEntries } from "./model"; import { REMOTE_HOST_WINDOW, useDeployment } from "./store"; +import { installDeploymentEditorHost } from "./windowBridge"; + +export { activateDeploymentWindow, DeploymentWindow } from "./window"; /** Deployed URL of the path the last menu was prepared for. */ let webUrlFor: { path: string; url: string | null } | null = null; @@ -70,6 +73,7 @@ export function activate(): () => void { unsubFocus, unsubProject, registerDeploymentActions(), + installDeploymentEditorHost(), registerToolWindow({ id: REMOTE_HOST_WINDOW, title: "Remote Host", diff --git a/src/features/deployment/window.test.ts b/src/features/deployment/window.test.ts new file mode 100644 index 00000000..fa314353 --- /dev/null +++ b/src/features/deployment/window.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { useProject } from "@/platform/project"; +import { useDeployment } from "./store"; +import { DEFAULT_DEPLOYMENT_SERVER } from "@/platform/remoteSettings"; + +const mocks = vi.hoisted(() => ({ + launch: vi.fn(), + project: vi.fn(), + open: vi.fn(), + cancel: vi.fn(), + ask: vi.fn(), + error: vi.fn(), + dirty: vi.fn(), + save: vi.fn(), + dispose: vi.fn(), + destroy: vi.fn(), + close: vi.fn(), + title: vi.fn(), + emit: vi.fn(async () => {}), + closeRequested: null as null | ((event: { preventDefault(): void }) => Promise), +})); +vi.mock("@tauri-apps/api/webviewWindow", () => ({ + getCurrentWebviewWindow: () => ({ + label: "deployment-project-2", + listen: async () => () => {}, + onCloseRequested: async (callback: typeof mocks.closeRequested) => { + mocks.closeRequested = callback; + return () => {}; + }, + setTitle: mocks.title, + close: mocks.close, + destroy: mocks.destroy, + }), +})); +vi.mock("@/platform/ipc", () => ({ + isTauri: () => true, + commands: { windowTakeLaunch: mocks.launch, projectCurrent: mocks.project }, +})); +vi.mock("@tauri-apps/api/event", () => ({ emitTo: mocks.emit })); +vi.mock("@/editor/core/embeddedDocuments", () => ({ + activateEmbeddedDocuments: () => ({ dirty: mocks.dirty, save: mocks.save, dispose: mocks.dispose }), +})); +vi.mock("@/workbench/messages", () => ({ ask: mocks.ask, showError: mocks.error })); +vi.mock("./actions", () => ({ openSync: mocks.open, closeSync: mocks.cancel })); +vi.mock("./windowBridge", () => ({ openDeploymentSource: vi.fn() })); +vi.mock("./Dialogs", () => ({ SecretDialog: () => null })); +vi.mock("./SyncDialog", () => ({ SyncDialog: () => null })); +vi.mock("@/workbench/OverlayHost", () => ({ OverlayHost: () => null })); +vi.mock("@/workbench/TitleBar", () => ({ TitleBar: () => null })); + +import { activateDeploymentWindow } from "./window"; + +let stop: (() => void) | undefined; +beforeEach(() => { + vi.clearAllMocks(); + mocks.launch.mockResolvedValue(null); + mocks.dirty.mockReturnValue(false); + mocks.save.mockResolvedValue(undefined); + mocks.dispose.mockResolvedValue(undefined); + mocks.ask.mockResolvedValue({ button: "cancel" }); + useProject.setState({ project: null, settings: { files: { autoSave: false } } as never }); + useDeployment.setState({ sync: null }); +}); +afterEach(() => { + stop?.(); + stop = undefined; +}); + +it("loads the parent project before comparing the launched server and local selection", async () => { + const server = { ...DEFAULT_DEPLOYMENT_SERVER, id: "s2", name: "Second project" }; + const project = { root: "/second", settings: { deployment: { servers: [server] } } }; + mocks.project.mockResolvedValue(project); + mocks.launch.mockResolvedValueOnce({ deployment: { server: "s2", paths: ["/second/a.php"] } }); + mocks.open.mockImplementation(async () => useDeployment.setState({ sync: { server: "s2" } as never })); + stop = activateDeploymentWindow("project-2"); + await vi.waitFor(() => + expect(mocks.open).toHaveBeenCalledWith(expect.objectContaining({ id: "s2" }), ["/second/a.php"]), + ); + expect(useProject.getState().project).toBe(project); +}); + +it("keeps the window and unsaved local edits when closing is cancelled or saving fails", async () => { + mocks.dirty.mockReturnValue(true); + stop = activateDeploymentWindow("project-2"); + const event = { preventDefault: vi.fn() }; + await mocks.closeRequested!(event); + expect(event.preventDefault).toHaveBeenCalled(); + expect(mocks.save).not.toHaveBeenCalled(); + expect(mocks.destroy).not.toHaveBeenCalled(); + // The project window waiting to close stops. + expect(mocks.emit).toHaveBeenCalledWith( + "project-2", + "openidea://auxiliary/closed", + expect.objectContaining({ label: "deployment-project-2", closed: false }), + ); + mocks.ask.mockResolvedValue({ button: "save" }); + mocks.save.mockRejectedValueOnce(new Error("Disk full")); + await mocks.closeRequested!(event); + expect(mocks.error).toHaveBeenCalledWith("Cannot close deployment comparison", "Disk full"); + expect(mocks.dispose).not.toHaveBeenCalled(); + expect(mocks.destroy).not.toHaveBeenCalled(); + await mocks.closeRequested!(event); + expect(mocks.cancel).toHaveBeenCalledOnce(); + expect(mocks.dispose).toHaveBeenCalledOnce(); + expect(mocks.destroy).toHaveBeenCalledOnce(); + expect(mocks.save.mock.invocationCallOrder.at(-1)).toBeLessThan(mocks.destroy.mock.invocationCallOrder[0]!); +}); diff --git a/src/features/deployment/window.tsx b/src/features/deployment/window.tsx new file mode 100644 index 00000000..076febfa --- /dev/null +++ b/src/features/deployment/window.tsx @@ -0,0 +1,187 @@ +/** + * The Sync with Deployed window (`deployment-`): one per project window, sharing that + * project's documents. + */ +import { useSyncExternalStore } from "react"; +import { emitTo } from "@tauri-apps/api/event"; +import { getCurrentWebviewWindow } from "@tauri-apps/api/webviewWindow"; +import { activateEmbeddedDocuments } from "@/editor/core/embeddedDocuments"; +import { AUX_CLOSED, type AuxCloseResult } from "@/platform/auxWindow"; +import { commands, type SyncStep } from "@/platform/ipc"; +import { useProject } from "@/platform/project"; +import { workflowSettings } from "@/platform/workflowSettings"; +import { isImeKey } from "@/platform/ime"; +import { useApplyTheme } from "@/workbench/theme"; +import { OverlayHost } from "@/workbench/OverlayHost"; +import { TitleBar } from "@/workbench/TitleBar"; +import { usePlatform } from "@/platform/platformInfo"; +import { WindowResizeHandles } from "@/workbench/WindowResizeHandles"; +import { provideEditorService } from "@/workbench/contributions"; +import { ask, showError } from "@/workbench/messages"; +import { Spinner, T, isPopupOpen } from "@/ui"; +import { closeSync, openSync } from "./actions"; +import { SecretDialog } from "./Dialogs"; +import { SyncDialog } from "./SyncDialog"; +import { patchSync, useDeployment } from "./store"; +import styles from "./Deployment.module.css"; +import { openDeploymentSource } from "./windowBridge"; + +const LAUNCH_EVENT = "openidea://launch"; + +let startupError: string | null = null; +const startupListeners = new Set<() => void>(); +let saveSyncDocuments: ((steps: SyncStep[]) => Promise) | undefined; +const subscribeStartup = (listener: () => void) => { + startupListeners.add(listener); + return () => { + startupListeners.delete(listener); + }; +}; +function fail(error: unknown) { + if (useDeployment.getState().sync) { + patchSync({ error: error instanceof Error ? error.message : String(error) }); + return; + } + startupError = error instanceof Error ? error.message : String(error); + startupListeners.forEach((listener) => listener()); +} + +export function activateDeploymentWindow(parent: string): () => void { + const native = getCurrentWebviewWindow(); + const documents = activateEmbeddedDocuments(); + provideEditorService({ + openFile: (path, options) => openDeploymentSource(parent, path, options), + openInput: (input, options) => (input.path ? openDeploymentSource(parent, input.path, options) : Promise.resolve()), + activePath: () => null, + activeCursor: () => null, + saveAll: documents.save, + }); + saveSyncDocuments = (steps) => { + const root = useProject.getState().project?.root; + return root + ? documents.savePaths(steps.map((step) => `${root.replace(/\/$/, "")}/${step.path}`)) + : Promise.resolve(); + }; + let disposed = false; + let closing = false; + let queue = Promise.resolve(); + const disposers: Array<() => void> = []; + // Tells the project window this window stays open, so it stops closing (see `closeDocumentWindows`). + const stayOpen = () => { + const result: AuxCloseResult = { label: native.label, closed: false, documents: [] }; + void emitTo(parent, AUX_CLOSED, result).catch(() => {}); + }; + const listen = (ready: Promise<() => void>) => void ready.then((off) => (disposed ? off() : disposers.push(off))); + const launch = () => { + queue = queue + .then(async () => { + if (disposed || closing) return; + const pending = await commands.windowTakeLaunch(); + if (!pending?.deployment) return; + if (useDeployment.getState().sync?.running) { + await showError( + "Deployment comparison", + "Wait for synchronization to finish before opening another comparison.", + ); + return; + } + const project = await commands.projectCurrent(); + if (!project) throw new Error("The project is no longer open."); + useProject.setState({ project, settings: project.settings }); + const server = workflowSettings().deployment.servers.find((item) => item.id === pending.deployment!.server); + if (!server) throw new Error("The deployment server is no longer available."); + await native.setTitle(`Diff Between 'local files' and 'files on ${server.name || server.id}'`); + await openSync(server, pending.deployment.paths); + if (!useDeployment.getState().sync && !closing) + throw new Error("No selected files are mapped to this deployment server."); + }) + .catch(fail); + }; + listen( + native.listen(LAUNCH_EVENT, launch).then((off) => { + launch(); + return off; + }), + ); + listen( + native.onCloseRequested(async (event) => { + event.preventDefault(); + if (closing) return; + closing = true; + try { + if (documents.dirty() && useProject.getState().settings?.files?.autoSave === false) { + const answer = await ask({ + title: "Save local changes?", + body: "Save the local files edited in this comparison before closing.", + buttons: [{ id: "save", label: "Save and Close", primary: true }], + }); + if (answer.button !== "save") return stayOpen(); + } + await documents.save(); + closeSync(); + useDeployment.getState().secret?.resolve(null); + await documents.dispose(); + await native.destroy(); + } catch (error) { + stayOpen(); + await showError("Cannot close deployment comparison", error instanceof Error ? error.message : String(error)); + } finally { + closing = false; + } + }), + ); + disposers.push( + useDeployment.subscribe((s, prev) => { + if (prev.sync && !s.sync && !closing) void native.close(); + }), + ); + const onKey = (event: KeyboardEvent) => { + if (event.defaultPrevented || isImeKey(event)) return; + if ( + (event.key === "Escape" && !isPopupOpen()) || + ((event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "w") + ) { + event.preventDefault(); + void native.close(); + } else if ((event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "s") { + event.preventDefault(); + void documents.save().catch(fail); + } + }; + window.addEventListener("keydown", onKey); + disposers.push(() => window.removeEventListener("keydown", onKey)); + return () => { + disposed = true; + saveSyncDocuments = undefined; + disposers.reverse().forEach((off) => off()); + }; +} + +export function DeploymentWindow() { + useApplyTheme(); + const sync = useDeployment((s) => s.sync); + const error = useSyncExternalStore(subscribeStartup, () => startupError); + const ownTitleBar = usePlatform((s) => s.overlayTitleBar || s.framelessTitleBar); + return ( + <> + {sync ? ( + saveSyncDocuments?.(steps) ?? Promise.resolve()} /> + ) : ( +
+ {ownTitleBar && } +
+ {error || ( + <> + + Opening deployment comparison… + + )} +
+
+ )} + + + + + ); +} diff --git a/src/features/deployment/windowBridge.ts b/src/features/deployment/windowBridge.ts new file mode 100644 index 00000000..9dab0cde --- /dev/null +++ b/src/features/deployment/windowBridge.ts @@ -0,0 +1,40 @@ +import { emitTo } from "@tauri-apps/api/event"; +import { getCurrentWebviewWindow } from "@tauri-apps/api/webviewWindow"; +import { call, commands, isTauri } from "@/platform/ipc"; +import { useProject } from "@/platform/project"; +import { editorService, type OpenOptions } from "@/workbench/contributions"; +import { windowBackgroundHex } from "@/workbench/theme"; + +const OPEN_SOURCE = "openidea://deployment/open-source"; + +export function showDeploymentComparison(server: string, paths: string[]) { + return call(commands.windowOpenDeployment(server, paths, windowBackgroundHex())); +} + +export function openDeploymentSource(parent: string, path: string, options?: OpenOptions) { + return emitTo(parent, OPEN_SOURCE, { path, options }); +} + +/** Jump to Source from the comparison opens the local file in its project's editor. */ +export function installDeploymentEditorHost(): () => void { + if (!isTauri()) return () => {}; + let disposed = false; + let unlisten: (() => void) | undefined; + void getCurrentWebviewWindow() + .listen<{ path: string; options?: OpenOptions }>(OPEN_SOURCE, ({ payload }) => { + const root = useProject.getState().project?.root; + if (root && payload.path.startsWith(`${root}/`)) { + void editorService() + .openFile(payload.path, payload.options) + .then(() => getCurrentWebviewWindow().setFocus()); + } + }) + .then((off) => { + if (disposed) off(); + else unlisten = off; + }); + return () => { + disposed = true; + unlisten?.(); + }; +} diff --git a/src/features/index.ts b/src/features/index.ts index 24df08c5..ec4fa3c7 100644 --- a/src/features/index.ts +++ b/src/features/index.ts @@ -12,7 +12,12 @@ import { activate as projectTree } from "./project-tree"; import { activate as search } from "./search"; import { activate as dependencies } from "./dependencies"; import { activate as codeInsight } from "./code-insight"; -import { activate as vcs, activateComparisonWindow, activateSettingsPages as vcsPages } from "./vcs"; +import { + activate as vcs, + activateComparisonWindow, + activateDiffViewer, + activateSettingsPages as vcsPages, +} from "./vcs"; import { activate as terminal } from "./terminal"; import { activate as run } from "./run"; import { activate as runAnything } from "./run-anything"; @@ -39,7 +44,7 @@ import { activate as services } from "./services"; import { activate as github } from "./github"; import { activate as httpClient } from "./http-client"; import { activate as database } from "./database"; -import { activate as deployment } from "./deployment"; +import { activate as deployment, activateDeploymentWindow } from "./deployment"; import { activate as agents, activateSettingsPages as agentsPages } from "./agents"; import { activate as vim, activateSettingsPages as vimPages } from "./vim"; import { activate as ssh } from "./ssh"; @@ -109,6 +114,7 @@ export function activateFeatures(): () => void { export { SettingsWindow } from "./settings"; export { ComparisonWindow } from "./vcs"; +export { DeploymentWindow } from "./deployment"; /** * A comparison window of project window `parent`: the changed files between two revisions and @@ -125,6 +131,19 @@ export function activateComparisonWindowFeatures(parent: string): () => void { ]); } +/** + * The Sync with Deployed window of project window `parent`: local files next to their copies on a + * deployment server, with the diff viewer but no workbench. + */ +export function activateDeploymentWindowFeatures(parent: string): () => void { + return activateAll("deployment window", [ + ["locale", appLocale], + ["settings", activateRuntimeSettings], + ["diff", activateDiffViewer], + ["deployment", () => activateDeploymentWindow(parent)], + ]); +} + /** * The native Settings window of project window `parent`: only the settings * pages (no workbench, project, editors or background services). diff --git a/src/features/vcs/DiffEditor.tsx b/src/features/vcs/DiffEditor.tsx index b00dd6e7..b1b83e2d 100644 --- a/src/features/vcs/DiffEditor.tsx +++ b/src/features/vcs/DiffEditor.tsx @@ -217,7 +217,10 @@ export function DiffEditor({ input, toolbarActions }: { input: EditorInput; tool /** Text of the sides that are not live documents. */ const [statics, setStatics] = useState<{ a: string | null; b: string | null } | null>(null); const [error, setError] = useState(null); - const [opts, setOptsState] = useState(loadDiffOptions); + const [opts, setOptsState] = useState(() => { + const saved = loadDiffOptions(); + return data.mode ? { ...saved, mode: data.mode } : saved; + }); const [lang, setLang] = useState(null); const [chunkInfo, setChunkInfo] = useState({ count: 0, current: -1, moves: 0, whitespaceOnly: false }); const [busy, setBusy] = useState(false); diff --git a/src/features/vcs/__tests__/diffToolbar.test.tsx b/src/features/vcs/__tests__/diffToolbar.test.tsx index e7f1e33d..bdac8d2e 100644 --- a/src/features/vcs/__tests__/diffToolbar.test.tsx +++ b/src/features/vcs/__tests__/diffToolbar.test.tsx @@ -71,6 +71,14 @@ beforeEach(() => { }); describe("Diff toolbar", () => { + it("can start an embedded comparison side by side without overwriting saved viewer preferences", async () => { + localStorage.setItem("openidea.vcs.diffOptions", JSON.stringify({ mode: "unified" })); + const mounted = render(); + await waitFor(() => expect(mounted.container.querySelectorAll(".cm-editor")).toHaveLength(2)); + expect(JSON.parse(localStorage.getItem("openidea.vcs.diffOptions")!).mode).toBe("unified"); + mounted.unmount(); + localStorage.removeItem("openidea.vcs.diffOptions"); + }); it("shows a binary notice for disk comparisons instead of identical empty text", async () => { probe.mockResolvedValue({ status: "ok", data: { size: 3, binary: true, tooLarge: false } }); const binaryInput = { diff --git a/src/features/vcs/diff-editor-tab.tsx b/src/features/vcs/diff-editor-tab.tsx index 12f19afb..5f9779a8 100644 --- a/src/features/vcs/diff-editor-tab.tsx +++ b/src/features/vcs/diff-editor-tab.tsx @@ -3,9 +3,11 @@ import { hideAllToolWindows, useToolWindowLayout } from "@/workbench/toolwindows import { shortcutLabel } from "@/platform/keymap"; import { IconButton } from "@/ui"; import { DiffEditor } from "./DiffEditor"; +import type { DiffData } from "./diffInput"; export function DiffEditorTab({ input }: { input: EditorInput }) { const maximized = useToolWindowLayout((state) => !!state.stashed && !Object.values(state.visible).some(Boolean)); + if ((input.data as DiffData).embedded) return ; return ( void { + return registerEditorKind({ + kind: DIFF_KIND, + component: lazyComponent(() => import("./diff-editor-tab").then((m) => m.DiffEditorTab)), + title: (input) => diffTitle(input.data as DiffData), + icon: () => "diff", + }); +} + export function activateSettingsPages(): () => void { return registerSettingsPage({ id: "vcs.log", @@ -230,12 +240,7 @@ export function activate(): () => void { component: lazyComponent(() => import("./GitLog").then((m) => m.GitToolWindow)), headerContent: lazyComponent(() => import("./GitLog").then((m) => m.GitTabs)), }), - registerEditorKind({ - kind: DIFF_KIND, - component: lazyComponent(() => import("./diff-editor-tab").then((m) => m.DiffEditorTab)), - title: (input) => diffTitle(input.data as DiffData), - icon: () => "diff", - }), + activateDiffViewer(), registerEditorKind({ kind: MERGE_KIND, component: lazyComponent(() => import("./MergeEditor").then((m) => m.MergeEditor)), diff --git a/src/main.tsx b/src/main.tsx index cb709b39..aa0490b4 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -5,9 +5,11 @@ import { Workbench } from "./workbench/Workbench"; import { activateSecondaryWindow, activateWorkbench } from "./workbench/activate"; import { activateComparisonWindowFeatures, + activateDeploymentWindowFeatures, activateFeatures, activateSettingsWindowFeatures, ComparisonWindow, + DeploymentWindow, SettingsWindow, } from "./features"; import { auxiliaryWindow } from "./platform/auxWindow"; @@ -28,7 +30,8 @@ if (import.meta.env.DEV && isTauri()) { installRepeatedDoubleClick(); installErrorLogging(); -// Auxiliary windows (Settings `settings-`, comparisons `comparison--`) load this page too. +// Auxiliary windows (Settings `settings-`, comparisons `comparison--`, Sync with +// Deployed `deployment-`) load this page too. const aux = auxiliaryWindow(); if (aux?.kind === "settings") { @@ -38,6 +41,9 @@ if (aux?.kind === "settings") { activateSecondaryWindow(); activateComparisonWindowFeatures(aux.parent); installKeyboardDispatcher(); +} else if (aux?.kind === "deployment") { + activateSecondaryWindow(); + activateDeploymentWindowFeatures(aux.parent); } else { activateWorkbench(); // `vite build --mode perf` is a production bundle for the performance harness (`perf/`). @@ -58,7 +64,15 @@ if (aux?.kind === "settings") { ReactDOM.createRoot(document.getElementById("root") as HTMLElement).render( - {aux?.kind === "settings" ? : aux?.kind === "comparison" ? : } + {aux?.kind === "settings" ? ( + + ) : aux?.kind === "comparison" ? ( + + ) : aux?.kind === "deployment" ? ( + + ) : ( + + )} , ); hideSplash(); diff --git a/src/platform/auxWindow.test.ts b/src/platform/auxWindow.test.ts index 27356664..1eca833f 100644 --- a/src/platform/auxWindow.test.ts +++ b/src/platform/auxWindow.test.ts @@ -7,6 +7,7 @@ describe("auxiliaryWindow", () => { expect(auxiliaryWindow("settings-project-2")).toEqual({ kind: "settings", parent: "project-2" }); expect(auxiliaryWindow("comparison-main-0")).toEqual({ kind: "comparison", parent: "main" }); expect(auxiliaryWindow("comparison-project-2-14")).toEqual({ kind: "comparison", parent: "project-2" }); + expect(auxiliaryWindow("deployment-project-2")).toEqual({ kind: "deployment", parent: "project-2" }); }); it("leaves project windows and malformed labels alone", () => { diff --git a/src/platform/auxWindow.ts b/src/platform/auxWindow.ts index 24a5b1f1..15d95c3b 100644 --- a/src/platform/auxWindow.ts +++ b/src/platform/auxWindow.ts @@ -1,5 +1,5 @@ /** - * Auxiliary windows: Settings and comparison windows that belong + * Auxiliary windows: Settings, comparison and deployment windows that belong * to a project window (their parent) and share its project. Labels name the * kind and the parent (`settings-main`, `comparison-project-2-3`), mirroring * `windows::auxiliary` in src-tauri/src/windows.rs. @@ -8,11 +8,11 @@ import { getAllWindows, getCurrentWindow } from "@tauri-apps/api/window"; import { TauriEvent } from "@tauri-apps/api/event"; import { isTauri } from "./ipc"; -export type AuxKind = "settings" | "comparison"; +export type AuxKind = "settings" | "comparison" | "deployment"; /** Kinds with several windows per parent (`--`); the others have one (`-`). */ const MULTIPLE: ReadonlySet = new Set(["comparison"]); -const KINDS: readonly AuxKind[] = ["settings", "comparison"]; +const KINDS: readonly AuxKind[] = ["settings", "comparison", "deployment"]; /** An auxiliary window reports to its parent whether it closed, and which documents it released. */ export const AUX_CLOSED = "openidea://auxiliary/closed"; diff --git a/src/platform/locales/actionTitles.test.ts b/src/platform/locales/actionTitles.test.ts index 18274c97..2af3d22f 100644 --- a/src/platform/locales/actionTitles.test.ts +++ b/src/platform/locales/actionTitles.test.ts @@ -4,7 +4,7 @@ import ts from "typescript"; import { expect, it } from "vitest"; import zh from "./text/zh.json"; -const root = fileURLToPath(new URL("../../../", import.meta.url)); +const root = fileURLToPath(new URL("../../../", import.meta.url)).replaceAll("\\", "/"); const config = ts.readConfigFile(`${root}tsconfig.json`, ts.sys.readFile); const parsed = ts.parseJsonConfigFileContent(config.config, ts.sys, root); const program = ts.createProgram(parsed.fileNames, parsed.options); diff --git a/src/platform/locales/text/fr.json b/src/platform/locales/text/fr.json index 64a26e97..2cc108f6 100644 --- a/src/platform/locales/text/fr.json +++ b/src/platform/locales/text/fr.json @@ -4284,24 +4284,15 @@ "Delete {n} items from the server?": "Supprimer {n} éléments du serveur ?", "Not mapped to a project path": "Non associé à un chemin du projet", "Unknown Host Key": "Clé d'hôte inconnue", - "Sync with {server}": "Synchroniser avec {server}", - "Hide identical files ({n})": "Masquer les fichiers identiques ({n})", - "Synchronize": "Synchroniser", "{n} of {total} files will change": "{n} fichiers sur {total} seront modifiés", "Comparing local and remote files…": "Comparaison des fichiers locaux et distants…", "Files to synchronize": "Fichiers à synchroniser", "Action": "Action", - "Local size": "Taille locale", - "Local modified": "Modification locale", - "Remote size": "Taille distante", - "Remote modified": "Modification distante", - "Local and remote files are identical": "Les fichiers locaux et distants sont identiques", "Synchronizing…": "Synchronisation…", "Local only": "Local uniquement", "Remote only": "Distant uniquement", "Local is newer": "Le local est plus récent", "Remote is newer": "Le distant est plus récent", - "Different size": "Taille différente", "Identical": "Identique", "Upload": "Envoyer", "Delete remote": "Supprimer le distant", diff --git a/src/platform/locales/text/ko.json b/src/platform/locales/text/ko.json index 18d7826c..e95c258c 100644 --- a/src/platform/locales/text/ko.json +++ b/src/platform/locales/text/ko.json @@ -4395,24 +4395,15 @@ "Delete {n} items from the server?": "서버에서 항목 {n}개를 삭제하시겠습니까?", "Not mapped to a project path": "프로젝트 경로에 매핑되지 않음", "Unknown Host Key": "알 수 없는 호스트 키", - "Sync with {server}": "{server}와(과) 동기화", - "Hide identical files ({n})": "동일한 파일 숨기기({n})", - "Synchronize": "동기화", "{n} of {total} files will change": "파일 {total}개 중 {n}개가 변경됩니다", "Comparing local and remote files…": "로컬 파일과 원격 파일 비교 중…", "Files to synchronize": "동기화할 파일", "Action": "액션", - "Local size": "로컬 크기", - "Local modified": "로컬 수정 시간", - "Remote size": "원격 크기", - "Remote modified": "원격 수정 시간", - "Local and remote files are identical": "로컬 파일과 원격 파일이 동일합니다", "Synchronizing…": "동기화 중…", "Local only": "로컬에만 있음", "Remote only": "원격에만 있음", "Local is newer": "로컬이 더 최신임", "Remote is newer": "원격이 더 최신임", - "Different size": "크기가 다름", "Identical": "동일함", "Upload": "업로드", "Delete remote": "원격 삭제", diff --git a/src/platform/locales/text/zh.json b/src/platform/locales/text/zh.json index 4f4ff1ed..3ac83fd5 100644 --- a/src/platform/locales/text/zh.json +++ b/src/platform/locales/text/zh.json @@ -4407,24 +4407,15 @@ "Delete {n} items from the server?": "要从服务器删除 {n} 个项目吗？", "Not mapped to a project path": "未映射到项目路径", "Unknown Host Key": "未知的主机密钥", - "Sync with {server}": "与 {server} 同步", - "Hide identical files ({n})": "隐藏相同的文件（{n}）", - "Synchronize": "同步", "{n} of {total} files will change": "{total} 个文件中有 {n} 个将被更改", "Comparing local and remote files…": "正在比较本地和远程文件…", "Files to synchronize": "要同步的文件", "Action": "操作", - "Local size": "本地大小", - "Local modified": "本地修改时间", - "Remote size": "远程大小", - "Remote modified": "远程修改时间", - "Local and remote files are identical": "本地和远程文件相同", "Synchronizing…": "正在同步…", "Local only": "仅本地", "Remote only": "仅远程", "Local is newer": "本地较新", "Remote is newer": "远程较新", - "Different size": "大小不同", "Identical": "相同", "Upload": "上传", "Delete remote": "删除远程", @@ -5720,6 +5711,46 @@ "The account's 5-hour and weekly limits, when the agent reports them (Claude Code)": "账户的 5 小时和每周限额，在智能体报告时显示（Claude Code）", "Session cost": "会话花费", "What the session has cost so far, as the agent reports it (API pricing, also shown on subscriptions)": "智能体报告的本会话目前花费（按 API 价格计算，订阅账户也会显示）", + "Local: {path}": "本地：{path}", + "Local: (does not exist)": "本地：（不存在）", + "Synchronize All": "同步全部", + "Deployment comparison options": "部署比较选项", + "Refresh comparison": "刷新比较", + "Show local-only files": "显示仅本地存在的文件", + "Show different files": "显示有差异的文件", + "Show remote-only files": "显示仅远程存在的文件", + "Compare by:": "比较依据：", + "Compare by": "比较依据", + "Size and Timestamp": "大小和时间戳", + "Binary Content": "二进制内容", + "Synchronize Selected": "同步所选文件", + "Stop content comparison": "停止内容比较", + "Filter:": "筛选：", + "Local files": "本地文件", + "Local name": "本地名称", + "Remote name": "远程名称", + "Space changes the selected operation. Enter synchronizes the selected file. Synchronize All applies all chosen operations.": "空格键切换所选操作，Enter 键同步所选文件。“同步全部”执行所有已选择的操作。", + "Resize file list and diff preview": "调整文件列表和差异预览的大小", + "The deployment server or project is no longer available": "部署服务器或项目已不可用", + "Select a file to compare its contents": "选择文件以比较其内容", + "Loading file contents…": "正在加载文件内容…", + "This file is too large for the text preview. Use Binary Content to compare it.": "此文件过大，无法进行文本预览。请使用“二进制内容”比较。", + "Deployment comparison": "部署比较", + "Wait for synchronization to finish before opening another comparison.": "请等待同步完成后再打开其他比较。", + "Save local changes?": "保存本地更改？", + "Save the local files edited in this comparison before closing.": "关闭前保存本次比较中编辑的本地文件。", + "Save and Close": "保存并关闭", + "Cannot close deployment comparison": "无法关闭部署比较", + "Deployment Comparison": "部署比较", + "Opening deployment comparison…": "正在打开部署比较…", + "Diff Between 'local files' and 'files on {server}'": "本地文件与“{server}”上的文件的差异", + "Comparing contents: {done} of {total}": "正在比较内容：{done}/{total}", + "A file changed during comparison. Refresh and compare again.": "比较期间文件已更改，请刷新后重新比较。", + "The project is no longer open.": "项目已关闭。", + "The deployment server is no longer available.": "部署服务器已不可用。", + "No selected files are mapped to this deployment server.": "所选文件均未映射到此部署服务器。", + "Cannot open comparison for {server}": "无法打开与 {server} 的比较", + "Could not refresh comparison: {error}": "无法刷新比较：{error}", "Build and run using": "构建和运行使用", "Gradle runs a main class as a JavaExec task on its source set's runtime classpath (Gradle's resource processing included). OpenIDEA launches it on the Java language server's classpath.": "Gradle 以 JavaExec 任务在源集的运行时类路径上运行主类（包括 Gradle 的资源处理）。OpenIDEA 在 Java 语言服务器的类路径上启动它。", "Set an absolute root path on the Connection tab first": "请先在“连接”页设置绝对根路径", From 65ca0390de485134d58f35e33471f0ff12cbd730 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:41:34 +0800 Subject: [PATCH 007/151] feat(run): choose the JRE and the module of an Application configuration (#1172) Application / Spring Boot configurations found their module only from the main class, so the same class in two modules failed with "Cannot determine the module for main class", and the JVM was always the module's. The form now has JRE (the module's JDK or an installed JDK) and Use classpath of module (found by main class, or a project module, stored as RunConfig.module). Run configuration selects can load their options when shown. Fixes #1153 --- src/features/run/ConfigForm.tsx | 15 ++++++-- src/features/run/jvmTypes.ts | 46 +++++++++++++++++++++++- src/features/run/runConfigTypes.test.tsx | 26 ++++++++++++++ src/features/run/types.ts | 10 ++++-- src/platform/locales/text/zh.json | 5 ++- src/workbench/contributions.ts | 4 ++- 6 files changed, 98 insertions(+), 8 deletions(-) diff --git a/src/features/run/ConfigForm.tsx b/src/features/run/ConfigForm.tsx index 159b04ed..c9c2ef8f 100644 --- a/src/features/run/ConfigForm.tsx +++ b/src/features/run/ConfigForm.tsx @@ -4,7 +4,7 @@ * Before launch list. */ import { parseJavaArgs } from "@/platform/javaLaunch"; -import { useMemo, useRef, useState, type KeyboardEvent, type ReactNode, type RefObject } from "react"; +import { useEffect, useMemo, useRef, useState, type KeyboardEvent, type ReactNode, type RefObject } from "react"; import clsx from "clsx"; import { isTauri, type BeforeLaunchTask, type RunConfig } from "@/platform/ipc"; import { useWorkflowSettings } from "@/platform/workflowSettings"; @@ -729,9 +729,20 @@ function TypeField({ const tx = useTx(); const root = useProject((s) => s.project?.root ?? null); const listId = `run-field-${field.key}`; + const [loaded, setLoaded] = useState>([]); + useEffect(() => { + let live = true; + void field + .loadOptions?.() + .then((options) => live && setLoaded(options)) + .catch(() => {}); + return () => { + live = false; + }; + }, [field]); if (field.control === "output") return ; if (field.control === "select") { - const options = [...(field.options ?? [])]; + const options = [...(field.options ?? []), ...loaded]; if (value && !options.some((o) => o.value === value)) options.push({ value, label: value }); return ( setValues((v) => ({ ...v, [name]: e.target.value }))} + onKeyDown={(e) => { + if (e.key === "Enter") ok(); + }} + /> + + + ))} + + + + + ); +} diff --git a/src/features/graphql/GraphQL.module.css b/src/features/graphql/GraphQL.module.css new file mode 100644 index 00000000..c6057e0d --- /dev/null +++ b/src/features/graphql/GraphQL.module.css @@ -0,0 +1,323 @@ +.view { + height: 100%; + display: flex; + min-height: 0; +} +.tabs { + min-width: 0; + border: 0; + background: none; +} +.toolbar { + flex: none; + width: 32px; + display: flex; + flex-direction: column; + align-items: center; + gap: 2px; + padding: 4px 0; + border-right: 1px solid var(--border); +} +.toolbarSep { + width: 16px; + height: 1px; + margin: 3px 0; + background: var(--border); +} +.body { + flex: 1; + min-width: 0; + min-height: 0; + display: flex; + flex-direction: column; +} +.tree { + flex: 1; + min-height: 0; +} +.status { + height: 26px; + flex: none; + display: flex; + align-items: center; + gap: 6px; + padding: 0 10px; + font-size: var(--fs-ui); + color: var(--fg-subtle); + white-space: nowrap; +} +.row { + flex: 1; + min-width: 0; + display: flex; + align-items: center; + gap: 6px; + padding-right: 10px; + white-space: nowrap; +} +.label { + overflow: hidden; + text-overflow: ellipsis; +} +.muted { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + color: var(--fg-subtle); +} +.ok { + color: var(--success); +} +.code { + font-weight: 600; +} +.result { + height: 100%; + display: flex; + flex-direction: column; + min-height: 0; +} +.resultHeader { + height: 28px; + flex: none; + display: flex; + align-items: center; + gap: 6px; + padding: 0 10px; + white-space: nowrap; + border-bottom: 1px solid var(--border); +} +/* Result header: ✓ on success, a red dot with errors (IDEA). */ +.dotFailed { + flex: none; + width: 7px; + height: 7px; + margin: 0 4px; + border-radius: 50%; + background: var(--danger); +} +/* The response with line numbers, scrolling together (IDEA's result editor). */ +.outputBox { + flex: 1; + min-height: 0; + display: flex; + align-items: flex-start; + overflow: auto; + background: var(--bg-editor); +} +.outputGutter, +.output { + margin: 0; + padding: 6px 10px; + font-family: var(--font-mono); + font-size: var(--fs-ui); + line-height: 1.6; +} +.outputGutter { + position: sticky; + left: 0; + flex: none; + min-width: 36px; + min-height: 100%; + padding-left: 8px; + background: var(--bg-editor); + color: var(--fg-line-number); + text-align: right; + user-select: none; +} +.output { + flex: 1; + white-space: pre; + user-select: text; +} + +/* Editor header. */ +.editorToolbar { + display: flex; + align-items: center; + gap: 4px; + height: var(--h-header); + padding: 0 8px; + background: var(--bg-editor); + border-bottom: 1px solid var(--border); + font-family: var(--font-ui); + font-size: var(--fs-ui); + color: var(--fg-secondary); +} +.editorToolbarSep { + width: 1px; + height: 16px; + margin: 0 4px; + background: var(--border); +} +.endpoint { + min-width: 160px; + max-width: 420px; +} +.variablesBox { + position: relative; + border-bottom: 1px solid var(--border); + background: var(--bg-editor); +} +.variablesLayer, +.variables { + box-sizing: border-box; + margin: 0; + padding: 6px 10px; + border: 0; + font-family: var(--font-mono); + font-size: var(--fs-ui); + line-height: 1.5; + white-space: pre; + tab-size: 2; +} +.variablesLayer { + position: absolute; + inset: 0; + overflow: hidden; + color: var(--fg); + pointer-events: none; +} +.variables { + position: relative; + display: block; + width: 100%; + height: 96px; + min-height: 48px; + resize: vertical; + overflow: auto; + outline: none; + background: transparent; + color: transparent; + caret-color: var(--fg); +} +.variables::placeholder { + color: var(--fg-subtle); +} +.variables::selection { + background: var(--bg-editor-selection); + color: transparent; +} + +/* + * graphql-config gutter: ▶ on each endpoint. As wide as the run gutter plus the Spring + * icon gutter beside it in Spring files (14px each), so ▶ keeps the same gap to the VCS bar. + */ +.runGutter { + width: 28px; +} +.runGutter :global(.cm-gutterElement) { + display: flex; + align-items: center; + justify-content: flex-start; +} +.runMarker { + width: 14px; + height: 14px; + display: flex; + align-items: center; + justify-content: center; + border-radius: var(--radius-small); + cursor: pointer; +} +.runMarker:hover { + background: var(--bg-hover); +} + +/* Query result and variables: JSON in the editor's syntax colours. */ +.output :global(.tok-propertyName), +.variablesLayer :global(.tok-propertyName) { + color: var(--syn-property); +} +.output :global(.tok-string), +.variablesLayer :global(.tok-string), +.output :global(.tok-string2), +.variablesLayer :global(.tok-string2) { + color: var(--syn-string); +} +.output :global(.tok-number), +.variablesLayer :global(.tok-number) { + color: var(--syn-number); +} +.output :global(.tok-bool), +.variablesLayer :global(.tok-bool), +.output :global(.tok-null), +.variablesLayer :global(.tok-null), +.output :global(.tok-atom), +.variablesLayer :global(.tok-atom), +.output :global(.tok-keyword), +.variablesLayer :global(.tok-keyword) { + color: var(--syn-keyword); +} +.output :global(.tok-punctuation), +.variablesLayer :global(.tok-punctuation), +.output :global(.tok-separator), +.variablesLayer :global(.tok-separator), +.output :global(.tok-squareBracket), +.variablesLayer :global(.tok-squareBracket), +.output :global(.tok-brace), +.variablesLayer :global(.tok-brace) { + color: var(--syn-punct); +} + +/* Text the last Execute GraphQL did not send (IDEA greys it out). */ +.unusedContext { + opacity: 0.45; +} + +/* GraphQL Environment Variables dialog: a Name / Value table (IDEA). */ +.envBox { + display: flex; + flex-direction: column; + height: 300px; + border: 1px solid var(--border); +} +.envToolbar { + flex: none; + display: flex; + gap: 2px; + padding: 2px 4px; + border-bottom: 1px solid var(--border); +} +.envTable { + width: 100%; + border-collapse: collapse; + table-layout: fixed; + font-size: var(--fs-ui); +} +.envTable th { + height: 24px; + padding: 0 8px; + text-align: left; + font-weight: normal; + color: var(--fg-secondary); + border-bottom: 1px solid var(--border); +} +.envTable th + th { + border-left: 1px solid var(--border); +} +.envTable td { + height: 24px; + padding: 0; +} +.envName { + padding: 0 8px !important; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.envValue { + box-sizing: border-box; + width: 100%; + height: 24px; + padding: 0 8px; + border: 0; + outline: none; + background: transparent; + color: var(--fg); + font: inherit; + font-family: var(--font-mono); +} +.envValue:focus { + background: var(--bg-selected); +} diff --git a/src/features/graphql/GraphQLView.tsx b/src/features/graphql/GraphQLView.tsx new file mode 100644 index 00000000..e6d787a3 --- /dev/null +++ b/src/features/graphql/GraphQLView.tsx @@ -0,0 +1,371 @@ +/** + * GraphQL tool window (IDEA's): "Schemas and Project Structure" lists each schema + * with its discovery summary, the errors the language server reports for its + * files and its endpoints; "Query Result" shows the last query's response. + */ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { Diagnostic } from "@/bindings"; +import { diagnosticsFor, subscribeDiagnostics } from "@/editor/lsp/diagnostics"; +import { highlightInto } from "@/editor/lsp/markdown"; +import { formatMessage, useTx } from "@/platform/i18n"; +import { openExternalUrl } from "@/platform/openUrl"; +import { basename, relativeDir, useProject } from "@/platform/project"; +import { EmptyState, Icon, IconButton, Spinner, Tabs, Tree, fileIconName, T } from "@/ui"; +import { editorService } from "@/workbench/contributions"; +import { + childNodes, + formatBody, + formatSize, + projectNodes, + responseErrorCount, + schemaProblems, + schemaStats, + type SchemaNode, +} from "./model"; +import { configurationFor, discoverSchemasNow, openConfiguration, useGraphQL, type GraphQLTab } from "./store"; +import styles from "./GraphQL.module.css"; + +const HELP_URL = "https://the-guild.dev/graphql/config/docs/user/usage"; + +export function GraphQLTabs() { + const tab = useGraphQL((s) => s.tab); + return ( + useGraphQL.setState({ tab: id as GraphQLTab })} + items={[ + { id: "schemas", label: "Schemas and Project Structure" }, + { id: "result", label: "Query Result" }, + ]} + /> + ); +} + +export function GraphQLView() { + const tab = useGraphQL((s) => s.tab); + return tab === "result" ? : ; +} + +function expandableKeys(nodes: readonly SchemaNode[], children: (n: SchemaNode) => SchemaNode[] | undefined): string[] { + return nodes.flatMap((n) => { + const c = children(n); + return c?.length ? [n.key, ...expandableKeys(c, children)] : []; + }); +} + +type DiagnosticsByPath = ReadonlyMap; + +function readDiagnostics(paths: readonly string[]): DiagnosticsByPath { + return new Map(paths.map((p) => [p, diagnosticsFor(p)])); +} + +/** Diagnostics of `paths`, updated as the language server reports them. */ +function useDiagnostics(paths: readonly string[]): DiagnosticsByPath { + const [diagnostics, setDiagnostics] = useState(() => readDiagnostics(paths)); + useEffect(() => { + const update = () => setDiagnostics(readDiagnostics(paths)); + update(); + const offs = paths.map((p) => subscribeDiagnostics(p, update)); + return () => offs.forEach((off) => off()); + }, [paths]); + return diagnostics; +} + +function SchemasView() { + const tx = useTx(); + const root = useProject((s) => s.project?.root ?? ""); + const projects = useGraphQL((s) => s.projects); + const discovering = useGraphQL((s) => s.discovering); + const expanded = useGraphQL((s) => s.expanded); + + useEffect(() => { + if (root && !projects && !discovering) void discoverSchemasNow(); + }, [root, projects, discovering]); + + const paths = useMemo(() => (projects ?? []).flatMap((p) => p.files.map((f) => f.path)), [projects]); + const diagnostics = useDiagnostics(paths); + const problemsOf = useCallback((path: string) => diagnostics.get(path) ?? [], [diagnostics]); + const children = useCallback((n: SchemaNode) => childNodes(n, problemsOf), [problemsOf]); + const roots = useMemo(() => projectNodes(projects ?? []), [projects]); + // IDEA opens each schema with its errors and endpoints. + const defaultExpanded = useMemo( + () => new Set(roots.flatMap((n) => [n.key, `${n.key}#errors`, `${n.key}#endpoints`])), + [roots], + ); + const effectiveExpanded = expanded ?? defaultExpanded; + + const renderNode = (n: SchemaNode) => { + switch (n.kind) { + case "project": { + const p = n.project; + return ( + + + {p.configPath ? basename(p.root) : tx("Default project-wide schema")} + {p.root} + + ); + } + case "summary": { + const p = n.project; + const detail = p.dynamic + ? tx("the configuration is code and is not evaluated") + : !p.files.length && !p.remote.length + ? tx("schema is empty") + : formatMessage(tx("{types} types, {inputs} inputs, {enums} enums, {directives} directives"), { + ...schemaStats(p), + }); + return ( + + + {tx("Schema discovery summary")} + - {detail} + + ); + } + case "file": + return ( + + + {basename(n.file.path)} + {relativeDir(root, n.file.path)} + + ); + case "remote": + return ( + + + {n.url} + + ); + case "errors": { + const count = schemaProblems(n.project, problemsOf).length; + return ( + + + {tx("Schema errors")} + {count > 0 && {count}} + + ); + } + case "error": { + const d = n.diagnostic; + return ( + + + {d.message} + + {basename(n.path)}:{d.range.start.line + 1} + + + ); + } + case "noErrors": + return ( + + + {tx("No errors found")} + + ); + case "endpoints": + return ( + + + {tx("Endpoints")} + + ); + case "endpoint": + return ( + + {n.endpoint.name} + {n.endpoint.url} + + ); + case "noEndpoints": + return ( + + + + {n.project.configPath + ? tx("No endpoints available in this schema") + : tx("No endpoints available in the default schema")} + + + ); + } + }; + + const labelOf = (n: SchemaNode): string => { + switch (n.kind) { + case "project": + return n.project.configPath ? basename(n.project.root) : tx("Default project-wide schema"); + case "summary": + return tx("Schema discovery summary"); + case "file": + return basename(n.file.path); + case "remote": + return n.url; + case "errors": + return tx("Schema errors"); + case "error": + return n.diagnostic.message; + case "noErrors": + return tx("No errors found"); + case "endpoints": + return tx("Endpoints"); + case "endpoint": + return n.endpoint.name; + case "noEndpoints": + return tx("No endpoints available in the default schema"); + } + }; + + return ( +
+
+ + + void openConfiguration()} + /> + + void openExternalUrl(HELP_URL)} + /> + + useGraphQL.setState({ expanded: new Set(expandableKeys(roots, children)) })} + /> + useGraphQL.setState({ expanded: new Set() })} + /> +
+
+ {!projects ? ( +
+ Discovering GraphQL schemas… +
+ ) : ( + + aria-label="GraphQL schemas" + className={styles.tree} + roots={roots} + getKey={(n) => n.key} + getLabel={labelOf} + getChildren={children} + isExpandable={(n) => !!children(n)?.length} + expanded={effectiveExpanded} + onExpandedChange={(next) => useGraphQL.setState({ expanded: next })} + renderNode={renderNode} + multiSelect={false} + onActivate={(n) => { + if (n.kind === "project" && n.project.configPath) void editorService().openFile(n.project.configPath); + else if (n.kind === "file") void editorService().openFile(n.file.path); + else if (n.kind === "error") { + const { start } = n.diagnostic.range; + void editorService().openFile(n.path, { selection: { line: start.line, col: start.col } }); + } + }} + /> + )} +
+
+ ); +} + +function QueryResultView() { + const tx = useTx(); + const result = useGraphQL((s) => s.result); + const running = useGraphQL((s) => s.running !== null); + if (!result) { + return ( + + {!running && ( + + Run a query with Execute GraphQL in the toolbar of a GraphQL file. + + )} + + ); + } + const errors = responseErrorCount(result.body); + const failed = !!result.error || errors > 0 || (result.status !== null && result.status >= 400); + const ms = result.totalMs === null ? "?" : Math.round(result.totalMs); + const size = formatSize(result.size ?? result.body?.length ?? 0); + return ( +
+
+ {running ? ( + + ) : failed ? ( + + ) : ( + + )} + + {result.error + ? formatMessage(tx("{name}: {error}"), { name: basename(result.path), error: result.error }) + : formatMessage(tx("{file}: {ms} ms execution time, {size} response"), { + file: basename(result.path), + ms, + size, + })} + {!result.error && errors === 1 && tx(", 1 error")} + {!result.error && errors > 1 && formatMessage(tx(", {count} errors"), { count: errors })} + +
+ +
+ ); +} + +/** The response in the editor's colours (JSON), like IDEA's result viewer. */ +function ResultBody({ text, json }: { text: string; json: boolean }) { + const tx = useTx(); + const ref = useRef(null); + useEffect(() => { + const el = ref.current; + if (!el) return; + el.textContent = text; + if (json) void highlightInto(el, text, "JSON"); + }, [text, json]); + const lines = text.split("\n").length; + return ( +
+
+ {Array.from({ length: lines }, (_, i) => ( +
{i + 1}
+ ))} +
+ 
+
+ ); +} diff --git a/src/features/graphql/editor.tsx b/src/features/graphql/editor.tsx new file mode 100644 index 00000000..61adbf3e --- /dev/null +++ b/src/features/graphql/editor.tsx @@ -0,0 +1,284 @@ +/** + * GraphQL in editors, after IDEA's GraphQL plugin (JetBrains/js-graphql-intellij-plugin): + * - the toolbar above GraphQL files: Open Configuration File, Edit Environment Variables, + * Toggle Variables Editor, the endpoint, Execute GraphQL, Rerun Introspection Query and + * Open Introspection Schema; the variables editor (JSON) shows below it; + * - schema files re-checked as they are edited (unresolved types); + * - in graphql-config files, ▶ on each endpoint runs its introspection query. + */ +import { RangeSet, StateEffect, StateField, type EditorState, type Extension } from "@codemirror/state"; +import { + Decoration, + EditorView, + GutterMarker, + ViewPlugin, + gutter, + showPanel, + type DecorationSet, + type Panel, + type ViewUpdate, +} from "@codemirror/view"; +import { useEffect, useRef } from "react"; +import { createRoot } from "react-dom/client"; +import type { EditorDocContext } from "@/editor/api"; +import { highlightInto } from "@/editor/lsp/markdown"; +import { useTx } from "@/platform/i18n"; +import { shortcutLabel } from "@/platform/keymap"; +import { basename } from "@/platform/project"; +import { isVirtualUri } from "@/platform/virtualDocs"; +import { IconButton, Select, gutterIconMarkup } from "@/ui"; +import { endpointLines, isConfigFile, outsideRanges, queryContext } from "./model"; +import { + chooseEndpoint, + configurationFor, + discoverSchemasNow, + editEnvironmentVariables, + endpointFor, + introspectEndpoint, + openConfiguration, + openIntrospectionSchema, + rerunIntrospection, + revalidateFile, + runQuery, + schemaOf, + setVariables, + toggleVariables, + useGraphQL, +} from "./store"; +import styles from "./GraphQL.module.css"; + +export const GRAPHQL_LANGUAGE = "graphql"; + +// --- query context: what Execute GraphQL sent, the rest greyed out (IDEA) --------------------- + +const setSentRanges = StateEffect.define>(); +const unusedMark = Decoration.mark({ class: styles.unusedContext ?? "" }); + +/** Greys out the text the last Execute GraphQL left out, until the text or selection changes. */ +const unusedContext = StateField.define({ + create: () => Decoration.none, + update(value, tr) { + for (const e of tr.effects) + if (e.is(setSentRanges)) + return Decoration.set( + outsideRanges(tr.state.doc.length, e.value) + .filter((r) => r.to > r.from) + .map((r) => unusedMark.range(r.from, r.to)), + ); + return tr.docChanged || tr.selection ? Decoration.none : value; + }, + provide: (f) => EditorView.decorations.from(f), +}); + +/** Execute GraphQL in `view`: the selection, else the operation at the caret (with its fragments). */ +export function runFromView(path: string, view: EditorView): Promise { + const { state } = view; + const { from, to } = state.selection.main; + const context = queryContext(state.doc.toString(), { from, to }); + if (view.state.field(unusedContext, false) !== undefined) + view.dispatch({ effects: setSentRanges.of(context.ranges) }); + return runQuery(path, context.text); +} + +function Toolbar({ path, view }: { path: string; view: EditorView }) { + const tx = useTx(); + const projects = useGraphQL((s) => s.projects); + const discovering = useGraphQL((s) => s.discovering); + const running = useGraphQL((s) => s.running !== null); + const endpoint = useGraphQL((s) => endpointFor(s, path)); + const variablesShown = useGraphQL((s) => s.variablesShown.has(path)); + useEffect(() => { + if (!projects && !discovering) void discoverSchemasNow(); + }, [projects, discovering]); + + const config = configurationFor(projects, path); + const schema = projects ? schemaOf(projects, path) : null; + const endpoints = schema?.endpoints ?? []; + return ( +
+ void openConfiguration(path)} + /> + void editEnvironmentVariables(path)} + /> + + toggleVariables(path)} + /> +