mirror of
https://github.com/l0ng-ai/openidea.git
synced 2026-11-21 16:03:46 +00:00
Snowflake is a built-in data source kind: connection dialog, key-pair, browser single sign-on, password or token sign-in, the database tree, SQL consoles, and Snowflake-aware SQL assistance and syntax colours. The driver speaks Snowflake's session protocol over the existing reqwest + rustls stack, with no JDBC/ODBC or native library and no new package. Metadata is read with SHOW / DESCRIBE, which need no running warehouse; introspected names are kept on disk for completion. Co-authored-by: vvnadolskyy <12991410+vvnadolskyy@users.noreply.github.com>
195 lines
6.9 KiB
TypeScript
195 lines
6.9 KiB
TypeScript
/**
|
|
* SQL tokens for completion and resolution: words, quoted identifiers, string
|
|
* literals, numbers and punctuation, with comments and whitespace dropped.
|
|
* Dialect-aware (MySQL backticks / `#` comments / backslash escapes,
|
|
* PostgreSQL dollar quoting, Snowflake `$$` strings / `//` comments / stages /
|
|
* variant paths). Offsets are UTF-16, like the editor's.
|
|
*/
|
|
import type { DbDriver } from "@/platform/ipc";
|
|
|
|
export type TokenKind = "word" | "quoted" | "string" | "number" | "param" | "punct";
|
|
|
|
export interface Token {
|
|
kind: TokenKind;
|
|
/** Source text (quotes included for `quoted` / `string`). */
|
|
text: string;
|
|
from: number;
|
|
to: number;
|
|
/** Upper-cased `text` of words, for keyword checks. */
|
|
upper: string;
|
|
value?: string;
|
|
}
|
|
|
|
export interface Lexed {
|
|
tokens: Token[];
|
|
/** Comments; a caret at `to` is still inside line comments and unterminated block comments. */
|
|
comments: Array<{ from: number; to: number; open: boolean }>;
|
|
}
|
|
|
|
const WORD_START = /[\p{L}_]/u;
|
|
const WORD_PART = /[\p{L}\p{N}_$]/u;
|
|
const STAGE_START = /[\p{L}_~%"]/u;
|
|
const STAGE_END = /[\s(),;']/;
|
|
const PATH_OWNER = /[\p{L}\p{N}_$)\]"']/u;
|
|
const TWO_CHAR = new Set(["::", "<=", ">=", "<>", "!=", "||", "->", "=>", ":="]);
|
|
|
|
export function lex(text: string, driver: DbDriver = "sqlite"): Lexed {
|
|
const mysql = driver === "mysql";
|
|
const clickhouse = driver === "clickhouse";
|
|
const snowflake = driver === "snowflake";
|
|
const tokens: Token[] = [];
|
|
const comments: Lexed["comments"] = [];
|
|
const n = text.length;
|
|
let i = 0;
|
|
const push = (kind: TokenKind, from: number, to: number) => {
|
|
const t = text.slice(from, to);
|
|
tokens.push({
|
|
kind,
|
|
text: t,
|
|
from,
|
|
to,
|
|
upper: kind === "word" ? t.toUpperCase() : t,
|
|
...(kind === "quoted" && clickhouse
|
|
? { value: t.slice(1, t.endsWith(t[0]!) ? -1 : undefined).replace(/\\([\\'"`])/g, "$1") }
|
|
: {}),
|
|
});
|
|
};
|
|
/** End of a quoted run starting at `i` (closing char doubled = escaped). */
|
|
const quoted = (start: number, close: string, backslash: boolean): number => {
|
|
let j = start + 1;
|
|
while (j < n) {
|
|
const c = text[j]!;
|
|
if (backslash && c === "\\") j += 2;
|
|
else if (c === close) {
|
|
if (text[j + 1] === close) j += 2;
|
|
else return j + 1;
|
|
} else j++;
|
|
}
|
|
return n;
|
|
};
|
|
while (i < n) {
|
|
const c = text[i]!;
|
|
const next = text[i + 1];
|
|
if (c === " " || c === "\t" || c === "\n" || c === "\r") {
|
|
i++;
|
|
} else if ((c === "-" && next === "-") || (c === "#" && mysql) || (c === "/" && next === "/" && snowflake)) {
|
|
const end = text.indexOf("\n", i);
|
|
const to = end < 0 ? n : end;
|
|
comments.push({ from: i, to, open: true });
|
|
i = to;
|
|
} else if (c === "/" && next === "*") {
|
|
const end = text.indexOf("*/", i + 2);
|
|
const to = end < 0 ? n : end + 2;
|
|
comments.push({ from: i, to, open: end < 0 });
|
|
i = to;
|
|
} else if (c === "'") {
|
|
const to = quoted(i, "'", mysql || clickhouse || snowflake);
|
|
push("string", i, to);
|
|
i = to;
|
|
} else if (c === '"') {
|
|
const to = quoted(i, '"', clickhouse);
|
|
push(mysql ? "string" : "quoted", i, to);
|
|
i = to;
|
|
} else if (c === "`" && (mysql || clickhouse)) {
|
|
const to = quoted(i, "`", clickhouse);
|
|
push("quoted", i, to);
|
|
i = to;
|
|
} else if (c === "$" && driver === "postgres") {
|
|
const m = /^\$([\p{L}_][\p{L}\p{N}_]*)?\$/u.exec(text.slice(i, i + 64));
|
|
if (m) {
|
|
const tag = m[0];
|
|
const close = text.indexOf(tag, i + tag.length);
|
|
const to = close < 0 ? n : close + tag.length;
|
|
push("string", i, to);
|
|
i = to;
|
|
} else {
|
|
let j = i + 1;
|
|
while (j < n && /\d/.test(text[j]!)) j++;
|
|
push("param", i, j);
|
|
i = j;
|
|
}
|
|
} else if (c === "$" && snowflake) {
|
|
if (next === "$") {
|
|
const close = text.indexOf("$$", i + 2);
|
|
const to = close < 0 ? n : close + 2;
|
|
push("string", i, to);
|
|
i = to;
|
|
} else {
|
|
// `$1` (positional column) / `$name` (session variable).
|
|
let j = i + 1;
|
|
while (j < n && WORD_PART.test(text[j]!)) j++;
|
|
push(j > i + 1 ? "param" : "punct", i, j);
|
|
i = j;
|
|
}
|
|
} else if (c === "@" && snowflake && next !== undefined && STAGE_START.test(next)) {
|
|
// A stage reference with its path: `@stage/dir/file.csv`, `@~/x`, `@%table`.
|
|
let j = i + 1;
|
|
while (j < n && !STAGE_END.test(text[j]!)) j++;
|
|
push("param", i, j);
|
|
i = j;
|
|
} else if (/\d/.test(c) || (c === "." && next !== undefined && /\d/.test(next))) {
|
|
let j = i + 1;
|
|
while (j < n && /[\d.eE_xXa-fA-F]/.test(text[j]!)) j++;
|
|
push("number", i, j);
|
|
i = j;
|
|
} else if (WORD_START.test(c)) {
|
|
let j = i + 1;
|
|
while (j < n && WORD_PART.test(text[j]!)) j++;
|
|
if (snowflake && text.startsWith("://", j)) {
|
|
// A location of PUT / GET (`file:///tmp/data.csv`): one token, and its `//` no comment.
|
|
while (j < n && !STAGE_END.test(text[j]!)) j++;
|
|
push("param", i, j);
|
|
} else push("word", i, j);
|
|
i = j;
|
|
} else if (
|
|
(c === "?" || c === ":" || c === "@") &&
|
|
next !== undefined &&
|
|
WORD_START.test(next) &&
|
|
!(c === ":" && text[i - 1] === ":") &&
|
|
// Snowflake `col:path` (also after `)`, `]` and quotes) reads a VARIANT.
|
|
!(c === ":" && snowflake && i > 0 && PATH_OWNER.test(text[i - 1]!))
|
|
) {
|
|
let j = i + 1;
|
|
while (j < n && WORD_PART.test(text[j]!)) j++;
|
|
push("param", i, j);
|
|
i = j;
|
|
} else {
|
|
const two = text.slice(i, i + 2);
|
|
const len = two === "->" && text[i + 2] === ">" ? 3 : TWO_CHAR.has(two) ? 2 : 1;
|
|
push("punct", i, i + len);
|
|
i += len;
|
|
}
|
|
}
|
|
return { tokens, comments };
|
|
}
|
|
|
|
/** Whether `pos` lies inside a string literal, quoted identifier or comment (not at its edge). */
|
|
export function inLiteral(lexed: Lexed, pos: number): "string" | "quoted" | "comment" | null {
|
|
for (const c of lexed.comments) if (pos > c.from && (pos < c.to || (c.open && pos === c.to))) return "comment";
|
|
for (const t of lexed.tokens) {
|
|
if (t.from >= pos) break;
|
|
if ((t.kind === "string" || t.kind === "quoted") && pos > t.from && (pos < t.to || !closed(t))) return t.kind;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/** Whether a quoted token has its closing quote. */
|
|
function closed(t: Token): boolean {
|
|
if (t.text.length < 2) return false;
|
|
const open = t.text[0]!;
|
|
if (open === "$") {
|
|
const tag = /^\$[\p{L}\p{N}_]*\$/u.exec(t.text)?.[0] ?? "$";
|
|
return t.text.length >= tag.length * 2 && t.text.endsWith(tag);
|
|
}
|
|
return t.text.endsWith(open);
|
|
}
|
|
|
|
/** The identifier a word / quoted token names (quotes removed, doubled quotes collapsed). */
|
|
export function identName(t: Token): string {
|
|
if (t.kind !== "quoted") return t.text;
|
|
if (t.value !== undefined) return t.value;
|
|
const q = t.text[0]!;
|
|
const body = t.text.endsWith(q) && t.text.length > 1 ? t.text.slice(1, -1) : t.text.slice(1);
|
|
return body.split(q + q).join(q);
|
|
}
|