From 8a32322c187ccc60ec7eafb61a9678f267a82282 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 25 Mar 2026 15:38:29 +0100 Subject: [PATCH] fix: auto-generate datatable SDK reference for app mode system prompt (#8522) The app mode AI chat system prompt had hand-written datatable API docs that were missing methods (fetchOneScalar, execute, query). This adds datatable-specific extraction to generate.py so the prompt stays in sync with the actual TypeScript and Python client APIs. Co-authored-by: Claude Opus 4.6 (1M context) --- .../lib/components/copilot/chat/app/core.ts | 17 +- system_prompts/auto-generated/index.d.ts | 1 + system_prompts/auto-generated/index.ts | 8 + system_prompts/auto-generated/prompts.ts | 148 ++++++++++++ .../auto-generated/sdks/datatable-python.md | 68 ++++++ .../sdks/datatable-typescript.md | 76 +++++++ system_prompts/generate.py | 215 ++++++++++++++++++ 7 files changed, 521 insertions(+), 12 deletions(-) create mode 100644 system_prompts/auto-generated/sdks/datatable-python.md create mode 100644 system_prompts/auto-generated/sdks/datatable-typescript.md diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 8fc88d4d16..c6a64a3e5a 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -10,6 +10,7 @@ import { createGetRunnableDetailsTool, type Tool } from '../shared' +import { getDatatableSdkReference } from '$system_prompts' import { aiChatManager } from '../AIChatManager.svelte' import type { ContextElement, @@ -842,38 +843,30 @@ For inline scripts, the code must have a \`main\` function as its entrypoint. Backend runnables should only perform **data operations** (SELECT, INSERT, UPDATE, DELETE) on **existing tables**. Never use CREATE TABLE, DROP TABLE, or ALTER TABLE inside runnables. -**TypeScript (Bun)**: +**TypeScript (Bun) example**: \`\`\`typescript import * as wmill from 'windmill-client'; export async function main(user_id: string) { const sql = ${datatableCall}; - - // Safe string interpolation (parameterized query) const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne(); return user; } \`\`\` -**Python**: +**Python example**: \`\`\`python import wmill def main(user_id: str): db = ${datatableCall} - - # Use positional arguments ($1, $2, etc.) user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one() return user \`\`\` -### Common Operations (for use in backend runnables) +### Datatable Client API Reference -- **Fetch all**: \`sql\`SELECT * FROM ${schemaPrefix}table\`.fetch()\` or \`db.query('SELECT * FROM ${schemaPrefix}table').fetch()\` -- **Fetch one**: \`.fetchOne()\` or \`.fetch_one()\` -- **Insert**: \`sql\`INSERT INTO ${schemaPrefix}table (col) VALUES (\${value})\`\` -- **Update**: \`sql\`UPDATE ${schemaPrefix}table SET col = \${value} WHERE id = \${id}\`\` -- **Delete**: \`sql\`DELETE FROM ${schemaPrefix}table WHERE id = \${id}\`\` +${getDatatableSdkReference()} ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY diff --git a/system_prompts/auto-generated/index.d.ts b/system_prompts/auto-generated/index.d.ts index ab649a2090..4f2dd469ac 100644 --- a/system_prompts/auto-generated/index.d.ts +++ b/system_prompts/auto-generated/index.d.ts @@ -1,3 +1,4 @@ export * from './prompts'; export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; +export declare function getDatatableSdkReference(): string; diff --git a/system_prompts/auto-generated/index.ts b/system_prompts/auto-generated/index.ts index b283892470..b2c4bf0ec8 100644 --- a/system_prompts/auto-generated/index.ts +++ b/system_prompts/auto-generated/index.ts @@ -37,3 +37,11 @@ export function getFlowPrompt(): string { prompts.OPENFLOW_SCHEMA ].filter(Boolean).join('\n\n'); } + +// Helper to get datatable SDK reference for app mode +export function getDatatableSdkReference(): string { + return [ + prompts.DATATABLE_SDK_TYPESCRIPT, + prompts.DATATABLE_SDK_PYTHON + ].filter(Boolean).join('\n\n'); +} diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index d30a5be3eb..6eb870e62b 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1371,6 +1371,154 @@ async def parallel(items, fn, concurrency: Optional[int] = None) # offset: Message offset to commit (from event['offset']) def commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None +`; + +export const DATATABLE_SDK_TYPESCRIPT = `## TypeScript Datatable API (windmill-client) + +Import: \`import * as wmill from 'windmill-client'\` + +SQL statement object with query content, arguments, and execution methods +\`\`\`typescript +type SqlStatement = { + /** Raw SQL content with formatted arguments */ + content: string; + + /** Argument values keyed by parameter name */ + args: Record; + + /** + * Execute the SQL query and return results + * @param params - Optional parameters including result collection mode + * @returns Query results based on the result collection mode + */ + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + /** + * Execute the SQL query and return only the first row + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; + + /** + * Execute the SQL query and return only the first row as a scalar value + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOneScalar( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise>; + + /** + * Execute the SQL query without fetching rows + * @param params - Optional parameters + */ + execute( + params?: Omit, "resultCollection"> + ): Promise; +}; +\`\`\` + +\`\`\`typescript +// Template tag function: sql\`SELECT * FROM table WHERE id = \${id}\`.fetch() +interface DatatableSqlTemplateFunction { + // Tagged template usage: + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; + query(sql: string, ...params: any[]): SqlStatement; +}; +\`\`\` + +Create a SQL template function for PostgreSQL/datatable queries +@param name - Database/datatable name (default: "main") +@returns SQL template function for building parameterized queries +@example +let sql = wmill.datatable() +let name = 'Robin' +let age = 21 +await sql\` + SELECT * FROM friends + WHERE name = \${name} AND age = \${age}::int +\`.fetch() +\`\`\`typescript +function datatable(name: string = "main"): DatatableSqlTemplateFunction +\`\`\` +`; + +export const DATATABLE_SDK_PYTHON = `## Python Datatable API (wmill) + +Import: \`import wmill\` + +# Get a DataTable client for SQL queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DataTableClient instance +def datatable(name: str = 'main') -> DataTableClient + +# Client for executing SQL queries against Windmill DataTables. +class DataTableClient: + # Initialize DataTableClient. + # + # Args: + # client: Windmill client instance + # name: DataTable name + def __init__(client: Windmill, name: str) + + # Execute a SQL query against the DataTable. + # + # Args: + # sql: SQL query string with $1, $2, etc. placeholders + # *args: Positional arguments to bind to query placeholders + # + # Returns: + # SqlQuery instance for fetching results + def query(sql: str, *args) -> SqlQuery + + +# Query result handler for DataTable and DuckLake queries. +class SqlQuery: + # Initialize SqlQuery. + # + # Args: + # sql: SQL query string + # fetch_fn: Function to execute the query + def __init__(sql: str, fetch_fn) + + # Execute query and fetch results. + # + # Args: + # result_collection: Optional result collection mode + # + # Returns: + # Query results + def fetch(result_collection: str | None = None) + + # Execute query and fetch first row of results. + # + # Returns: + # First row of query results + def fetch_one() + + # Execute query and fetch first row of results. Return result as a scalar value. + # + # Returns: + # First row of query result as a scalar value + def fetch_one_scalar() + + # Execute query and don't return any results. + # + def execute() + + `; export const OPENFLOW_SCHEMA = `## OpenFlow Schema diff --git a/system_prompts/auto-generated/sdks/datatable-python.md b/system_prompts/auto-generated/sdks/datatable-python.md new file mode 100644 index 0000000000..752019a9a3 --- /dev/null +++ b/system_prompts/auto-generated/sdks/datatable-python.md @@ -0,0 +1,68 @@ +## Python Datatable API (wmill) + +Import: `import wmill` + +# Get a DataTable client for SQL queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DataTableClient instance +def datatable(name: str = 'main') -> DataTableClient + +# Client for executing SQL queries against Windmill DataTables. +class DataTableClient: + # Initialize DataTableClient. + # + # Args: + # client: Windmill client instance + # name: DataTable name + def __init__(client: Windmill, name: str) + + # Execute a SQL query against the DataTable. + # + # Args: + # sql: SQL query string with $1, $2, etc. placeholders + # *args: Positional arguments to bind to query placeholders + # + # Returns: + # SqlQuery instance for fetching results + def query(sql: str, *args) -> SqlQuery + + +# Query result handler for DataTable and DuckLake queries. +class SqlQuery: + # Initialize SqlQuery. + # + # Args: + # sql: SQL query string + # fetch_fn: Function to execute the query + def __init__(sql: str, fetch_fn) + + # Execute query and fetch results. + # + # Args: + # result_collection: Optional result collection mode + # + # Returns: + # Query results + def fetch(result_collection: str | None = None) + + # Execute query and fetch first row of results. + # + # Returns: + # First row of query results + def fetch_one() + + # Execute query and fetch first row of results. Return result as a scalar value. + # + # Returns: + # First row of query result as a scalar value + def fetch_one_scalar() + + # Execute query and don't return any results. + # + def execute() + + diff --git a/system_prompts/auto-generated/sdks/datatable-typescript.md b/system_prompts/auto-generated/sdks/datatable-typescript.md new file mode 100644 index 0000000000..0256515911 --- /dev/null +++ b/system_prompts/auto-generated/sdks/datatable-typescript.md @@ -0,0 +1,76 @@ +## TypeScript Datatable API (windmill-client) + +Import: `import * as wmill from 'windmill-client'` + +SQL statement object with query content, arguments, and execution methods +```typescript +type SqlStatement = { + /** Raw SQL content with formatted arguments */ + content: string; + + /** Argument values keyed by parameter name */ + args: Record; + + /** + * Execute the SQL query and return results + * @param params - Optional parameters including result collection mode + * @returns Query results based on the result collection mode + */ + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + /** + * Execute the SQL query and return only the first row + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; + + /** + * Execute the SQL query and return only the first row as a scalar value + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOneScalar( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise>; + + /** + * Execute the SQL query without fetching rows + * @param params - Optional parameters + */ + execute( + params?: Omit, "resultCollection"> + ): Promise; +}; +``` + +```typescript +// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch() +interface DatatableSqlTemplateFunction { + // Tagged template usage: + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; + query(sql: string, ...params: any[]): SqlStatement; +}; +``` + +Create a SQL template function for PostgreSQL/datatable queries +@param name - Database/datatable name (default: "main") +@returns SQL template function for building parameterized queries +@example +let sql = wmill.datatable() +let name = 'Robin' +let age = 21 +await sql` + SELECT * FROM friends + WHERE name = ${name} AND age = ${age}::int +`.fetch() +```typescript +function datatable(name: string = "main"): DatatableSqlTemplateFunction +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index cf57a3cecb..034c94e20d 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -655,6 +655,202 @@ def generate_schema_files(cli_schemas: dict[str, dict]) -> dict[str, str]: return schema_yaml_content +# ============================================================================= +# Datatable SDK Extraction +# ============================================================================= + + +TS_SQL_UTILS_PATH = TS_SDK_DIR / "sqlUtils.ts" + + +def extract_datatable_ts_sdk() -> str: + """Extract datatable-specific type definitions from TypeScript SDK (sqlUtils.ts). + + Reads the source file and extracts the public API surface: + - SqlStatement type (fetch, fetchOne, fetchOneScalar, execute methods) + - DatatableSqlTemplateFunction interface (template tag + query method) + - datatable() function signature + """ + if not TS_SQL_UTILS_PATH.exists(): + print(f" Warning: sqlUtils.ts not found at {TS_SQL_UTILS_PATH}") + return '' + + content = TS_SQL_UTILS_PATH.read_text() + + md = "## TypeScript Datatable API (windmill-client)\n\n" + md += "Import: `import * as wmill from 'windmill-client'`\n\n" + + # Extract exported type/interface/function definitions from sqlUtils.ts + # We use extract_balanced to handle nested braces correctly + + # 1. Extract SqlStatement type + match = re.search(r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+type\s+SqlStatement\s*=\s*', content) + if match: + jsdoc_raw = match.group(1) + brace_start = content.index('{', match.end() - 1) + body, end = extract_balanced(content, brace_start, '{', '}') + if end != -1: + if jsdoc_raw: + md += clean_jsdoc(jsdoc_raw) + "\n" + md += "```typescript\n" + md += f"type SqlStatement = {{\n{_indent_body(body)}\n}};\n" + md += "```\n\n" + + # 2. Extract DatatableSqlTemplateFunction interface + match = re.search( + r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+interface\s+DatatableSqlTemplateFunction\s+extends\s+SqlTemplateFunction\s*', + content + ) + if match: + brace_start = content.index('{', match.end() - 1) + body, end = extract_balanced(content, brace_start, '{', '}') + if end != -1: + md += "```typescript\n" + md += "// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch()\n" + md += f"interface DatatableSqlTemplateFunction {{\n" + md += f" // Tagged template usage:\n" + md += f" (strings: TemplateStringsArray, ...values: any[]): SqlStatement;\n" + md += f"{_indent_body(body)}\n" + md += "};\n" + md += "```\n\n" + + # 3. Extract datatable() function + match = re.search( + r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+function\s+datatable\s*\(([^)]*)\)\s*:\s*(\S+)', + content + ) + if match: + jsdoc_raw, params, return_type = match.groups() + if jsdoc_raw: + md += clean_jsdoc(jsdoc_raw) + "\n" + md += "```typescript\n" + md += f"function datatable({params.strip()}): {return_type}\n" + md += "```\n" + + return md + + +def extract_datatable_py_sdk(py_content: str) -> str: + """Extract datatable-specific class/function definitions from Python SDK. + + Uses Python AST to extract: + - datatable() function + - DataTableClient class with query() method + - SqlQuery class with fetch(), fetch_one(), fetch_one_scalar(), execute() methods + """ + if not py_content: + return '' + + try: + tree = ast.parse(py_content) + except SyntaxError as e: + print(f" Warning: Could not parse Python SDK for datatable extraction: {e}") + return '' + + md = "## Python Datatable API (wmill)\n\n" + md += "Import: `import wmill`\n\n" + + # Target classes and the top-level datatable function + target_classes = {'DataTableClient', 'SqlQuery'} + + # 1. Extract datatable() top-level function + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == 'datatable': + docstring = ast.get_docstring(node) or '' + params = _format_py_params(node) + return_ann = f" -> {ast.unparse(node.returns)}" if node.returns else '' + if docstring: + for line in docstring.split('\n'): + md += f"# {line}\n" + md += f"def datatable({params}){return_ann}\n\n" + break + + # 2. Extract target classes with their public methods + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name in target_classes: + class_doc = ast.get_docstring(node) or '' + if class_doc: + for line in class_doc.split('\n'): + md += f"# {line}\n" + md += f"class {node.name}:\n" + + for item in node.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + if item.name.startswith('_') and item.name != '__init__': + continue + docstring = ast.get_docstring(item) or '' + params = _format_py_params(item, skip_self=True) + return_ann = f" -> {ast.unparse(item.returns)}" if item.returns else '' + async_prefix = 'async ' if isinstance(item, ast.AsyncFunctionDef) else '' + if docstring: + for line in docstring.split('\n'): + md += f" # {line}\n" + md += f" {async_prefix}def {item.name}({params}){return_ann}\n\n" + + md += "\n" + + return md + + +def _format_py_params(node: ast.FunctionDef, skip_self: bool = False) -> str: + """Format function parameters from AST node.""" + params = [] + args = node.args + num_defaults = len(args.defaults) + num_args = len(args.args) + + for i, arg in enumerate(args.args): + if skip_self and arg.arg == 'self': + continue + param_str = arg.arg + if arg.annotation: + param_str += f": {ast.unparse(arg.annotation)}" + default_idx = i - (num_args - num_defaults) + if default_idx >= 0: + default = args.defaults[default_idx] + param_str += f" = {ast.unparse(default)}" + params.append(param_str) + + if args.vararg: + vararg_str = f"*{args.vararg.arg}" + if args.vararg.annotation: + vararg_str += f": {ast.unparse(args.vararg.annotation)}" + params.append(vararg_str) + + for i, arg in enumerate(args.kwonlyargs): + param_str = arg.arg + if arg.annotation: + param_str += f": {ast.unparse(arg.annotation)}" + if args.kw_defaults[i]: + param_str += f" = {ast.unparse(args.kw_defaults[i])}" + params.append(param_str) + + if args.kwarg: + kwarg_str = f"**{args.kwarg.arg}" + if args.kwarg.annotation: + kwarg_str += f": {ast.unparse(args.kwarg.annotation)}" + params.append(kwarg_str) + + return ', '.join(params) + + +def _indent_body(body: str) -> str: + """Clean and re-indent a type body for readable output.""" + lines = body.strip().split('\n') + result = [] + for line in lines: + stripped = line.strip() + if stripped: + # Keep JSDoc comments and method signatures with consistent indentation + if not stripped.startswith('//') and not stripped.startswith('/*') and not stripped.startswith('*'): + result.append(f" {stripped}") + else: + result.append(f" {stripped}") + else: + result.append('') + return '\n'.join(result) + + # ============================================================================= # Skill Generation # ============================================================================= @@ -947,6 +1143,13 @@ def main(): (OUTPUT_SDKS_DIR / "python.md").write_text(py_sdk_md) print(f" Found {len(py_functions)} functions, {len(py_classes)} classes") + # Extract datatable-specific SDK docs (for app mode system prompt) + print("Extracting datatable SDK docs...") + datatable_ts_md = extract_datatable_ts_sdk() + datatable_py_md = extract_datatable_py_sdk(py_content) + (OUTPUT_SDKS_DIR / "datatable-typescript.md").write_text(datatable_ts_md) + (OUTPUT_SDKS_DIR / "datatable-python.md").write_text(datatable_py_md) + # Read base prompts print("Assembling complete prompts...") base_dir = SCRIPT_DIR / "base" @@ -1009,6 +1212,10 @@ def main(): 'SDK_TYPESCRIPT': ts_sdk_md, 'SDK_PYTHON': py_sdk_md, + # Datatable-specific SDK docs (for app mode) + 'DATATABLE_SDK_TYPESCRIPT': datatable_ts_md, + 'DATATABLE_SDK_PYTHON': datatable_py_md, + # Schema (raw YAML content) 'OPENFLOW_SCHEMA': openflow_content, @@ -1077,6 +1284,14 @@ export function getFlowPrompt(): string { prompts.OPENFLOW_SCHEMA ].filter(Boolean).join('\\n\\n'); } + +// Helper to get datatable SDK reference for app mode +export function getDatatableSdkReference(): string { + return [ + prompts.DATATABLE_SDK_TYPESCRIPT, + prompts.DATATABLE_SDK_PYTHON + ].filter(Boolean).join('\\n\\n'); +} """ (OUTPUT_GENERATED_DIR / "index.ts").write_text(index_content)