fix: auto-generate datatable SDK reference for app mode system prompt (#8522)

The app mode AI chat system prompt had hand-written datatable API docs
that were missing methods (fetchOneScalar, execute, query). This adds
datatable-specific extraction to generate.py so the prompt stays in
sync with the actual TypeScript and Python client APIs.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

frontend/src/lib/components/copilot/chat/app/core.ts

@@ -10,6 +10,7 @@ import {
 	createGetRunnableDetailsTool,
 	type Tool
 } from '../shared'
+import { getDatatableSdkReference } from '$system_prompts'
 import { aiChatManager } from '../AIChatManager.svelte'
 import type {
 	ContextElement,
@@ -842,38 +843,30 @@ For inline scripts, the code must have a \`main\` function as its entrypoint.
 
 Backend runnables should only perform **data operations** (SELECT, INSERT, UPDATE, DELETE) on **existing tables**. Never use CREATE TABLE, DROP TABLE, or ALTER TABLE inside runnables.
 
-**TypeScript (Bun)**:
+**TypeScript (Bun) example**:
 \`\`\`typescript
 import * as wmill from 'windmill-client';
 
 export async function main(user_id: string) {
   const sql = ${datatableCall};
-
-  // Safe string interpolation (parameterized query)
   const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne();
   return user;
 }
 \`\`\`
 
-**Python**:
+**Python example**:
 \`\`\`python
 import wmill
 
 def main(user_id: str):
     db = ${datatableCall}
-
-    # Use positional arguments ($1, $2, etc.)
     user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one()
     return user
 \`\`\`
 
-### Common Operations (for use in backend runnables)
+### Datatable Client API Reference
 
-- **Fetch all**: \`sql\`SELECT * FROM ${schemaPrefix}table\`.fetch()\` or \`db.query('SELECT * FROM ${schemaPrefix}table').fetch()\`
-- **Fetch one**: \`.fetchOne()\` or \`.fetch_one()\`
-- **Insert**: \`sql\`INSERT INTO ${schemaPrefix}table (col) VALUES (\${value})\`\`
-- **Update**: \`sql\`UPDATE ${schemaPrefix}table SET col = \${value} WHERE id = \${id}\`\`
-- **Delete**: \`sql\`DELETE FROM ${schemaPrefix}table WHERE id = \${id}\`\`
+${getDatatableSdkReference()}
 
 ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY
 

system_prompts/auto-generated/index.d.ts

@@ -1,3 +1,4 @@
 export * from './prompts';
 export declare function getScriptPrompt(language: string): string;
 export declare function getFlowPrompt(): string;
+export declare function getDatatableSdkReference(): string;

system_prompts/auto-generated/index.ts

@@ -37,3 +37,11 @@ export function getFlowPrompt(): string {
     prompts.OPENFLOW_SCHEMA
   ].filter(Boolean).join('\n\n');
 }
+
+// Helper to get datatable SDK reference for app mode
+export function getDatatableSdkReference(): string {
+  return [
+    prompts.DATATABLE_SDK_TYPESCRIPT,
+    prompts.DATATABLE_SDK_PYTHON
+  ].filter(Boolean).join('\n\n');
+}

system_prompts/auto-generated/prompts.ts

@@ -1371,6 +1371,154 @@ async def parallel(items, fn, concurrency: Optional[int] = None)
 #     offset: Message offset to commit (from event['offset'])
 def commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None
 
+`;
+
+export const DATATABLE_SDK_TYPESCRIPT = `## TypeScript Datatable API (windmill-client)
+
+Import: \`import * as wmill from 'windmill-client'\`
+
+SQL statement object with query content, arguments, and execution methods
+\`\`\`typescript
+type SqlStatement<T> = {
+  /** Raw SQL content with formatted arguments */
+  content: string;
+
+  /** Argument values keyed by parameter name */
+  args: Record<string, any>;
+
+  /**
+  * Execute the SQL query and return results
+  * @param params - Optional parameters including result collection mode
+  * @returns Query results based on the result collection mode
+  */
+  fetch<ResultCollectionT extends ResultCollection = "last_statement_all_rows">(
+  params?: FetchParams<ResultCollectionT | ResultCollection> // The union is for auto-completion
+  ): Promise<SqlResult<T, ResultCollectionT>>;
+
+  /**
+  * Execute the SQL query and return only the first row
+  * @param params - Optional parameters
+  * @returns First row of the query result
+  */
+  fetchOne(
+  params?: Omit<FetchParams<"last_statement_first_row">, "resultCollection">
+  ): Promise<SqlResult<T, "last_statement_first_row">>;
+
+  /**
+  * Execute the SQL query and return only the first row as a scalar value
+  * @param params - Optional parameters
+  * @returns First row of the query result
+  */
+  fetchOneScalar(
+  params?: Omit<
+  FetchParams<"last_statement_first_row_scalar">,
+  "resultCollection"
+  >
+  ): Promise<SqlResult<T, "last_statement_first_row_scalar">>;
+
+  /**
+  * Execute the SQL query without fetching rows
+  * @param params - Optional parameters
+  */
+  execute(
+  params?: Omit<FetchParams<"last_statement_first_row">, "resultCollection">
+  ): Promise<void>;
+};
+\`\`\`
+
+\`\`\`typescript
+// Template tag function: sql\`SELECT * FROM table WHERE id = \${id}\`.fetch()
+interface DatatableSqlTemplateFunction {
+  // Tagged template usage:
+  <T = any>(strings: TemplateStringsArray, ...values: any[]): SqlStatement<T>;
+  query<T = any>(sql: string, ...params: any[]): SqlStatement<T>;
+};
+\`\`\`
+
+Create a SQL template function for PostgreSQL/datatable queries
+@param name - Database/datatable name (default: "main")
+@returns SQL template function for building parameterized queries
+@example
+let sql = wmill.datatable()
+let name = 'Robin'
+let age = 21
+await sql\`
+  SELECT * FROM friends
+    WHERE name = \${name} AND age = \${age}::int
+\`.fetch()
+\`\`\`typescript
+function datatable(name: string = "main"): DatatableSqlTemplateFunction
+\`\`\`
+`;
+
+export const DATATABLE_SDK_PYTHON = `## Python Datatable API (wmill)
+
+Import: \`import wmill\`
+
+# Get a DataTable client for SQL queries.
+# 
+# Args:
+#     name: Database name (default: "main")
+# 
+# Returns:
+#     DataTableClient instance
+def datatable(name: str = 'main') -> DataTableClient
+
+# Client for executing SQL queries against Windmill DataTables.
+class DataTableClient:
+    # Initialize DataTableClient.
+    # 
+    # Args:
+    #     client: Windmill client instance
+    #     name: DataTable name
+    def __init__(client: Windmill, name: str)
+
+    # Execute a SQL query against the DataTable.
+    # 
+    # Args:
+    #     sql: SQL query string with $1, $2, etc. placeholders
+    #     *args: Positional arguments to bind to query placeholders
+    # 
+    # Returns:
+    #     SqlQuery instance for fetching results
+    def query(sql: str, *args) -> SqlQuery
+
+
+# Query result handler for DataTable and DuckLake queries.
+class SqlQuery:
+    # Initialize SqlQuery.
+    # 
+    # Args:
+    #     sql: SQL query string
+    #     fetch_fn: Function to execute the query
+    def __init__(sql: str, fetch_fn)
+
+    # Execute query and fetch results.
+    # 
+    # Args:
+    #     result_collection: Optional result collection mode
+    # 
+    # Returns:
+    #     Query results
+    def fetch(result_collection: str | None = None)
+
+    # Execute query and fetch first row of results.
+    # 
+    # Returns:
+    #     First row of query results
+    def fetch_one()
+
+    # Execute query and fetch first row of results. Return result as a scalar value.
+    # 
+    # Returns:
+    #     First row of query result as a scalar value
+    def fetch_one_scalar()
+
+    # Execute query and don't return any results.
+    #         
+    def execute()
+
+
 `;
 
 export const OPENFLOW_SCHEMA = `## OpenFlow Schema

system_prompts/auto-generated/sdks/datatable-python.md

@@ -0,0 +1,68 @@
+## Python Datatable API (wmill)
+
+Import: `import wmill`
+
+# Get a DataTable client for SQL queries.
+# 
+# Args:
+#     name: Database name (default: "main")
+# 
+# Returns:
+#     DataTableClient instance
+def datatable(name: str = 'main') -> DataTableClient
+
+# Client for executing SQL queries against Windmill DataTables.
+class DataTableClient:
+    # Initialize DataTableClient.
+    # 
+    # Args:
+    #     client: Windmill client instance
+    #     name: DataTable name
+    def __init__(client: Windmill, name: str)
+
+    # Execute a SQL query against the DataTable.
+    # 
+    # Args:
+    #     sql: SQL query string with $1, $2, etc. placeholders
+    #     *args: Positional arguments to bind to query placeholders
+    # 
+    # Returns:
+    #     SqlQuery instance for fetching results
+    def query(sql: str, *args) -> SqlQuery
+
+
+# Query result handler for DataTable and DuckLake queries.
+class SqlQuery:
+    # Initialize SqlQuery.
+    # 
+    # Args:
+    #     sql: SQL query string
+    #     fetch_fn: Function to execute the query
+    def __init__(sql: str, fetch_fn)
+
+    # Execute query and fetch results.
+    # 
+    # Args:
+    #     result_collection: Optional result collection mode
+    # 
+    # Returns:
+    #     Query results
+    def fetch(result_collection: str | None = None)
+
+    # Execute query and fetch first row of results.
+    # 
+    # Returns:
+    #     First row of query results
+    def fetch_one()
+
+    # Execute query and fetch first row of results. Return result as a scalar value.
+    # 
+    # Returns:
+    #     First row of query result as a scalar value
+    def fetch_one_scalar()
+
+    # Execute query and don't return any results.
+    #         
+    def execute()
+
+

system_prompts/auto-generated/sdks/datatable-typescript.md

@@ -0,0 +1,76 @@
+## TypeScript Datatable API (windmill-client)
+
+Import: `import * as wmill from 'windmill-client'`
+
+SQL statement object with query content, arguments, and execution methods
+```typescript
+type SqlStatement<T> = {
+  /** Raw SQL content with formatted arguments */
+  content: string;
+
+  /** Argument values keyed by parameter name */
+  args: Record<string, any>;
+
+  /**
+  * Execute the SQL query and return results
+  * @param params - Optional parameters including result collection mode
+  * @returns Query results based on the result collection mode
+  */
+  fetch<ResultCollectionT extends ResultCollection = "last_statement_all_rows">(
+  params?: FetchParams<ResultCollectionT | ResultCollection> // The union is for auto-completion
+  ): Promise<SqlResult<T, ResultCollectionT>>;
+
+  /**
+  * Execute the SQL query and return only the first row
+  * @param params - Optional parameters
+  * @returns First row of the query result
+  */
+  fetchOne(
+  params?: Omit<FetchParams<"last_statement_first_row">, "resultCollection">
+  ): Promise<SqlResult<T, "last_statement_first_row">>;
+
+  /**
+  * Execute the SQL query and return only the first row as a scalar value
+  * @param params - Optional parameters
+  * @returns First row of the query result
+  */
+  fetchOneScalar(
+  params?: Omit<
+  FetchParams<"last_statement_first_row_scalar">,
+  "resultCollection"
+  >
+  ): Promise<SqlResult<T, "last_statement_first_row_scalar">>;
+
+  /**
+  * Execute the SQL query without fetching rows
+  * @param params - Optional parameters
+  */
+  execute(
+  params?: Omit<FetchParams<"last_statement_first_row">, "resultCollection">
+  ): Promise<void>;
+};
+```
+
+```typescript
+// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch()
+interface DatatableSqlTemplateFunction {
+  // Tagged template usage:
+  <T = any>(strings: TemplateStringsArray, ...values: any[]): SqlStatement<T>;
+  query<T = any>(sql: string, ...params: any[]): SqlStatement<T>;
+};
+```
+
+Create a SQL template function for PostgreSQL/datatable queries
+@param name - Database/datatable name (default: "main")
+@returns SQL template function for building parameterized queries
+@example
+let sql = wmill.datatable()
+let name = 'Robin'
+let age = 21
+await sql`
+  SELECT * FROM friends
+    WHERE name = ${name} AND age = ${age}::int
+`.fetch()
+```typescript
+function datatable(name: string = "main"): DatatableSqlTemplateFunction
+```

system_prompts/generate.py

@@ -655,6 +655,202 @@ def generate_schema_files(cli_schemas: dict[str, dict]) -> dict[str, str]:
     return schema_yaml_content
 
 
+# =============================================================================
+# Datatable SDK Extraction
+# =============================================================================
+
+
+TS_SQL_UTILS_PATH = TS_SDK_DIR / "sqlUtils.ts"
+
+
+def extract_datatable_ts_sdk() -> str:
+    """Extract datatable-specific type definitions from TypeScript SDK (sqlUtils.ts).
+
+    Reads the source file and extracts the public API surface:
+    - SqlStatement<T> type (fetch, fetchOne, fetchOneScalar, execute methods)
+    - DatatableSqlTemplateFunction interface (template tag + query method)
+    - datatable() function signature
+    """
+    if not TS_SQL_UTILS_PATH.exists():
+        print(f"  Warning: sqlUtils.ts not found at {TS_SQL_UTILS_PATH}")
+        return ''
+
+    content = TS_SQL_UTILS_PATH.read_text()
+
+    md = "## TypeScript Datatable API (windmill-client)\n\n"
+    md += "Import: `import * as wmill from 'windmill-client'`\n\n"
+
+    # Extract exported type/interface/function definitions from sqlUtils.ts
+    # We use extract_balanced to handle nested braces correctly
+
+    # 1. Extract SqlStatement<T> type
+    match = re.search(r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+type\s+SqlStatement<T>\s*=\s*', content)
+    if match:
+        jsdoc_raw = match.group(1)
+        brace_start = content.index('{', match.end() - 1)
+        body, end = extract_balanced(content, brace_start, '{', '}')
+        if end != -1:
+            if jsdoc_raw:
+                md += clean_jsdoc(jsdoc_raw) + "\n"
+            md += "```typescript\n"
+            md += f"type SqlStatement<T> = {{\n{_indent_body(body)}\n}};\n"
+            md += "```\n\n"
+
+    # 2. Extract DatatableSqlTemplateFunction interface
+    match = re.search(
+        r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+interface\s+DatatableSqlTemplateFunction\s+extends\s+SqlTemplateFunction\s*',
+        content
+    )
+    if match:
+        brace_start = content.index('{', match.end() - 1)
+        body, end = extract_balanced(content, brace_start, '{', '}')
+        if end != -1:
+            md += "```typescript\n"
+            md += "// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch()\n"
+            md += f"interface DatatableSqlTemplateFunction {{\n"
+            md += f"  // Tagged template usage:\n"
+            md += f"  <T = any>(strings: TemplateStringsArray, ...values: any[]): SqlStatement<T>;\n"
+            md += f"{_indent_body(body)}\n"
+            md += "};\n"
+            md += "```\n\n"
+
+    # 3. Extract datatable() function
+    match = re.search(
+        r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+function\s+datatable\s*\(([^)]*)\)\s*:\s*(\S+)',
+        content
+    )
+    if match:
+        jsdoc_raw, params, return_type = match.groups()
+        if jsdoc_raw:
+            md += clean_jsdoc(jsdoc_raw) + "\n"
+        md += "```typescript\n"
+        md += f"function datatable({params.strip()}): {return_type}\n"
+        md += "```\n"
+
+    return md
+
+
+def extract_datatable_py_sdk(py_content: str) -> str:
+    """Extract datatable-specific class/function definitions from Python SDK.
+
+    Uses Python AST to extract:
+    - datatable() function
+    - DataTableClient class with query() method
+    - SqlQuery class with fetch(), fetch_one(), fetch_one_scalar(), execute() methods
+    """
+    if not py_content:
+        return ''
+
+    try:
+        tree = ast.parse(py_content)
+    except SyntaxError as e:
+        print(f"  Warning: Could not parse Python SDK for datatable extraction: {e}")
+        return ''
+
+    md = "## Python Datatable API (wmill)\n\n"
+    md += "Import: `import wmill`\n\n"
+
+    # Target classes and the top-level datatable function
+    target_classes = {'DataTableClient', 'SqlQuery'}
+
+    # 1. Extract datatable() top-level function
+    for node in tree.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == 'datatable':
+            docstring = ast.get_docstring(node) or ''
+            params = _format_py_params(node)
+            return_ann = f" -> {ast.unparse(node.returns)}" if node.returns else ''
+            if docstring:
+                for line in docstring.split('\n'):
+                    md += f"# {line}\n"
+            md += f"def datatable({params}){return_ann}\n\n"
+            break
+
+    # 2. Extract target classes with their public methods
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef) and node.name in target_classes:
+            class_doc = ast.get_docstring(node) or ''
+            if class_doc:
+                for line in class_doc.split('\n'):
+                    md += f"# {line}\n"
+            md += f"class {node.name}:\n"
+
+            for item in node.body:
+                if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    if item.name.startswith('_') and item.name != '__init__':
+                        continue
+                    docstring = ast.get_docstring(item) or ''
+                    params = _format_py_params(item, skip_self=True)
+                    return_ann = f" -> {ast.unparse(item.returns)}" if item.returns else ''
+                    async_prefix = 'async ' if isinstance(item, ast.AsyncFunctionDef) else ''
+                    if docstring:
+                        for line in docstring.split('\n'):
+                            md += f"    # {line}\n"
+                    md += f"    {async_prefix}def {item.name}({params}){return_ann}\n\n"
+
+            md += "\n"
+
+    return md
+
+
+def _format_py_params(node: ast.FunctionDef, skip_self: bool = False) -> str:
+    """Format function parameters from AST node."""
+    params = []
+    args = node.args
+    num_defaults = len(args.defaults)
+    num_args = len(args.args)
+
+    for i, arg in enumerate(args.args):
+        if skip_self and arg.arg == 'self':
+            continue
+        param_str = arg.arg
+        if arg.annotation:
+            param_str += f": {ast.unparse(arg.annotation)}"
+        default_idx = i - (num_args - num_defaults)
+        if default_idx >= 0:
+            default = args.defaults[default_idx]
+            param_str += f" = {ast.unparse(default)}"
+        params.append(param_str)
+
+    if args.vararg:
+        vararg_str = f"*{args.vararg.arg}"
+        if args.vararg.annotation:
+            vararg_str += f": {ast.unparse(args.vararg.annotation)}"
+        params.append(vararg_str)
+
+    for i, arg in enumerate(args.kwonlyargs):
+        param_str = arg.arg
+        if arg.annotation:
+            param_str += f": {ast.unparse(arg.annotation)}"
+        if args.kw_defaults[i]:
+            param_str += f" = {ast.unparse(args.kw_defaults[i])}"
+        params.append(param_str)
+
+    if args.kwarg:
+        kwarg_str = f"**{args.kwarg.arg}"
+        if args.kwarg.annotation:
+            kwarg_str += f": {ast.unparse(args.kwarg.annotation)}"
+        params.append(kwarg_str)
+
+    return ', '.join(params)
+
+
+def _indent_body(body: str) -> str:
+    """Clean and re-indent a type body for readable output."""
+    lines = body.strip().split('\n')
+    result = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped:
+            # Keep JSDoc comments and method signatures with consistent indentation
+            if not stripped.startswith('//') and not stripped.startswith('/*') and not stripped.startswith('*'):
+                result.append(f"  {stripped}")
+            else:
+                result.append(f"  {stripped}")
+        else:
+            result.append('')
+    return '\n'.join(result)
+
+
 # =============================================================================
 # Skill Generation
 # =============================================================================
@@ -947,6 +1143,13 @@ def main():
     (OUTPUT_SDKS_DIR / "python.md").write_text(py_sdk_md)
     print(f"  Found {len(py_functions)} functions, {len(py_classes)} classes")
 
+    # Extract datatable-specific SDK docs (for app mode system prompt)
+    print("Extracting datatable SDK docs...")
+    datatable_ts_md = extract_datatable_ts_sdk()
+    datatable_py_md = extract_datatable_py_sdk(py_content)
+    (OUTPUT_SDKS_DIR / "datatable-typescript.md").write_text(datatable_ts_md)
+    (OUTPUT_SDKS_DIR / "datatable-python.md").write_text(datatable_py_md)
+
     # Read base prompts
     print("Assembling complete prompts...")
     base_dir = SCRIPT_DIR / "base"
@@ -1009,6 +1212,10 @@ def main():
         'SDK_TYPESCRIPT': ts_sdk_md,
         'SDK_PYTHON': py_sdk_md,
 
+        # Datatable-specific SDK docs (for app mode)
+        'DATATABLE_SDK_TYPESCRIPT': datatable_ts_md,
+        'DATATABLE_SDK_PYTHON': datatable_py_md,
+
         # Schema (raw YAML content)
         'OPENFLOW_SCHEMA': openflow_content,
 
@@ -1077,6 +1284,14 @@ export function getFlowPrompt(): string {
     prompts.OPENFLOW_SCHEMA
   ].filter(Boolean).join('\\n\\n');
 }
+
+// Helper to get datatable SDK reference for app mode
+export function getDatatableSdkReference(): string {
+  return [
+    prompts.DATATABLE_SDK_TYPESCRIPT,
+    prompts.DATATABLE_SDK_PYTHON
+  ].filter(Boolean).join('\\n\\n');
+}
 """
     (OUTPUT_GENERATED_DIR / "index.ts").write_text(index_content)