feat(mcp): add script preview testing tool (#6417)

* add endpoint to test script

* add same for flow

* better tool spec + remove flow preview from tools

* fix

* fix wrong required fields

* feat(mcp): add warning for missing required fields in schema properties

- Add stderr warning when x-mcp-required-fields contains fields not found in body schema properties
- Prevents silent misconfigurations in MCP tool generation
- Helps debug schema validation issues

Co-authored-by: centdix <centdix@users.noreply.github.com>

---------

Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
Co-authored-by: centdix <centdix@users.noreply.github.com>

backend/generate_mcp_endpoints_tools/generate_mcp_tools.py

@@ -41,7 +41,7 @@ def load_openapi_spec(file_path: str) -> Dict[str, Any]:
         print(f"Error loading OpenAPI spec: {e}", file=sys.stderr)
         sys.exit(1)
 
-def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Optional[Dict[str, Any]], spec: Dict[str, Any]) -> tuple:
+def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Optional[Dict[str, Any]], spec: Dict[str, Any], required_fields: Optional[List[str]] = None) -> tuple:
     """Extract separate schemas for path parameters, query parameters, and request body."""
     path_params_schema = {
         "type": "object",
@@ -92,6 +92,20 @@ def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Opt
     # Process request body if present
     if request_body:
         body_schema = extract_request_body_schema(request_body, spec)
+        
+        # If we have required fields specified and a body schema, update the required array
+        if body_schema and required_fields:
+            if 'required' not in body_schema:
+                body_schema['required'] = []
+            
+            # Add each required field if it exists in the schema properties
+            for field in required_fields:
+                if 'properties' in body_schema and field in body_schema['properties']:
+                    if field not in body_schema['required']:
+                        body_schema['required'].append(field)
+                else:
+                    # Log warning when a required field is missing from schema properties
+                    print(f"Warning: Required field '{field}' not found in body schema properties", file=sys.stderr)
     
     # Return None for empty schemas
     path_params_schema = path_params_schema if path_params_schema['properties'] else None
@@ -199,6 +213,7 @@ def find_mcp_tools(spec: Dict[str, Any]) -> List[Dict[str, Any]]:
                     'method': method.upper(),
                     'parameters': operation.get('parameters', []),
                     'requestBody': operation.get('requestBody'),
+                    'required_fields': operation.get('x-mcp-required-fields', []),
                 }
                 tools.append(tool)
     
@@ -227,7 +242,7 @@ pub fn all_tools() -> Vec<EndpointTool> {
         
         # Generate separate schemas
         path_params_schema, query_params_schema, body_schema = extract_separate_schemas(
-            tool['parameters'], tool['requestBody'], spec
+            tool['parameters'], tool['requestBody'], spec, tool['required_fields']
         )
         
         path_params_rust = schema_to_rust_value(path_params_schema)

backend/windmill-api/openapi.yaml

@@ -7193,6 +7193,36 @@ paths:
                 type: string
                 format: uuid
 
+  /w/{workspace}/jobs/run_wait_result/preview:
+    post:
+      summary: run script preview and wait for result
+      operationId: runScriptPreviewAndWaitResult
+      x-mcp-tool: true
+      x-mcp-instructions: Allows testing a script before deploying it. For typescript code, the language to send is either bun or deno. By default, send bun if no deno specific code is detected.
+      x-mcp-required-fields:
+        - content
+        - language
+        - args
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+
+      requestBody:
+        description: preview
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/Preview"
+
+      responses:
+        "200":
+          description: job result
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/jobs/workflow_as_code/{job_id}/{entrypoint}:
     post:
       summary: run code-workflow task
@@ -7302,6 +7332,30 @@ paths:
                 type: string
                 format: uuid
 
+  /w/{workspace}/jobs/run_wait_result/preview_flow:
+    post:
+      summary: run flow preview and wait for result
+      operationId: runFlowPreviewAndWaitResult
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+
+      requestBody:
+        description: preview
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/FlowPreview"
+
+      responses:
+        "200":
+          description: job result
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/jobs/queue/list:
     get:
       summary: list all queued jobs
@@ -15170,10 +15224,13 @@ components:
       properties:
         content:
           type: string
+          description: The code to run
         path:
           type: string
+          description: The path to the script
         script_hash:
           type: string
+          description: The hash of the script
         args:
           $ref: "#/components/schemas/ScriptArgs"
         language:

backend/windmill-api/src/jobs.rs

@@ -177,12 +177,14 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route("/run/preview", post(run_preview_script))
+        .route("/run_wait_result/preview", post(run_wait_result_preview_script))
         .route(
             "/run/preview_bundle",
             post(run_bundle_preview_script).layer(axum::extract::DefaultBodyLimit::disable()),
         )
         .route("/add_batch_jobs/:n", post(add_batch_jobs))
         .route("/run/preview_flow", post(run_preview_flow_job))
+        .route("/run_wait_result/preview_flow", post(run_wait_result_preview_flow))
         .route("/list", get(list_jobs))
         .route(
             "/list_selected_job_groups",
@@ -5207,6 +5209,28 @@ async fn run_preview_script(
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
+async fn run_wait_result_preview_script(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(run_query): Query<RunJobQuery>,
+    Json(preview): Json<Preview>,
+) -> error::Result<Response> {
+
+    let (_status_code, uuid) = run_preview_script(
+        authed.clone(), 
+        Extension(db.clone()), 
+        Extension(user_db.clone()), 
+        Path(w_id.clone()), 
+        Query(run_query.clone()), 
+        Json(preview)
+    ).await?;
+    let uuid = uuid.parse::<Uuid>().map_err(|_| Error::BadRequest("Invalid UUID".to_string()))?;
+    let result = run_wait_result(&db, uuid, w_id, None, &authed.username).await;
+    return result;
+}
+
 async fn run_bundle_preview_script(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -5865,6 +5889,20 @@ async fn run_preview_flow_job(
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
+async fn run_wait_result_preview_flow(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(run_query): Query<RunJobQuery>,
+    Json(raw_flow): Json<PreviewFlow>,
+) -> error::Result<Response> {
+    let (_status_code, uuid) = run_preview_flow_job(authed.clone(), Extension(db.clone()), Extension(user_db.clone()), Path(w_id.clone()), Query(run_query.clone()), Json(raw_flow)).await?;
+    let uuid = uuid.parse::<Uuid>().map_err(|_| Error::BadRequest("Invalid UUID".to_string()))?;
+    let result = run_wait_result(&db, uuid, w_id, None, &authed.username).await;
+    return result;
+}
+
 pub async fn run_job_by_hash(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,

backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs

@@ -564,6 +564,86 @@ pub fn all_tools() -> Vec<EndpointTool> {
 })),
         body_schema: None,
     },
+    EndpointTool {
+        name: Cow::Borrowed("runScriptPreviewAndWaitResult"),
+        description: Cow::Borrowed("run script preview and wait for result"),
+        instructions: Cow::Borrowed("Allows testing a script before deploying it. For typescript code, the language to send is either bun or deno. By default, send bun if no deno specific code is detected."),
+        path: Cow::Borrowed("/w/{workspace}/jobs/run_wait_result/preview"),
+        method: Cow::Borrowed("POST"),
+        path_params_schema: None,
+        query_params_schema: None,
+        body_schema: Some(serde_json::json!({
+        "type": "object",
+        "properties": {
+                "content": {
+                        "type": "string",
+                        "description": "The code to run"
+                },
+                "path": {
+                        "type": "string",
+                        "description": "The path to the script"
+                },
+                "script_hash": {
+                        "type": "string",
+                        "description": "The hash of the script"
+                },
+                "args": {
+                        "type": "object",
+                        "description": "The arguments to pass to the script or flow",
+                        "additionalProperties": {}
+                },
+                "language": {
+                        "type": "string",
+                        "enum": [
+                                "python3",
+                                "deno",
+                                "go",
+                                "bash",
+                                "powershell",
+                                "postgresql",
+                                "mysql",
+                                "bigquery",
+                                "snowflake",
+                                "mssql",
+                                "oracledb",
+                                "graphql",
+                                "nativets",
+                                "bun",
+                                "php",
+                                "rust",
+                                "ansible",
+                                "csharp",
+                                "nu",
+                                "java",
+                                "ruby",
+                                "duckdb"
+                        ]
+                },
+                "tag": {
+                        "type": "string"
+                },
+                "kind": {
+                        "type": "string",
+                        "enum": [
+                                "code",
+                                "identity",
+                                "http"
+                        ]
+                },
+                "dedicated_worker": {
+                        "type": "boolean"
+                },
+                "lock": {
+                        "type": "string"
+                }
+        },
+        "required": [
+                "args",
+                "content",
+                "language"
+        ]
+})),
+    },
     EndpointTool {
         name: Cow::Borrowed("listQueue"),
         description: Cow::Borrowed("list all queued jobs"),