From 7b29159a43b3c7f4557dc77e3d1eb2adcd0ee243 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 5 Jan 2026 11:56:27 +0100 Subject: [PATCH] fix param for openai (#7483) --- .../src/ai/providers/openai.rs | 6 +- integration_tests/ai_agent_tests/conftest.py | 12 + .../ai_agent_tests/test_completion_params.py | 227 ++++++++++++++++++ 3 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 integration_tests/ai_agent_tests/test_completion_params.py diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index c575dba153..ae31cc6c83 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -217,7 +217,7 @@ pub struct ResponsesApiRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub max_completion_tokens: Option, + pub max_output_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub text: Option, } @@ -395,7 +395,7 @@ impl OpenAIQueryBuilder { tools, stream: Some(true), temperature: args.temperature, - max_completion_tokens: args.max_tokens, + max_output_tokens: args.max_tokens, text, }; @@ -440,7 +440,7 @@ impl OpenAIQueryBuilder { tools, stream: None, // Image generation doesn't use streaming temperature: args.temperature, - max_completion_tokens: args.max_tokens, + max_output_tokens: args.max_tokens, text: None, // No structured output for image generation }; diff --git a/integration_tests/ai_agent_tests/conftest.py b/integration_tests/ai_agent_tests/conftest.py index b219480fad..9c1e65f71a 100644 --- a/integration_tests/ai_agent_tests/conftest.py +++ b/integration_tests/ai_agent_tests/conftest.py @@ -219,6 +219,8 @@ def create_ai_agent_flow( streaming: bool | None = None, include_user_images: bool = False, output_type: str | None = None, + temperature: float | None = None, + max_completion_tokens: int | None = None, ) -> dict[str, Any]: """ Create a FlowValue for an AI agent. @@ -231,6 +233,8 @@ def create_ai_agent_flow( streaming: Optional flag to enable streaming responses include_user_images: If True, adds user_images input from flow_input output_type: Optional output type ("text" or "image") + temperature: Optional temperature for sampling (0.0-2.0) + max_completion_tokens: Optional maximum tokens for completion Returns: A FlowValue dictionary ready to be sent to preview_flow @@ -257,6 +261,14 @@ def create_ai_agent_flow( if output_type is not None: input_transforms["output_type"] = {"type": "static", "value": output_type} + # Add temperature if provided + if temperature is not None: + input_transforms["temperature"] = {"type": "static", "value": temperature} + + # Add max_completion_tokens if provided + if max_completion_tokens is not None: + input_transforms["max_completion_tokens"] = {"type": "static", "value": max_completion_tokens} + module_value = { "type": "aiagent", "input_transforms": input_transforms, diff --git a/integration_tests/ai_agent_tests/test_completion_params.py b/integration_tests/ai_agent_tests/test_completion_params.py new file mode 100644 index 0000000000..c937c540a6 --- /dev/null +++ b/integration_tests/ai_agent_tests/test_completion_params.py @@ -0,0 +1,227 @@ +""" +Completion parameter tests for AI agents. + +Tests that AI agents correctly handle temperature and max_completion_tokens: +- Default parameters (undefined) +- Low temperature (0.0 - deterministic) +- High temperature (0.9 - more random) +- Low max_completion_tokens (10 - short response) +- High max_completion_tokens (4096 - longer response allowed) +- Combined parameters +""" + +import pytest + +from .conftest import AIAgentTestClient, create_ai_agent_flow +from .providers import ALL_PROVIDERS, get_provider_ids + + +class TestCompletionParams: + """Test AI agent temperature and max_completion_tokens parameters.""" + + @pytest.mark.parametrize( + "provider_config", + ALL_PROVIDERS, + ids=get_provider_ids(ALL_PROVIDERS), + ) + def test_default_params( + self, + client: AIAgentTestClient, + setup_providers, + provider_config, + ): + """ + Test with default parameters (no temperature or max_completion_tokens). + This serves as a baseline to ensure the agent works without these params. + """ + flow_value = create_ai_agent_flow( + provider_input_transform=provider_config["input_transform"], + system_prompt="You are a helpful assistant. Be concise.", + output_type="text", + ) + + result = client.run_preview_flow( + flow_value=flow_value, + args={"user_message": "What is 2 + 2? Answer with just the number."}, + ) + + assert result is not None + # Result should contain the answer + result_str = str(result) + assert "4" in result_str, f"Expected '4' in result: {result}" + + print(f"Default params result from {provider_config['name']}: {result}") + + @pytest.mark.parametrize( + "provider_config", + ALL_PROVIDERS, + ids=get_provider_ids(ALL_PROVIDERS), + ) + def test_low_temperature( + self, + client: AIAgentTestClient, + setup_providers, + provider_config, + ): + """ + Test with temperature=0.0 (deterministic output). + Low temperature should produce more focused, consistent responses. + """ + flow_value = create_ai_agent_flow( + provider_input_transform=provider_config["input_transform"], + system_prompt="You are a helpful assistant. Be concise.", + output_type="text", + temperature=0.0, + ) + + result = client.run_preview_flow( + flow_value=flow_value, + args={"user_message": "What is 2 + 2? Answer with just the number."}, + ) + + assert result is not None + result_str = str(result) + assert "4" in result_str, f"Expected '4' in result: {result}" + + print(f"Low temperature (0.0) result from {provider_config['name']}: {result}") + + @pytest.mark.parametrize( + "provider_config", + ALL_PROVIDERS, + ids=get_provider_ids(ALL_PROVIDERS), + ) + def test_high_temperature( + self, + client: AIAgentTestClient, + setup_providers, + provider_config, + ): + """ + Test with temperature=0.9 (more random output). + High temperature should still produce valid responses. + """ + flow_value = create_ai_agent_flow( + provider_input_transform=provider_config["input_transform"], + system_prompt="You are a helpful assistant. Be concise.", + output_type="text", + temperature=0.9, + ) + + result = client.run_preview_flow( + flow_value=flow_value, + args={"user_message": "What is 2 + 2? Answer with just the number."}, + ) + + assert result is not None + # With high temperature, the model might be more creative but should still respond + result_str = str(result) + # We just verify we got a non-empty response + assert len(result_str) > 0, f"Expected non-empty result: {result}" + + print(f"High temperature (0.9) result from {provider_config['name']}: {result}") + + @pytest.mark.parametrize( + "provider_config", + ALL_PROVIDERS, + ids=get_provider_ids(ALL_PROVIDERS), + ) + def test_low_max_tokens( + self, + client: AIAgentTestClient, + setup_providers, + provider_config, + ): + """ + Test with max_completion_tokens=10 (short response). + The response should be truncated or very short. + """ + flow_value = create_ai_agent_flow( + provider_input_transform=provider_config["input_transform"], + system_prompt="You are a helpful assistant.", + output_type="text", + max_completion_tokens=10, + ) + + result = client.run_preview_flow( + flow_value=flow_value, + args={"user_message": "Explain the theory of relativity in detail."}, + ) + + assert result is not None + # The response should be truncated due to low max_tokens + # We verify we got some response (even if truncated) + result_str = str(result) + assert len(result_str) > 0, f"Expected non-empty result: {result}" + + print(f"Low max_tokens (10) result from {provider_config['name']}: {result}") + + @pytest.mark.parametrize( + "provider_config", + ALL_PROVIDERS, + ids=get_provider_ids(ALL_PROVIDERS), + ) + def test_high_max_tokens( + self, + client: AIAgentTestClient, + setup_providers, + provider_config, + ): + """ + Test with max_completion_tokens=4096 (longer response allowed). + The model should be able to produce longer responses if needed. + """ + flow_value = create_ai_agent_flow( + provider_input_transform=provider_config["input_transform"], + system_prompt="You are a helpful assistant. Be concise.", + output_type="text", + max_completion_tokens=4096, + ) + + result = client.run_preview_flow( + flow_value=flow_value, + args={"user_message": "What is 2 + 2? Answer with just the number."}, + ) + + assert result is not None + result_str = str(result) + assert "4" in result_str, f"Expected '4' in result: {result}" + + print(f"High max_tokens (4096) result from {provider_config['name']}: {result}") + + @pytest.mark.parametrize( + "provider_config", + ALL_PROVIDERS, + ids=get_provider_ids(ALL_PROVIDERS), + ) + def test_combined_params( + self, + client: AIAgentTestClient, + setup_providers, + provider_config, + ): + """ + Test with both temperature and max_completion_tokens set. + Verifies that both parameters work together correctly. + """ + flow_value = create_ai_agent_flow( + provider_input_transform=provider_config["input_transform"], + system_prompt="You are a helpful assistant. Be concise.", + output_type="text", + temperature=0.5, + max_completion_tokens=100, + ) + + result = client.run_preview_flow( + flow_value=flow_value, + args={"user_message": "What is 2 + 2? Answer with just the number."}, + ) + + assert result is not None + result_str = str(result) + assert "4" in result_str, f"Expected '4' in result: {result}" + + print(f"Combined params (temp=0.5, max_tokens=100) result from {provider_config['name']}: {result}") + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"])