fix(openai): adapt multipart image edits to native JSON (#457)

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-10-17 14:52:13 +08:00
committed by GitHub
co-authored by Zyy Codex
parent 0b02d0d98a
commit f4eceb076b
8 changed files with 519 additions and 22 deletions
@@ -1,6 +1,6 @@
//! OpenAI Provider 原生非流式 JSON 端点的公共交付边界
use std::net::IpAddr;
use std::{future::Future, net::IpAddr};
use axum::{
body::{Body, Bytes},
@@ -30,7 +30,7 @@ use super::{
};
/// Provider 自有端点也先冻结身份,再通过统一请求链进入原有执行与结算路径
pub(super) async fn provider_endpoint_response<F>(
pub(super) async fn provider_endpoint_response<F, Fut>(
service: OpenAiService,
client: AuthenticatedClient,
input: RequestInput,
@@ -39,7 +39,8 @@ pub(super) async fn provider_endpoint_response<F>(
decode: F,
) -> Response
where
F: FnOnce(Bytes, &HeaderMap) -> Result<Operation, GatewayError> + Send + 'static,
F: FnOnce(Bytes, HeaderMap) -> Fut + Send + 'static,
Fut: Future<Output = Result<Operation, GatewayError>> + Send,
{
let execution = service.execution();
let prepared = match execution.prepare_execution(client).await {
@@ -58,7 +59,7 @@ where
if protocol != "openai" {
return Err(MiddlewareError::Rejected);
}
let operation = request.apply_capabilities(decode(body, &headers)?)?;
let operation = request.apply_capabilities(decode(body, headers).await?)?;
let started = match service
.start_prepared_provider_endpoint(
prepared, operation,
@@ -100,12 +100,17 @@ async fn handle_image_request(
.await
}
fn image_operation(
async fn image_operation(
body: Bytes,
headers: &HeaderMap,
mut headers: HeaderMap,
kind: ImageRequestKind,
) -> Result<Operation, GatewayError> {
let mut context = OpenAiRequestHeaders::from_headers(headers).session_context();
let body = if kind == ImageRequestKind::Edit {
super::request::normalize_edit_body(body, &mut headers).await?
} else {
body
};
let mut context = OpenAiRequestHeaders::from_headers(&headers).session_context();
if let Some(turn_id) = headers
.get("x-codex-image-turn-id")
.and_then(|value| value.to_str().ok())
@@ -1,5 +1,6 @@
//! OpenAI/Codex Images JSON 透明 adapter
//! OpenAI/Codex Images JSON 透传与图片上传适配
mod http;
mod request;
pub(crate) use http::{image_edits, image_generations};
@@ -0,0 +1,131 @@
//! 将图片编辑 multipart 上传映射到原生 Images JSON,保留图片字节与字段语义
use axum::{
body::{Body, Bytes},
extract::{DefaultBodyLimit, FromRequest, Multipart, Request},
http::{
HeaderMap, HeaderValue,
header::{CONTENT_LENGTH, CONTENT_TYPE},
},
};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use serde::ser::{SerializeMap, Serializer as _};
use serde_json::{Number, Value, json};
pub(super) async fn normalize_edit_body(
body: Bytes,
headers: &mut HeaderMap,
) -> Result<Bytes, GatewayError> {
let is_multipart = headers
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.split(';').next())
.is_some_and(|media| media.trim().eq_ignore_ascii_case("multipart/form-data"));
if !is_multipart {
return Ok(body);
}
if headers.get_all(CONTENT_TYPE).iter().count() != 1 {
return Err(invalid("Image edit requires a single Content-Type header"));
}
let mut request = Request::new(Body::from(body));
*request.headers_mut() = headers.clone();
// 原路由不设正文私有上限,重新提取时不能引入 Axum 默认的 2 MiB 限制
DefaultBodyLimit::disable().apply(&mut request);
let mut multipart = Multipart::from_request(request, &())
.await
.map_err(|_| invalid("Image edit multipart boundary is invalid"))?;
let mut body = Vec::new();
let mut serializer = serde_json::Serializer::new(&mut body);
// 直接序列化字段,避免经过 Map 后覆盖同名参数
let mut fields = serializer.serialize_map(None).map_err(encode_error)?;
let mut images = Vec::new();
while let Some(field) = multipart
.next_field()
.await
.map_err(|_| invalid("Image edit multipart body is invalid"))?
{
let name = field
.name()
.ok_or_else(|| invalid("Image edit multipart fields must have names"))?
.to_owned();
let is_file = field.file_name().is_some();
let bytes = field
.bytes()
.await
.map_err(|_| invalid("Image edit multipart field is invalid"))?;
if matches!(name.as_str(), "image" | "image[]") {
// 按表单中的出现顺序追加,多图顺序参与编辑语义
images.push(image_reference(&bytes)?);
continue;
}
let value = if name == "mask" {
image_reference(&bytes)?
} else {
if is_file {
return Err(invalid(
"Only image, image[] and mask may contain image files",
));
}
scalar_field(&name, &bytes)?
};
fields
.serialize_entry(&name, &value)
.map_err(encode_error)?;
}
if images.is_empty() {
return Err(invalid(
"Image edit multipart requires an image or image[] upload",
));
}
fields
.serialize_entry("images", &images)
.map_err(encode_error)?;
fields.end().map_err(encode_error)?;
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.remove(CONTENT_LENGTH);
Ok(Bytes::from(body))
}
fn scalar_field(name: &str, bytes: &[u8]) -> Result<Value, GatewayError> {
match name {
"n" | "output_compression" | "partial_images" => serde_json::from_slice::<Number>(bytes)
.map(Value::Number)
.map_err(|_| invalid("Image edit numeric fields must contain JSON numbers")),
"stream" => serde_json::from_slice::<bool>(bytes)
.map(Value::Bool)
.map_err(|_| invalid("Image edit stream field must contain a JSON boolean")),
// 其余表单字段保留名称与文本值,不因字段尚未识别而丢弃或拒绝
_ => {
let text = std::str::from_utf8(bytes)
.map_err(|_| invalid("Image edit text fields must be valid UTF-8"))?;
Ok(Value::String(text.to_owned()))
}
}
}
fn image_reference(bytes: &[u8]) -> Result<Value, GatewayError> {
// 与官方 Original 路径一样保留 PNG/JPEG/WebP 原始字节;不依赖文件名或上传 MIME
let mime = if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
"image/png"
} else if bytes.starts_with(b"\xff\xd8\xff") {
"image/jpeg"
} else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP") {
"image/webp"
} else {
return Err(invalid("Image edit uploads must be PNG, JPEG or WebP"));
};
Ok(json!({ "image_url": format!("data:{mime};base64,{}", STANDARD.encode(bytes)) }))
}
fn invalid(message: &'static str) -> GatewayError {
GatewayError::new(GatewayErrorKind::InvalidRequest, message)
}
fn encode_error(_: serde_json::Error) -> GatewayError {
GatewayError::new(
GatewayErrorKind::Internal,
"Could not encode image edit JSON",
)
}
@@ -54,7 +54,7 @@ pub(crate) async fn standalone_search(
},
client_ip,
user_agent,
search_operation,
async move |body, headers| search_operation(body, &headers),
)
.await
}
@@ -1,5 +1,7 @@
//! 图像生成与编辑 HTTP 接口的请求传递和响应交付测试
mod request;
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
@@ -260,7 +262,7 @@ impl ExecutionSession for ImageSession {
}
#[tokio::test]
async fn image_routes_should_not_decode_bodies_and_should_preserve_both_directions() {
async fn image_json_routes_should_preserve_both_directions_without_decoding() {
let mut execution = ImageExecution::new();
let middleware = Arc::new(crate::openai::middleware::RequestMiddleware {
expected_operation: Some(OperationKind::GenerateImage),
@@ -282,12 +284,6 @@ async fn image_routes_should_not_decode_bodies_and_should_preserve_both_directio
"application/json",
br#"{"model":"gpt-image-2","images":[{"image_url":"data:image/png;base64,AAEC"}],"prompt":"add fog","prompt":"duplicate stays raw"}"#.as_slice(),
),
(
"/v1/images/edits",
ImageRequestKind::Edit,
"multipart/form-data; boundary=image-boundary",
b"--image-boundary\r\nContent-Disposition: form-data; name=\"image\"; filename=\"image.png\"\r\nContent-Type: image/png\r\n\r\n\x89PNG\x00\xff\r\n--image-boundary--\r\n".as_slice(),
),
];
for (path, _, content_type, body) in &cases {
@@ -332,11 +328,7 @@ async fn image_routes_should_not_decode_bodies_and_should_preserve_both_directio
assert_eq!(captured.len(), cases.len());
assert_eq!(
*middleware.endpoints.lock().unwrap(),
vec![
"/v1/images/generations",
"/v1/images/edits",
"/v1/images/edits"
]
vec!["/v1/images/generations", "/v1/images/edits"]
);
for (captured, (endpoint, kind, _, body)) in captured.iter().zip(cases.iter()) {
assert_eq!(captured.provider, "openai");
@@ -0,0 +1,347 @@
//! 图片上传与原生 JSON 的等价性、拒绝边界及请求中间件顺序测试
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header::AUTHORIZATION},
};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde_json::{Value, json};
use std::sync::Arc;
use tower::ServiceExt;
use super::{IMAGE_RESPONSE, ImageExecution, api_router};
const CONTENT_TYPE: &str = "multipart/form-data; boundary=image-boundary";
const PNG: &[u8] = b"\x89PNG\r\n\x1a\n\x00\xfforiginal png bytes\r\n";
const JPEG: &[u8] = b"\xff\xd8\xff\x00original jpeg bytes\xff\xd9";
const WEBP: &[u8] = b"RIFF\x10\x00\x00\x00WEBPoriginal webp bytes";
fn multipart_body(parts: &[(&str, Option<&str>, &[u8])]) -> Vec<u8> {
let mut body = Vec::new();
for (name, filename, bytes) in parts {
body.extend_from_slice(
format!("--image-boundary\r\nContent-Disposition: form-data; name=\"{name}\"")
.as_bytes(),
);
if let Some(filename) = filename {
body.extend_from_slice(
format!("; filename=\"{filename}\"\r\nContent-Type: application/octet-stream")
.as_bytes(),
);
}
body.extend_from_slice(b"\r\n\r\n");
body.extend_from_slice(bytes);
body.extend_from_slice(b"\r\n");
}
body.extend_from_slice(b"--image-boundary--\r\n");
body
}
#[tokio::test]
async fn multipart_edit_should_match_native_json_fields_and_image_bytes() {
let execution = ImageExecution::new();
let router = api_router(execution.clone()).await;
let body = multipart_body(&[
("image", Some("first.bin"), PNG),
("prompt", None, " 保留空格\r\n\"和引号\" ".as_bytes()),
("image[]", Some("second.bin"), JPEG),
("mask", Some("mask.bin"), PNG),
("image[]", Some("third.bin"), WEBP),
("model", None, b"gpt-image-2"),
("background", None, b"transparent"),
("quality", None, b"high"),
("size", None, b"1024x1024"),
("input_fidelity", None, b"high"),
("moderation", None, b"auto"),
("output_format", None, b"png"),
("response_format", None, b"b64_json"),
("user", None, b"000123"),
("safety_identifier", None, b"false"),
("n", None, b"2"),
("output_compression", None, b"80"),
("partial_images", None, b"0"),
("stream", None, b"false"),
]);
let native_json = json!({
"images": [
{"image_url": format!("data:image/png;base64,{}", STANDARD.encode(PNG))},
{"image_url": format!("data:image/jpeg;base64,{}", STANDARD.encode(JPEG))},
{"image_url": format!("data:image/webp;base64,{}", STANDARD.encode(WEBP))},
],
"mask": {"image_url": format!("data:image/png;base64,{}", STANDARD.encode(PNG))},
"prompt": " 保留空格\r\n\"和引号\" ",
"model": "gpt-image-2",
"background": "transparent",
"quality": "high",
"size": "1024x1024",
"input_fidelity": "high",
"moderation": "auto",
"output_format": "png",
"response_format": "b64_json",
"user": "000123",
"safety_identifier": "false",
"n": 2,
"output_compression": 80,
"partial_images": 0,
"stream": false,
});
for (content_type, body) in [
(CONTENT_TYPE, body),
(
"application/json",
serde_json::to_vec(&native_json).unwrap(),
),
] {
let response = router
.clone()
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", content_type)
.header("content-length", body.len())
.header("x-codex-image-turn-id", "turn-multipart-edit")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
to_bytes(response.into_body(), usize::MAX)
.await
.unwrap()
.as_ref(),
IMAGE_RESPONSE
);
}
let captured = execution.captured();
assert_eq!(captured.len(), 2);
assert_eq!(captured[0].context, captured[1].context);
for request in &captured {
let json: Value = serde_json::from_slice(&request.body).unwrap();
assert_eq!(json, native_json);
assert_eq!(request.context["image_turn_id"], "turn-multipart-edit");
for (reference, original) in json["images"]
.as_array()
.unwrap()
.iter()
.zip([PNG, JPEG, WEBP])
{
let (_, encoded) = reference["image_url"]
.as_str()
.unwrap()
.split_once(',')
.unwrap();
assert_eq!(STANDARD.decode(encoded).unwrap(), original);
}
}
}
#[tokio::test]
async fn multipart_edit_should_preserve_omitted_parameters_and_exact_numbers() {
let execution = ImageExecution::new();
let response = api_router(execution.clone())
.await
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", CONTENT_TYPE)
.body(Body::from(multipart_body(&[
("image", Some("image.png"), PNG),
("prompt", None, b"false"),
// 参数值的可接受范围仍由上游决定,适配不能经浮点数中转损失精度
("n", None, b"9007199254740993"),
])))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let captured = execution.captured();
let body: Value = serde_json::from_slice(&captured[0].body).unwrap();
assert_eq!(
body,
json!({
"images": [{"image_url": format!("data:image/png;base64,{}", STANDARD.encode(PNG))}],
"prompt": "false",
"n": 9_007_199_254_740_993_u64,
})
);
}
#[tokio::test]
async fn multipart_edit_should_forward_unknown_fields_and_repeated_keys_in_order() {
let execution = ImageExecution::new();
let router = api_router(execution.clone()).await;
let body = multipart_body(&[
("future_field", None, b"123"),
("prompt", None, b"first"),
("image", Some("image.png"), PNG),
("future_field", None, b"false"),
("prompt", None, b"second"),
("n", None, b"1"),
("n", None, b"2"),
("mask", Some("first.png"), PNG),
("mask", Some("second.webp"), WEBP),
]);
let native_json = format!(
concat!(
r#"{{"future_field":"123","prompt":"first","future_field":"false","prompt":"second","n":1,"n":2,"#,
r#""mask":{{"image_url":"data:image/png;base64,{}"}},"mask":{{"image_url":"data:image/webp;base64,{}"}},"#,
r#""images":[{{"image_url":"data:image/png;base64,{}"}}]}}"#,
),
STANDARD.encode(PNG),
STANDARD.encode(WEBP),
STANDARD.encode(PNG),
);
for (content_type, body) in [
(CONTENT_TYPE, body),
("application/json", native_json.as_bytes().to_vec()),
] {
let response = router
.clone()
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", content_type)
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
}
let captured = execution.captured();
assert_eq!(captured.len(), 2);
// Value 会折叠重复键,这里直接比较原生 JSON 字节,确认每次出现都保留
assert_eq!(captured[0].body, captured[1].body);
assert_eq!(captured[0].body.as_ref(), native_json.as_bytes());
}
#[tokio::test]
async fn multipart_edit_should_reject_fields_that_cannot_be_mapped_without_loss() {
let cases = [
(
"invalid_number",
vec![("n", None, b"not-a-number".as_slice())],
),
("invalid_boolean", vec![("stream", None, b"yes".as_slice())]),
("invalid_utf8", vec![("prompt", None, b"\xff".as_slice())]),
(
"file_in_scalar",
vec![("prompt", Some("prompt.txt"), b"text".as_slice())],
),
(
"unsupported_image",
vec![("image", Some("image.gif"), b"GIF89a".as_slice())],
),
(
"unsupported_mask",
vec![("mask", Some("mask.png"), b"not an image".as_slice())],
),
];
for (name, mut parts) in cases {
parts.push(("image", Some("image.png"), PNG));
let execution = ImageExecution::new();
let response = api_router(execution.clone())
.await
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", CONTENT_TYPE)
.body(Body::from(multipart_body(&parts)))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{name}");
let error: Value =
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
.unwrap();
assert_eq!(error["error"]["code"], "invalid_request", "{name}");
assert!(execution.captured().is_empty(), "{name}");
}
}
#[tokio::test]
async fn multipart_edit_should_reject_invalid_envelopes_before_provider_execution() {
for (content_type, body) in [
(
"multipart/form-data",
multipart_body(&[("image", Some("image.png"), PNG)]),
),
(CONTENT_TYPE, b"--image-boundary\r\ntruncated".to_vec()),
(CONTENT_TYPE, multipart_body(&[("prompt", None, b"edit")])),
] {
let execution = ImageExecution::new();
let response = api_router(execution.clone())
.await
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", content_type)
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert!(execution.captured().is_empty());
}
}
#[tokio::test]
async fn multipart_edit_should_not_inherit_the_default_extractor_size_limit() {
let mut image = PNG.to_vec();
image.resize(2 * 1024 * 1024 + 1, 0xff);
let execution = ImageExecution::new();
let response = api_router(execution.clone())
.await
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", CONTENT_TYPE)
.body(Body::from(multipart_body(&[(
"image",
Some("image.png"),
&image,
)])))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let captured = execution.captured();
let body: Value = serde_json::from_slice(&captured[0].body).unwrap();
let (_, encoded) = body["images"][0]["image_url"]
.as_str()
.unwrap()
.split_once(',')
.unwrap();
assert_eq!(STANDARD.decode(encoded).unwrap(), image);
}
#[tokio::test]
async fn request_middleware_should_run_before_multipart_decoding() {
let mut execution = ImageExecution::new();
let middleware = Arc::new(crate::openai::middleware::RequestMiddleware {
expected_operation: Some(gateway_core::operation::OperationKind::GenerateImage),
short_circuit_json: Some(bytes::Bytes::from_static(IMAGE_RESPONSE)),
..Default::default()
});
Arc::get_mut(&mut execution).unwrap().middleware = Some(middleware.clone());
let response = api_router(execution.clone())
.await
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", CONTENT_TYPE)
.body(Body::from("invalid multipart handled by middleware"))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(*middleware.endpoints.lock().unwrap(), ["/v1/images/edits"]);
assert!(execution.captured().is_empty());
}
+21 -1
View File
@@ -197,13 +197,33 @@ WebSocket message 和 frame 不设置网关私有长度上限;协议可接受
| `GET` | `/v1/responses` | 通过 HTTP Upgrade 建立 Responses WebSocket |
| `POST` | `/v1/alpha/search` | Codex standalone web search;JSON 请求与响应正文原样转发 |
| `POST` | `/v1/images/generations` | 通过 OpenAI Provider 发起图像生成;JSON 请求与响应正文原样转发 |
| `POST` | `/v1/images/edits` | 通过 OpenAI Provider 发起图像编辑;JSON 请求与响应正文原样转发 |
| `POST` | `/v1/images/edits` | 通过 OpenAI Provider 发起图像编辑;JSON 原样转发,multipart 上传转为原生 Images JSON |
| `GET` | `/v1/models` | 返回当前 Client Key 账号范围内各 Provider 的可用公开模型并集;有两种响应形态,见下 |
| `GET` | `/v1/models/{model_id}` | 返回 OpenAI 兼容的单模型详情 |
| `POST` | `/v1/providers/{provider}/models/{model}/count_tokens` | 调用指定 Provider/model 声明的精确 Token 计数;请求和响应均为 JSON |
| `GET` / `POST` | `/v1/providers/{provider}/http/{endpoint}` | 调用指定 Provider 预先声明的固定 HTTP 操作;不是任意 URL 代理 |
| `GET` | `/v1/usage` | 查询当前 Client Key 的日与周额度,仅使用网关已结算的 USD 账本 |
### Images 请求
Images 使用原生非流式 JSON 端点,JSON 请求与响应正文原样转发。
`/v1/images/edits` 还接受 `multipart/form-data`,将上传转换为原生 Images JSON 请求
multipart 的 `image` 和 `image[]` 上传按出现顺序组成 `images` 数组,每项为
`{"image_url":"data:image/...;base64,..."}`;`mask` 上传转换为同结构的单个对象。
图片支持 PNG、JPEG、WebP,根据内容识别格式,原始文件字节直接编码,不缩放或重新编码
| 表单字段 | JSON 类型 |
| --- | --- |
| `n`、`output_compression`、`partial_images` | JSON 数字,不经浮点数中转 |
| `stream` | JSON 布尔值 `true` 或 `false`;转换不增加流式响应支持 |
| 其余文本字段,包括未知字段 | 字符串,保留字段名、空白与原值 |
未提供的参数保持缺省;除 `image` / `image[]` 聚合外,其余同名字段按出现顺序保留为重复 JSON 键,
解释与校验交给上游。
缺少图片、损坏的 multipart、无效的 UTF-8 或字段类型返回 `400`。
需要为扩展字段指定其他 JSON 类型,或传入 `file_id` 等图片引用时,使用 JSON 请求
### Provider 自有操作
这两类 `/v1/providers/*` 路由使用同一 Client Key 鉴权、账号组范围、Provider 账号资格、租约、