greptimedb/tests-integration/tests/http.rs

// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::BTreeMap;
use std::io::Write;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;

use api::greptime_proto::io::prometheus::write::v2::histogram::{Count, ZeroCount};
use api::greptime_proto::io::prometheus::write::v2::metadata::MetricType as RemoteWriteV2MetricType;
use api::greptime_proto::io::prometheus::write::v2::{
    BucketSpan, Histogram, Metadata as RemoteWriteV2Metadata, Request as RemoteWriteV2Request,
    Sample as RemoteWriteV2Sample, TimeSeries as RemoteWriteV2TimeSeries,
};
use api::prom_store::remote::label_matcher::Type as MatcherType;
use api::prom_store::remote::{
    Label, LabelMatcher, Query, ReadRequest, ReadResponse, Sample, TimeSeries, WriteRequest,
};
use api::v1::RowInsertRequests;
use auth::{UserProviderRef, user_provider_from_option};
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use base64::prelude::{BASE64_STANDARD, Engine as _};
use chrono::Utc;
use cmd::options::GreptimeOptions;
use common_base::Plugins;
use common_catalog::consts::{
    DEFAULT_PRIVATE_SCHEMA_NAME, TRACE_TABLE_NAME, trace_operations_table_name,
    trace_services_table_name,
};
use common_config::Configurable;
use common_error::status_code::StatusCode as ErrorCode;
use common_frontend::slow_query_event::{
    SLOW_QUERY_TABLE_NAME, SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
};
use common_memory_manager::OnExhaustedPolicy;
use common_options::plugin_options::StandaloneFlag;
use flate2::Compression;
use flate2::write::GzEncoder;
use log_query::{AggFunc, Context, Limit, LogExpr, LogQuery, TimeFilter};
use loki_proto::logproto::{EntryAdapter, LabelPairAdapter, PushRequest, StreamAdapter};
use loki_proto::prost_types::Timestamp;
use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
use opentelemetry_proto::tonic::collector::metrics::v1::{
    ExportMetricsServiceRequest, ExportMetricsServiceResponse,
};
use opentelemetry_proto::tonic::collector::trace::v1::{
    ExportTraceServiceRequest, ExportTraceServiceResponse,
};
use pipeline::{
    GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME, GREPTIME_INTERNAL_TRACE_PIPELINE_V2_NAME,
};
use prost::Message;
use rstest_reuse::apply;
use serde_json::{Value, json};
use servers::http::handler::HealthResponse;
use servers::http::header::constants::{
    GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME, GREPTIME_LOG_TABLE_NAME_HEADER_NAME,
    GREPTIME_PIPELINE_NAME_HEADER_NAME,
};
use servers::http::header::{GREPTIME_DB_HEADER_NAME, GREPTIME_TIMEZONE_HEADER_NAME};
use servers::http::otlp::GoogleRpcStatus;
use servers::http::prometheus::{
    Column, PromQueryResult, PrometheusJsonResponse, PrometheusResponse,
};
use servers::http::result::error_result::ErrorResponse;
use servers::http::result::greptime_result_v1::GreptimedbV1Response;
use servers::http::result::influxdb_result_v1::{InfluxdbOutput, InfluxdbV1Response};
use servers::http::test_helpers::{TestClient, TestResponse};
use servers::http::{GreptimeQueryOutput, HttpOptions, HttpServerBuilder};
use servers::prom_remote_write::v2::test_util as remote_write_v2;
use servers::prom_remote_write::validation::PromValidationMode;
use servers::prom_store::{self, mock_timeseries_new_label};
use servers::request_memory_limiter::ServerMemoryLimiter;
use session::context::QueryContextRef;
use standalone::options::StandaloneOptions;
use table::table_name::TableName;
use tests_integration::test_util::{
    MockInstanceImpl, StorageType, TestGuard, assert_wal_delta, build_test_prom_server,
    setup_test_http_app, setup_test_http_app_with_frontend,
    setup_test_http_app_with_frontend_and_custom_options,
    setup_test_http_app_with_frontend_and_slow_query_threshold,
    setup_test_http_app_with_frontend_and_user_provider, setup_test_prom_app_with_frontend,
    setup_test_prom_app_with_frontend_and_memory_limiter,
    setup_test_prom_app_with_frontend_batched,
};
use urlencoding::encode;
use yaml_rust::YamlLoader;

use crate::both_deployment_cases;
use crate::event_recorder_test_util::assert_procedure_actor_by_table;

/// Registers tests that exercise nothing below the HTTP layer, so running them
/// against a remote object store only pays bucket setup and teardown.
#[macro_export]
macro_rules! http_local_tests {
    ($($(#[$meta:meta])* $test:ident),*,) => {
        mod integration_http_local_test {
            $(
                #[tokio::test(flavor = "multi_thread")]
                $(
                    #[$meta]
                )*
                async fn $test() {
                    let store_type = tests_integration::test_util::StorageType::File;
                    // Support both unit tests and fallible tests without discarding errors.
                    let result = $crate::http::$test(store_type).await;
                    assert_eq!(
                        std::process::Termination::report(result),
                        std::process::ExitCode::SUCCESS,
                    );
                }
            )*
        }
    };
}

#[macro_export]
macro_rules! http_test {
    ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
        paste::item! {
            mod [<integration_http_ $service:lower _test>] {
                $(
                    #[tokio::test(flavor = "multi_thread")]
                    $(
                        #[$meta]
                    )*
                    async fn [< $test >]() {
                        let store_type = tests_integration::test_util::StorageType::$service;
                        if store_type.test_on() {
                            // Support both unit tests and fallible tests without discarding errors.
                            let result = $crate::http::$test(store_type).await;
                            assert_eq!(
                                std::process::Termination::report(result),
                                std::process::ExitCode::SUCCESS,
                            );
                        }
                    }
                )*
            }
        }
    };
}

#[macro_export]
macro_rules! http_tests {
     ($($service:ident),*) => {
        $(
            http_test!(
                $service,

                test_http_auth,
                test_sql_api,
                test_http_analyze_stream_tql,
                test_http_sql_slow_query,
                test_prometheus_promql_api,
                test_promql_over_non_millisecond_physical_tables,
                test_prometheus_label_replace_response,
                test_prom_http_api,
                test_prom_remote_compression_bomb_rejected,
                test_prom_remote_write_decoded_body_charged,
                test_influxdb_write_decoded_body_charged,
                test_config_api,
                test_dashboard_api,
                test_prometheus_remote_write,
                test_prometheus_remote_write_v2,
                test_prometheus_remote_write_v2_native_histogram,
                test_prometheus_remote_write_batched,
                test_prometheus_remote_write_batched_mixed_time_index_units,
                test_prometheus_remote_write_batched_interceptor_time_index_units,
                test_prometheus_remote_write_v2_batched_interceptor_time_index_units,
                test_prometheus_remote_write_batched_microsecond_physical_table,
                test_prometheus_remote_write_batched_conflicting_physical_selections,
                test_prometheus_remote_special_labels,
                test_prometheus_remote_schema_labels,
                test_prometheus_remote_write_with_pipeline,
                test_vm_proto_remote_write,

                test_pipeline_api,
                test_pipeline_name_in_header,
                test_plain_text_ingestion,
                test_pipeline_auto_transform,
                test_pipeline_auto_transform_with_select,
                test_identity_pipeline,
                test_identity_pipeline_with_null_column,
                test_identity_pipeline_with_flatten,
                test_identity_pipeline_with_custom_ts,
                test_pipeline_dispatcher,
                test_pipeline_suffix_template,
                test_pipeline_context,
                test_pipeline_with_vrl,
                test_pipeline_with_hint_vrl,
                test_pipeline_one_to_many_vrl,
                test_pipeline_2,
                test_pipeline_skip_error,
                test_pipeline_filter,
                test_pipeline_create_table,
                test_pipeline_index_options,

                test_otlp_metrics_new,
                test_otlp_exponential_histogram,
                test_otlp_metric_translation_strategies,
                test_otlp_metrics_resource_info_conflicts,
                test_otlp_traces_v0,
                test_otlp_traces_v1,
                test_otlp_traces_v2,
                test_otlp_traces_v1_entity_graph,
                test_otlp_logs,
                test_loki_pb_logs,
                test_loki_pb_logs_with_pipeline,
                test_loki_json_logs,
                test_loki_json_logs_with_pipeline,
                test_elasticsearch_logs,
                test_elasticsearch_logs_with_index,
                test_splunk_logs,
                test_splunk_raw,
                test_log_query,
                test_jaeger_query_api,
                test_jaeger_query_api_for_trace_v1,
                test_jaeger_query_api_for_trace_v2,

                test_influxdb_write,
                test_influxdb_write_with_hints,
                test_influxdb_write_with_append_mode_hint,
                test_http_memory_limit,
            );
        )*
    };
}

pub async fn test_http_auth(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();

    let user_provider = user_provider_from_option(
        "static_user_provider:cmd:greptime_user=greptime_pwd,readonly_user:ro=readonly_pwd,writeonly_user:wo=writeonly_pwd",
    )
    .unwrap();

    let (app, mut guard) = setup_test_http_app_with_frontend_and_user_provider(
        store_type,
        "sql_api",
        Some(user_provider),
    )
    .await;
    let client = TestClient::new(app).await;

    // 1. no auth
    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::UNAUTHORIZED);

    // 2. wrong auth
    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .header("Authorization", basic_auth("greptime_user", "wrong_pwd"))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::UNAUTHORIZED);

    // 3. right auth
    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .header("Authorization", basic_auth("greptime_user", "greptime_pwd"))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 4. readonly user cannot write
    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .header("Authorization", basic_auth("readonly_user", "readonly_pwd"))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?db=public&sql=create table auth_test(ts timestamp time index);")
        .header("Authorization", basic_auth("readonly_user", "readonly_pwd"))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::FORBIDDEN);

    // 5. writeonly user cannot read
    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .header(
            "Authorization",
            basic_auth("writeonly_user", "writeonly_pwd"),
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::FORBIDDEN);
    let res = client
        .get("/v1/sql?db=public&sql=create table auth_test(ts timestamp time index);")
        .header(
            "Authorization",
            basic_auth("writeonly_user", "writeonly_pwd"),
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let actor_query_client = &client;
    assert_procedure_actor_by_table(
        "create_table",
        "auth_test",
        "writeonly_user",
        |query| async move {
            let res = actor_query_client
                .get(&format!("/v1/sql?db=public&sql={}", encode(&query)))
                .header("Authorization", basic_auth("greptime_user", "greptime_pwd"))
                .send()
                .await;
            let Ok(body) = serde_json::from_str::<GreptimedbV1Response>(&res.text().await) else {
                return false;
            };
            matches!(
                body.output().first(),
                Some(GreptimeQueryOutput::Records(records))
                    if records.rows().first().and_then(|row| row.first()).and_then(Value::as_bool)
                        == Some(true)
            )
        },
    )
    .await;
    let res = client
        .get("/v1/sql?db=public&sql=insert into auth_test values(1);")
        .header(
            "Authorization",
            basic_auth("writeonly_user", "writeonly_pwd"),
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?db=public&sql=select * from auth_test;")
        .header(
            "Authorization",
            basic_auth("writeonly_user", "writeonly_pwd"),
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::FORBIDDEN);

    guard.remove_all().await;
}

fn basic_auth(username: &str, password: &str) -> String {
    format!(
        "basic {}",
        BASE64_STANDARD.encode(format!("{username}:{password}"))
    )
}

fn assert_remote_write_v2_written_headers(headers: &HeaderMap, samples: &str) {
    assert_remote_write_v2_written_headers_with_histograms(headers, samples, "0")
}

fn assert_remote_write_v2_written_headers_with_histograms(
    headers: &HeaderMap,
    samples: &str,
    histograms: &str,
) {
    assert_eq!(
        Some(samples),
        headers
            .get("X-Prometheus-Remote-Write-Samples-Written")
            .map(|x| x.to_str().unwrap())
    );
    assert_eq!(
        Some(histograms),
        headers
            .get("X-Prometheus-Remote-Write-Histograms-Written")
            .map(|x| x.to_str().unwrap())
    );
    assert_eq!(
        Some("0"),
        headers
            .get("X-Prometheus-Remote-Write-Exemplars-Written")
            .map(|x| x.to_str().unwrap())
    );
}

pub async fn test_http_auth_from_standalone_user_provider_config() {
    common_telemetry::init_default_ut_logging();

    let config = tempfile::NamedTempFile::new().unwrap();
    let user_provider = "static_user_provider:cmd:greptime_user=greptime_pwd";
    std::fs::write(
        config.path(),
        format!("user_provider = \"{user_provider}\"\n"),
    )
    .unwrap();

    let options =
        GreptimeOptions::<StandaloneOptions>::load_layered_options(config.path().to_str(), "")
            .unwrap();
    let fe_opts = options.component.frontend_options();

    let mut plugins = Plugins::new();
    plugins.insert(StandaloneFlag);
    plugins::setup_frontend_plugins_pre_build(&mut plugins, &[], &fe_opts, None)
        .await
        .unwrap();
    let user_provider = plugins.get::<UserProviderRef>();

    let (app, mut guard) = setup_test_http_app_with_frontend_and_user_provider(
        StorageType::File,
        "sql_api_user_provider_config",
        user_provider,
    )
    .await;
    let client = TestClient::new(app).await;

    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .send()
        .await;
    let status = res.status();
    let body = res.text().await;
    assert_eq!(
        status,
        StatusCode::UNAUTHORIZED,
        "unexpected response body: {body}"
    );

    let res = client
        .get("/v1/sql?db=public&sql=show tables;")
        .header("Authorization", basic_auth("greptime_user", "greptime_pwd"))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    guard.remove_all().await;
}

#[tokio::test]
pub async fn test_cors() {
    let (app, mut guard) = setup_test_http_app_with_frontend(StorageType::File, "test_cors").await;
    let client = TestClient::new(app).await;

    let res = client.get("/health").send().await;

    assert_eq!(res.status(), StatusCode::OK);
    assert_eq!(
        res.headers()
            .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
            .expect("expect cors header origin"),
        "*"
    );

    let res = client
        .options("/health")
        .header("Access-Control-Request-Headers", "x-greptime-auth")
        .header("Access-Control-Request-Method", "DELETE")
        .header("Origin", "https://example.com")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    assert_eq!(
        res.headers()
            .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
            .expect("expect cors header origin"),
        "*"
    );
    assert_eq!(
        res.headers()
            .get(http::header::ACCESS_CONTROL_ALLOW_HEADERS)
            .expect("expect cors header headers"),
        "*"
    );
    assert_eq!(
        res.headers()
            .get(http::header::ACCESS_CONTROL_ALLOW_METHODS)
            .expect("expect cors header methods"),
        "GET,POST,PUT,DELETE,HEAD"
    );

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_sql_skip_wal(distributed: bool) {
    let mut cases = vec![HttpWalCase::new(
        "SQL",
        "/v1/sql",
        "application/x-www-form-urlencoded",
        "sql=INSERT+INTO+wal_sql+VALUES+(1%2C-1)%2C(2%2C1)",
    )];
    let mut sql_get = HttpWalCase::new(
        "SQL GET",
        "/v1/sql?sql=INSERT%20INTO%20wal_sql%20VALUES%20(1%2C-1)%2C(2%2C1)",
        "application/x-www-form-urlencoded",
        Vec::new(),
    );
    sql_get.get = true;
    cases.push(sql_get);
    for case in &mut cases {
        case.create_table = Some(if distributed {
            "CREATE TABLE IF NOT EXISTS wal_sql (ts TIMESTAMP TIME INDEX, v INT) PARTITION ON COLUMNS (v) (v < 0, v >= 0)"
        } else {
            "CREATE TABLE IF NOT EXISTS wal_sql (ts TIMESTAMP TIME INDEX, v INT)"
        });
        case.expected_written_nodes = distributed.then_some(2);
    }
    check_http_skip_wal("sql", &cases, distributed).await;
}

pub async fn test_sql_api(store_type: StorageType) {
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "sql_api").await;
    let client = TestClient::new(app).await;
    let res = client.get("/v1/sql").send().await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    let body = serde_json::from_str::<ErrorResponse>(&res.text().await).unwrap();
    assert_eq!(body.code(), 1004);
    assert_eq!(body.error(), "sql parameter is required.");

    let res = client
        .get("/v1/sql?sql=select * from numbers limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let output = body.output();
    assert_eq!(output.len(), 1);
    assert_eq!(
        output[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records" :{"schema":{"column_schemas":[{"name":"number","data_type":"UInt32"}]},"rows":[[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]],"total_rows":10}
        })).unwrap()
    );

    // test influxdb_v1 result format
    let res = client
        .get("/v1/sql?format=influxdb_v1&sql=select * from numbers limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<InfluxdbV1Response>(&res.text().await).unwrap();
    let output = body.results();
    assert_eq!(output.len(), 1);
    assert_eq!(
        output[0],
        serde_json::from_value::<InfluxdbOutput>(json!({
            "statement_id":0,"series":[{"name":"","columns":["number"],"values":[[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]]}]
        })).unwrap()
    );

    // test json result format
    let res = client
        .get("/v1/sql?format=json&sql=select * from numbers limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = res.json::<Value>().await;
    let data = body.get("data").expect("Missing 'data' field in response");

    let expected = json!([
        {"number": 0}, {"number": 1}, {"number": 2}, {"number": 3}, {"number": 4},
        {"number": 5}, {"number": 6}, {"number": 7}, {"number": 8}, {"number": 9}
    ]);
    assert_eq!(data, &expected);

    // test insert and select
    let res = client
        .get("/v1/sql?sql=insert into demo values('host, \"name', 66.6, 1024, 0)")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // select *
    let res = client
        .get("/v1/sql?sql=select * from demo limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let output = body.output();
    assert_eq!(output.len(), 1);

    assert_eq!(
        output[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"schema":{"column_schemas":[{"name":"host","data_type":"String"},{"name":"cpu","data_type":"Float64"},{"name":"memory","data_type":"Float64"},{"name":"ts","data_type":"TimestampMillisecond"}]},"rows":[["host, \"name",66.6,1024.0,0]],"total_rows":1}
        })).unwrap()
    );

    // select with projections
    let res = client
        .get("/v1/sql?sql=select cpu, ts from demo limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let output = body.output();
    assert_eq!(output.len(), 1);

    assert_eq!(
        output[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"schema":{"column_schemas":[{"name":"cpu","data_type":"Float64"},{"name":"ts","data_type":"TimestampMillisecond"}]},"rows":[[66.6,0]],"total_rows":1}
        })).unwrap()
    );

    // select with column alias
    let res = client
        .get("/v1/sql?sql=select cpu as c, ts as time from demo limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let output = body.output();
    assert_eq!(output.len(), 1);
    assert_eq!(
        output[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"schema":{"column_schemas":[{"name":"c","data_type":"Float64"},{"name":"time","data_type":"TimestampMillisecond"}]},"rows":[[66.6,0]],"total_rows":1}
        })).unwrap()
    );

    // test multi-statement
    let res = client
        .get("/v1/sql?sql=select cpu, ts from demo limit 1;select cpu, ts from demo where ts > 0;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let outputs = body.output();
    assert_eq!(outputs.len(), 2);
    assert_eq!(
        outputs[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"schema":{"column_schemas":[{"name":"cpu","data_type":"Float64"},{"name":"ts","data_type":"TimestampMillisecond"}]},"rows":[[66.6,0]],"total_rows":1}
        })).unwrap()
    );
    assert_eq!(
        outputs[1],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"rows":[], "schema":{"column_schemas":[{"name":"cpu","data_type":"Float64"},{"name":"ts","data_type":"TimestampMillisecond"}]}, "total_rows":0}
        }))
        .unwrap()
    );

    // test multi-statement with error
    let res = client
        .get("/v1/sql?sql=select cpu, ts from demo limit 1;select cpu, ts from demo2 where ts > 0;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    let body = serde_json::from_str::<ErrorResponse>(&res.text().await).unwrap();
    assert!(body.error().contains("Table not found"));
    assert_eq!(body.code(), ErrorCode::TableNotFound as u32);

    // test database given
    let res = client
        .get("/v1/sql?db=public&sql=select cpu, ts from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let outputs = body.output();
    assert_eq!(outputs.len(), 1);
    assert_eq!(
        outputs[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"schema":{"column_schemas":[{"name":"cpu","data_type":"Float64"},{"name":"ts","data_type":"TimestampMillisecond"}]},"rows":[[66.6,0]],"total_rows":1}
        })).unwrap()
    );

    // test database not found
    let res = client
        .get("/v1/sql?db=notfound&sql=select cpu, ts from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    let body = serde_json::from_str::<ErrorResponse>(&res.text().await).unwrap();
    assert_eq!(body.code(), ErrorCode::DatabaseNotFound as u32);

    // test catalog-schema given
    let res = client
        .get("/v1/sql?db=greptime-public&sql=select cpu, ts from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let outputs = body.output();
    assert_eq!(outputs.len(), 1);
    assert_eq!(
        outputs[0],
        serde_json::from_value::<GreptimeQueryOutput>(json!({
            "records":{"schema":{"column_schemas":[{"name":"cpu","data_type":"Float64"},{"name":"ts","data_type":"TimestampMillisecond"}]},"rows":[[66.6,0]],"total_rows":1}
        })).unwrap()
    );

    // test invalid catalog
    let res = client
        .get("/v1/sql?db=notfound2-schema&sql=select cpu, ts from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    let body = serde_json::from_str::<ErrorResponse>(&res.text().await).unwrap();
    assert_eq!(body.code(), ErrorCode::DatabaseNotFound as u32);

    // test invalid schema
    let res = client
        .get("/v1/sql?db=greptime-schema&sql=select cpu, ts from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    let body = serde_json::from_str::<ErrorResponse>(&res.text().await).unwrap();
    assert_eq!(body.code(), ErrorCode::DatabaseNotFound as u32);

    // test analyze format
    let res = client
        .get("/v1/sql?sql=explain analyze format json select cpu, ts from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<GreptimedbV1Response>(&res.text().await).unwrap();
    let output = body.output();
    assert_eq!(output.len(), 1);
    // this is something only json format can show
    assert!(format!("{:?}", output[0]).contains("\\\"param\\\""));

    // test csv format
    let res = client
        .get("/v1/sql?format=csv&sql=select cpu,ts,host from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = &res.text().await;
    // Must be escaped correctly: 66.6,0,"host, ""name"
    assert_eq!(body, "66.6,0,\"host, \"\"name\"\r\n");

    // csv with names
    let res = client
        .get("/v1/sql?format=csvWithNames&sql=select cpu,ts,host from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = &res.text().await;
    assert_eq!(body, "cpu,ts,host\r\n66.6,0,\"host, \"\"name\"\r\n");

    // csv with names and types
    let res = client
        .get("/v1/sql?format=csvWithNamesAndTypes&sql=select cpu,ts,host from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = &res.text().await;
    assert_eq!(
        body,
        "cpu,ts,host\r\nFloat64,TimestampMillisecond,String\r\n66.6,0,\"host, \"\"name\"\r\n"
    );
    // test null format
    let res = client
        .get("/v1/sql?format=null&sql=select cpu,ts,host from demo limit 1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = &res.text().await;
    assert!(body.contains("1 rows in set."));
    assert!(body.ends_with("sec.\n"));

    // test parse method
    let res = client.get("/v1/sql/parse?sql=desc table t").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    assert_eq!(
        res.text().await,
        r#"[{"DescribeTable":{"name":[{"Identifier":{"value":"t","quote_style":null,"span":{"start":{"line":0,"column":0},"end":{"line":0,"column":0}}}}]}}]"#,
    );

    // test timezone header
    let res = client
        .get("/v1/sql?&sql=show variables system_time_zone")
        .header(
            TryInto::<HeaderName>::try_into(GREPTIME_TIMEZONE_HEADER_NAME.to_string()).unwrap(),
            "Asia/Shanghai",
        )
        .send()
        .await
        .text()
        .await;
    assert!(res.contains("SYSTEM_TIME_ZONE") && res.contains("UTC"));
    let res = client
        .get("/v1/sql?&sql=show variables time_zone")
        .header(
            TryInto::<HeaderName>::try_into(GREPTIME_TIMEZONE_HEADER_NAME.to_string()).unwrap(),
            "Asia/Shanghai",
        )
        .send()
        .await
        .text()
        .await;
    assert!(res.contains("TIME_ZONE") && res.contains("Asia/Shanghai"));
    let res = client
        .get("/v1/sql?&sql=show variables system_time_zone")
        .send()
        .await
        .text()
        .await;
    assert!(res.contains("SYSTEM_TIME_ZONE") && res.contains("UTC"));
    let res = client
        .get("/v1/sql?&sql=show variables time_zone")
        .send()
        .await
        .text()
        .await;
    assert!(res.contains("TIME_ZONE") && res.contains("UTC"));
    guard.remove_all().await;
}

#[tokio::test]
async fn test_sql_format_api() {
    let (app, mut guard) =
        setup_test_http_app_with_frontend(StorageType::File, "sql_format_api").await;
    let client = TestClient::new(app).await;

    // missing sql
    let res = client.get("/v1/sql/format").send().await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    let body = serde_json::from_str::<ErrorResponse>(&res.text().await).unwrap();
    assert_eq!(body.code(), 1004);
    assert_eq!(body.error(), "sql parameter is required.");

    // with sql
    let res = client
        .get("/v1/sql/format?sql=select%201%20as%20x")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let json: serde_json::Value = serde_json::from_str(&res.text().await).unwrap();
    let formatted = json
        .get("formatted")
        .and_then(|v| v.as_str())
        .unwrap_or_default();
    assert_eq!(formatted, "SELECT 1 AS x;");

    // complex query
    let complex_query = "WITH RECURSIVE slow_cte AS (SELECT 1 AS n, md5(CAST(random() AS STRING)) AS hash UNION ALL SELECT n + 1, md5(concat(hash, n)) FROM slow_cte WHERE n < 4500) SELECT COUNT(*) FROM slow_cte";
    let encoded_complex_query = encode(complex_query);

    let query_params = format!("/v1/sql/format?sql={encoded_complex_query}");
    let res = client.get(&query_params).send().await;
    assert_eq!(res.status(), StatusCode::OK);

    let json: serde_json::Value = serde_json::from_str(&res.text().await).unwrap();
    let formatted = json
        .get("formatted")
        .and_then(|v| v.as_str())
        .unwrap_or_default();
    assert_eq!(
        formatted,
        "WITH RECURSIVE slow_cte AS (SELECT 1 AS n, md5(CAST(random() AS STRING)) AS hash UNION ALL SELECT n + 1, md5(concat(hash, n)) FROM slow_cte WHERE n < 4500) SELECT COUNT(*) FROM slow_cte;"
    );

    guard.remove_all().await;
}

pub async fn test_http_analyze_stream_tql(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "analyze_stream_tql").await;
    let client = TestClient::new(app).await;

    let res = client
        .post("/v1/sql/analyze/stream")
        .header("Accept", "text/event-stream")
        .form(&BTreeMap::from([(
            "sql".to_string(),
            "TQL ANALYZE VERBOSE (0, 10, '5s') demo".to_string(),
        )]))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    assert!(
        res.headers()
            .get("content-type")
            .and_then(|value| value.to_str().ok())
            .is_some_and(|value| value.starts_with("text/event-stream"))
    );
    let body = res.text().await;
    let final_event = body
        .split("\n\n")
        .find(|event| event.lines().any(|line| line == "event: final"))
        .expect(&body);
    let payload = final_event
        .lines()
        .find_map(|line| line.strip_prefix("data: "))
        .map(|data| serde_json::from_str::<Value>(data).unwrap())
        .unwrap();
    assert_eq!(payload["state"], "final");
    assert!(payload["metrics"].as_array().is_some_and(|v| !v.is_empty()));
    assert!(
        payload["output"]["records"]["rows"]
            .as_array()
            .is_some_and(|v| !v.is_empty())
    );

    let res = client
        .post("/v1/sql/analyze/stream")
        .header("Accept", "text/event-stream")
        .form(&BTreeMap::from([(
            "sql".to_string(),
            "TQL ANALYZE VERBOSE FORMAT JSON (0, 10, '5s') demo".to_string(),
        )]))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    assert!(res.text().await.contains("event: final"));

    guard.remove_all().await;
}

pub async fn test_http_sql_slow_query(store_type: StorageType) {
    let (app, mut guard) = setup_test_http_app_with_frontend_and_slow_query_threshold(
        store_type,
        "sql_api",
        Duration::from_millis(100),
    )
    .await;
    let client = TestClient::new(app).await;

    let slow_query = "SELECT count(*) FROM generate_series(1, 50000000)";
    let encoded_slow_query = encode(slow_query);

    let query_params = format!("/v1/sql?sql={encoded_slow_query}");
    let res = client.get(&query_params).send().await;
    assert_eq!(res.status(), StatusCode::OK);

    let table = format!("{}.{}", DEFAULT_PRIVATE_SCHEMA_NAME, SLOW_QUERY_TABLE_NAME);
    let query = format!(
        "SELECT {} FROM {table} WHERE {} = '{slow_query}'",
        SLOW_QUERY_TABLE_QUERY_COLUMN_NAME, SLOW_QUERY_TABLE_QUERY_COLUMN_NAME
    );

    let expected = format!(r#"[["{}"]]"#, slow_query);
    wait_for_data(&client, &query, &expected).await;

    guard.remove_all().await;
}

pub async fn test_prometheus_promql_api(store_type: StorageType) {
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "promql_api").await;
    let client = TestClient::new(app).await;

    let res = client
        .get("/v1/promql?query=abs(demo{host=\"Hangzhou\"})&start=0&end=100&step=5s")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let json_text = res.text().await;
    assert!(serde_json::from_str::<GreptimedbV1Response>(&json_text).is_ok());

    let res = client
        .get("/v1/promql?query=1&start=0&end=100&step=5s&format=csv")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let csv_body = &res.text().await;
    assert_eq!(
        "0,1.0\r\n5000,1.0\r\n10000,1.0\r\n15000,1.0\r\n20000,1.0\r\n25000,1.0\r\n30000,1.0\r\n35000,1.0\r\n40000,1.0\r\n45000,1.0\r\n50000,1.0\r\n55000,1.0\r\n60000,1.0\r\n65000,1.0\r\n70000,1.0\r\n75000,1.0\r\n80000,1.0\r\n85000,1.0\r\n90000,1.0\r\n95000,1.0\r\n100000,1.0\r\n",
        csv_body
    );

    guard.remove_all().await;
}

pub async fn test_prometheus_label_replace_response(store_type: StorageType) {
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "prometheus_label_replace_response").await;
    let client = TestClient::new(app).await;

    let query = encode(r#"label_replace(demo, "host_copy", "$1", "host", "(.*)")"#);
    let res = client
        .get(&format!("/v1/prometheus/api/v1/query?query={query}&time=0"))
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!({
            "resultType": "vector",
            "result": [{
                "metric": {
                    "__name__": "demo",
                    "host": "host1",
                    "host_copy": "host1"
                },
                "value": [0.0, "1.1"]
            }]
        }))
        .unwrap()
    );

    // A series whose source value does not match keeps its labels, without `host_copy`.
    let query = encode(r#"label_replace(demo, "host_copy", "$1", "host", "other(.*)")"#);
    let res = client
        .get(&format!("/v1/prometheus/api/v1/query?query={query}&time=0"))
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!({
            "resultType": "vector",
            "result": [{
                "metric": {
                    "__name__": "demo",
                    "host": "host1"
                },
                "value": [0.0, "1.1"]
            }]
        }))
        .unwrap()
    );

    guard.remove_all().await;
}

pub async fn test_prom_http_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_prom_app_with_frontend(store_type, "promql_api").await;
    let client = TestClient::new(app).await;

    // format_query
    let res = client
        .get("/v1/prometheus/api/v1/format_query?query=foo/bar")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    assert_eq!(
        res.text().await.as_str(),
        r#"{"status":"success","data":"foo / bar"}"#
    );

    // instant query
    let res = client
        .get("/v1/prometheus/api/v1/query?query=up&time=1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/prometheus/api/v1/query?query=up&time=1")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // instant query 1+1
    let res = client
        .get("/v1/prometheus/api/v1/query?query=1%2B1&time=1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(
            json!({"resultType":"scalar","result":[1.0,"2.0"]})
        )
        .unwrap()
    );

    // range query
    let res = client
        .get("/v1/prometheus/api/v1/query_range?query=up&start=1&end=100&step=5")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/prometheus/api/v1/query_range?query=up&start=1&end=100&step=5")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/prometheus/api/v1/query_range?query=count(count(up))&start=1&end=100&step=5")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    // query a metric table backed by a physical table with nanosecond precision to ensure
    // time cast and __tsid path work with logical table aliasing.
    let res = client
        .get(
            "/v1/prometheus/api/v1/query_range?query=demo_metrics_with_nanos&start=0&end=10&step=1",
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    let res = client
        .get("/v1/prometheus/api/v1/query_range?query=up&start=1&end=100&step=0.5")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/prometheus/api/v1/query_range?query=up&start=1&end=100&step=0.5")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // labels
    let res = client
        .get("/v1/prometheus/api/v1/labels?match[]=demo")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/prometheus/api/v1/labels?match[]=up")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/prometheus/api/v1/labels?match[]=demo&start=0")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["__name__", "host",])).unwrap()
    );

    // labels without match[] param
    let res = client.get("/v1/prometheus/api/v1/labels").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "__name__", "env", "host", "idc", "number",
        ]))
        .unwrap()
    );

    // labels query with multiple match[] params
    let res = client
        .get("/v1/prometheus/api/v1/labels?match[]=up&match[]=down")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/prometheus/api/v1/labels?match[]=up&match[]=down")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // series
    let res = client
        .get("/v1/prometheus/api/v1/series?match[]=demo{}&start=0&end=0")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");

    let PrometheusResponse::Series(mut series) = body.data else {
        unreachable!()
    };
    let actual = series
        .remove(0)
        .into_iter()
        .collect::<BTreeMap<Column, String>>();
    let expected = BTreeMap::from([
        ("__name__".into(), "demo".to_string()),
        ("host".into(), "host1".to_string()),
    ]);
    assert_eq!(actual, expected);

    let res = client
        .post("/v1/prometheus/api/v1/series?match[]=up&match[]=down")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    // An empty array will be deserialized into PrometheusResponse::Labels.
    // So here we compare the text directly.
    assert_eq!(res.text().await, r#"{"status":"success","data":[]}"#);

    // label values
    // should return error if there is no match[]
    let res = client
        .get("/v1/prometheus/api/v1/label/instance/values")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "error");
    assert!(
        prom_resp
            .error
            .is_some_and(|err| err.eq_ignore_ascii_case("match[] parameter is required"))
    );
    assert!(
        prom_resp
            .error_type
            .is_some_and(|err| err.eq_ignore_ascii_case("InvalidArguments"))
    );

    // single match[]
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1", "host2"])).unwrap()
    );

    // single match[]
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=300")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1"])).unwrap()
    );

    // single match[] with __name__
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]={__name__%3D%22demo%22}&start=0&end=300")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1"])).unwrap()
    );

    // single match[]
    let res = client
        .get("/v1/prometheus/api/v1/label/idc/values?match[]=demo_metrics_with_nanos&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["idc1"])).unwrap()
    );

    // match labels.
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=multi_labels{idc=\"idc1\", env=\"dev\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1", "host2"])).unwrap()
    );

    // special labels
    let res = client
        .get("/v1/prometheus/api/v1/label/__schema__/values?start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "greptime_private",
            "information_schema",
            "public"
        ]))
        .unwrap()
    );

    // special labels
    let res = client
        .get("/v1/prometheus/api/v1/label/__schema__/values?match[]=demo&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["public"])).unwrap()
    );

    // special labels
    let res = client
        .get("/v1/prometheus/api/v1/label/__database__/values?match[]=demo&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["public"])).unwrap()
    );

    // special labels
    let res = client
        .get("/v1/prometheus/api/v1/label/__database__/values?match[]=multi_labels{idc=\"idc1\", env=\"dev\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["public"])).unwrap()
    );

    // match special labels.
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=multi_labels{__schema__=\"public\", idc=\"idc1\", env=\"dev\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1", "host2"])).unwrap()
    );

    // match special labels.
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=multi_labels{__schema__=\"information_schema\", idc=\"idc1\", env=\"dev\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!([])).unwrap()
    );

    // search field name
    let res = client
        .get("/v1/prometheus/api/v1/label/__field__/values?match[]=demo")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["val"])).unwrap()
    );

    // limit
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=600&limit=1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1"])).unwrap()
    );

    // limit 0
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=600&limit=0")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1", "host2"])).unwrap()
    );

    // limit 10
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=600&limit=10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1", "host2"])).unwrap()
    );

    // special labels limit
    let res = client
        .get("/v1/prometheus/api/v1/label/__schema__/values?start=0&end=600&limit=2")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "greptime_private",
            "information_schema",
        ]))
        .unwrap()
    );

    // query an empty database should return nothing
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=600")
        .header(GREPTIME_DB_HEADER_NAME.clone(), "nonexistent")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!([])).unwrap()
    );

    // db header will be overrode by `db` in param
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=demo&start=0&end=600&db=public")
        .header(GREPTIME_DB_HEADER_NAME.clone(), "nonexistent")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!(["host1", "host2"])).unwrap()
    );

    // multiple match[]
    let res = client
        .get("/v1/prometheus/api/v1/label/instance/values?match[]=up&match[]=system_metrics")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none());
    assert!(prom_resp.error_type.is_none());

    // query non-string value
    let res = client
        .get("/v1/prometheus/api/v1/label/host/values?match[]=mito")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // query non-exist label in metric table
    let res = client
        .get("/v1/prometheus/api/v1/label/not_exist_label/values?match[]=demo&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none() && prom_resp.error_type.is_none());
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!([])).unwrap()
    );

    // query `__name__` without match[]
    // create a physical table and a logical table
    let res = client
        .get("/v1/sql?sql=create table physical_table (`ts` timestamp time index, `message` string) with ('physical_metric_table' = 'true');")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    let res = client
        .get("/v1/sql?sql=create table logic_table (`ts` timestamp time index, `message` string) with ('on_physical_table' = 'physical_table');")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    // query `__name__`
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none());
    assert!(prom_resp.error_type.is_none());
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "demo",
            "demo_metrics",
            "demo_metrics_with_nanos",
            "logic_table",
            "multi_labels",
        ]))
        .unwrap()
    );

    // query `__name__`
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={__name__=\"demo\"}")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none());
    assert!(prom_resp.error_type.is_none());
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!(["demo",])).unwrap()
    );

    // query `__name__`
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={__name__=~\".*demo.*\"}")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none());
    assert!(prom_resp.error_type.is_none());
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "demo",
            "demo_metrics",
            "demo_metrics_with_nanos",
        ]))
        .unwrap()
    );

    // query `__name__` by a matcher on an ordinary label: the metric engine
    // physical tables are scanned, so only metrics carrying the label value are
    // returned. `demo_metrics` shares `phy` with `demo` but has no `host` value.
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={host=\"host1\"}&start=0&end=600")
        .send()
        .await;
    let status = res.status();
    let text = res.text().await;
    assert_eq!(status, StatusCode::OK, "{text}");
    let prom_resp = serde_json::from_str::<PrometheusJsonResponse>(&text).unwrap();
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none());
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!(["demo", "multi_labels"])).unwrap()
    );

    // `__name__` matchers narrow the names the data resolved: `multi_labels`
    // also carries `idc="idc1"` but its name does not match.
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={__name__=~\"demo.*\", idc=\"idc1\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(prom_resp.status, "success");
    assert!(prom_resp.error.is_none());
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "demo_metrics",
            "demo_metrics_with_nanos",
        ]))
        .unwrap()
    );

    // The time range selects the series: `demo` carries `host="host2"` only at
    // t=600, so narrowing the range drops it while `multi_labels` at t=0 stays.
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={host=\"host2\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!(["demo", "multi_labels"])).unwrap()
    );
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={host=\"host2\"}&start=0&end=300")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!(["multi_labels"])).unwrap()
    );

    // Logical metrics sharing a physical table have NULL in the label columns
    // they don't use. Prometheus reads a label a series doesn't carry as the
    // empty string, so `demo_metrics` and `demo_metrics_with_nanos` — neither of
    // which has a `host` label — match both of these.
    //
    // `.%2B` is `.+`; a bare `+` decodes to a space in a query string. Grafana
    // encodes it the same way.
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={__name__=~\".%2B\", host=\"\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "demo_metrics",
            "demo_metrics_with_nanos",
        ]))
        .unwrap()
    );

    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={__name__=~\".%2B\", host!=\"host1\"}&start=0&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!([
            "demo",
            "demo_metrics",
            "demo_metrics_with_nanos",
            "multi_labels",
        ]))
        .unwrap()
    );

    // A pre-epoch RFC3339 bound is a valid range, not a panic.
    let res = client
        .get("/v1/prometheus/api/v1/label/__name__/values?match[]={host=\"host1\"}&start=1969-12-31T23:59:59Z&end=600")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let prom_resp = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        prom_resp.data,
        serde_json::from_value::<PrometheusResponse>(json!(["demo", "multi_labels"])).unwrap()
    );

    // buildinfo
    let res = client
        .get("/v1/prometheus/api/v1/status/buildinfo")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
    assert_eq!(body.status, "success");

    // parse_query
    let res = client
        .get("/v1/prometheus/api/v1/parse_query?query=http_requests")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let data = res.text().await;
    // we don't have deserialization for ast so we keep test simple and compare
    //  the json output directly.
    // the correctness should be covered by parser. In this test we only check
    //  response format.
    let expected = "{\"status\":\"success\",\"data\":{\"type\":\"vectorSelector\",\"name\":\"http_requests\",\"matchers\":[],\"offset\":0,\"startOrEnd\":null,\"timestamp\":null}}";
    assert_eq!(expected, data);

    let res = client
        .get("/v1/prometheus/api/v1/parse_query?query=not http_requests")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    let data = res.text().await;
    let expected = "{\"status\":\"error\",\"error\":\"invalid promql query\",\"errorType\":\"InvalidArguments\"}";
    assert_eq!(expected, data);

    // range_query with __name__ not-equal matcher
    let res = client
        .post("/v1/prometheus/api/v1/query_range?query=count by(__name__)({__name__=~'demo.*'})&start=1&end=100&step=5")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let data = res.text().await;
    assert!(
        data.contains("{\"__name__\":\"demo_metrics\"}")
            && data.contains("{\"__name__\":\"demo\"}")
    );

    let res = client
        .post("/v1/prometheus/api/v1/query_range?query=count by(__name__)({__name__=~'demo_metrics'})&start=1&end=100&step=5")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let data = res.text().await;
    assert!(
        data.contains("{\"__name__\":\"demo_metrics\"}")
            && !data.contains("{\"__name__\":\"demo\"}")
    );

    guard.remove_all().await;
}

/// A tiny compressed body must not be able to demand an unbounded decoded
/// allocation on the Prometheus remote-storage endpoints. The five-byte raw
/// Snappy varint (`ff ff ff ff 0f`) declares a ~4 GiB decoded length and must
/// be rejected by the decoded-size cap.
pub async fn test_prom_remote_compression_bomb_rejected(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_prom_app_with_frontend(store_type, "prom_remote_bomb").await;
    let client = TestClient::new(app).await;

    // Remote read with the five-byte varint bomb.
    let res = client
        .post("/v1/prometheus/read")
        .body(Vec::from([0xffu8, 0xff, 0xff, 0xff, 0x0f]))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    // Remote write v1 with the same bomb, via both codecs (exercising the
    // fallback: the zstd attempt fails on malformed input and the snappy
    // fallback must reject the declared size).
    for encoding in ["snappy", "zstd"] {
        let res = client
            .post("/v1/prometheus/write")
            .header("Content-Encoding", encoding)
            .body(Vec::from([0xffu8, 0xff, 0xff, 0xff, 0x0f]))
            .send()
            .await;
        assert_eq!(
            res.status(),
            StatusCode::BAD_REQUEST,
            "bomb with Content-Encoding {encoding} must be rejected"
        );
    }

    // The server must still serve valid requests afterwards.
    let write_request = WriteRequest {
        timeseries: prom_store::mock_timeseries(),
        ..Default::default()
    };
    let compressed = prom_store::snappy_compress(&write_request.encode_to_vec()).unwrap();
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    guard.remove_all().await;
}

/// The decoded (not encoded) size of a Prometheus remote write body must be
/// charged against the aggregate request-memory quota.
pub async fn test_prom_remote_write_decoded_body_charged(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    // 4 KiB aggregate quota with fail-fast policy.
    let memory_limiter = ServerMemoryLimiter::new(4096, OnExhaustedPolicy::Fail);
    let (app, mut guard) = setup_test_prom_app_with_frontend_and_memory_limiter(
        store_type,
        "prom_remote_write_quota",
        Some(memory_limiter),
    )
    .await;
    let client = TestClient::new(app).await;

    // A valid write request whose decoded protobuf clearly exceeds the quota
    // while its Snappy wire body stays far below it.
    let write_request = WriteRequest {
        timeseries: vec![TimeSeries {
            labels: vec![
                Label {
                    name: "__name__".to_string(),
                    value: "quota_metric".to_string(),
                },
                Label {
                    name: "payload".to_string(),
                    // ~8 KiB decoded; compresses to a few hundred bytes.
                    value: "p".repeat(8 * 1024),
                },
            ],
            samples: vec![Sample {
                value: 1.0,
                timestamp: 1000,
            }],
            ..Default::default()
        }],
        ..Default::default()
    };
    let compressed = prom_store::snappy_compress(&write_request.encode_to_vec()).unwrap();
    assert!(
        compressed.len() < 2048,
        "wire body must stay under the quota"
    );

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed)
        .send()
        .await;
    let status = res.status();
    let body = res.text().await;
    assert_eq!(
        status,
        StatusCode::TOO_MANY_REQUESTS,
        "decoded body larger than the aggregate quota must be rejected, got body: {body}"
    );

    guard.remove_all().await;
}

/// Decompressed ingestion bodies (routes using `RequestDecompressionLayer`)
/// must be charged against the aggregate request-memory quota as they are
/// decoded, not only their compressed wire size.
pub async fn test_influxdb_write_decoded_body_charged(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    // 4 KiB aggregate quota with fail-fast policy.
    let memory_limiter = ServerMemoryLimiter::new(4096, OnExhaustedPolicy::Fail);
    let http_opts = servers::http::HttpOptions {
        addr: format!("127.0.0.1:{}", common_test_util::ports::get_port()),
        ..Default::default()
    };
    let (app, mut guard) = setup_test_http_app_with_frontend_and_custom_options(
        store_type,
        "test_influxdb_decoded_quota",
        None,
        Some(http_opts),
        Some(memory_limiter),
    )
    .await;
    let client = TestClient::new(app).await;

    // A gzip member expanding to ~8 KiB of line protocol from a tiny wire body.
    let lines: Vec<String> = (0..200)
        .map(|i| format!("decoded_quota,host=h{} v=1.0,ts={}", i % 20, 1700000000 + i))
        .collect();
    let decoded = lines.join("\n");
    assert!(decoded.len() > 4096);
    let compressed = zstd::stream::encode_all(decoded.as_bytes(), 3).unwrap();
    assert!(
        compressed.len() < 2048,
        "wire body must stay under the quota"
    );

    // Compressed write whose decoded body exceeds the quota: rejected while
    // streaming (the extractor surfaces the aborted body as 400).
    let res = client
        .post("/v1/influxdb/write?db=public")
        .header("Content-Type", "text/plain")
        .header("Content-Encoding", "zstd")
        .body(compressed)
        .send()
        .await;
    assert_eq!(
        res.status(),
        StatusCode::TOO_MANY_REQUESTS,
        "decoded body larger than the aggregate quota must be rejected with 429"
    );

    // The server still serves a small compressed write within the quota.
    let small = "decoded_quota,host=ok v=1.0,ts=1700000000";
    let compressed_small = zstd::stream::encode_all(small.as_bytes(), 3).unwrap();
    let res = client
        .post("/v1/influxdb/write?db=public")
        .header("Content-Type", "text/plain")
        .header("Content-Encoding", "zstd")
        .body(compressed_small)
        .send()
        .await;
    assert_eq!(
        res.status(),
        StatusCode::NO_CONTENT,
        "small compressed write must succeed, got: {}",
        res.text().await
    );

    guard.remove_all().await;
}

/// PromQL must behave identically regardless of the physical metric table's
/// time index unit: the same samples are remote-written into physical tables
/// pre-created with second/micro/nano time indexes (plus a millisecond
/// baseline), and every PromQL response must match the baseline exactly.
/// Regression test for <https://github.com/GreptimeTeam/greptimedb/issues/9231>.
pub async fn test_promql_over_non_millisecond_physical_tables(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "promql_non_ms_units").await;
    let client = TestClient::new(app).await;

    let units = [
        ("ms", "timestamp(3)"),
        ("second", "timestamp(0)"),
        ("micro", "timestamp(6)"),
        ("nano", "timestamp(9)"),
    ];

    for (suffix, ts_type) in units {
        let db = format!("promql_units_{suffix}");
        let res = client
            .get(&format!("/v1/sql?db=public&sql=create database {db}"))
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK, "create database {db}");

        let res = client
            .get(&format!(
                "/v1/sql?db={db}&sql=CREATE TABLE greptime_physical_table \
                 (greptime_timestamp {ts_type} NOT NULL, greptime_value DOUBLE NULL, \
                 TIME INDEX (greptime_timestamp)) \
                 ENGINE = metric WITH ('physical_metric_table' = 'true')"
            ))
            .send()
            .await;
        assert_eq!(
            res.status(),
            StatusCode::OK,
            "create physical table with {ts_type} in {db}"
        );

        // unit_gauge{job="demo"} = 1.0 @ 1000ms, 2.0 @ 2000ms
        let write_request = WriteRequest {
            timeseries: vec![TimeSeries {
                labels: vec![
                    Label {
                        name: prom_store::METRIC_NAME_LABEL.to_string(),
                        value: "unit_gauge".to_string(),
                    },
                    Label {
                        name: "job".to_string(),
                        value: "demo".to_string(),
                    },
                ],
                samples: vec![
                    Sample {
                        value: 1.0,
                        timestamp: 1000,
                    },
                    Sample {
                        value: 2.0,
                        timestamp: 2000,
                    },
                ],
                ..Default::default()
            }],
            ..Default::default()
        };
        let compressed = prom_store::snappy_compress(&write_request.encode_to_vec()).unwrap();
        let write_url = format!("/v1/prometheus/write?db={db}");
        let res = client
            .post(write_url.as_str())
            .header("Content-Encoding", "snappy")
            .body(compressed)
            .send()
            .await;
        assert_eq!(
            res.status(),
            StatusCode::NO_CONTENT,
            "remote write into {db}"
        );
    }

    let query_paths = [
        "query?query=unit_gauge&time=2",
        "query_range?query=unit_gauge&start=0&end=5&step=1",
        "query?query=avg_over_time(unit_gauge[1m])&time=2",
        "query?query=rate(unit_gauge[1m])&time=2",
        "query?query=unit_gauge[1m]&time=2",
    ];
    for path in query_paths {
        let mut baseline: Option<(String, PrometheusResponse)> = None;
        for (suffix, _) in units {
            let db = format!("promql_units_{suffix}");
            let res = client
                .get(&format!("/v1/prometheus/api/v1/{path}&db={db}"))
                .send()
                .await;
            assert_eq!(res.status(), StatusCode::OK, "promql `{path}` on {db}");
            let body = res.json::<PrometheusJsonResponse>().await;
            assert_eq!(body.status, "success", "promql `{path}` on {db}");
            match &baseline {
                None => baseline = Some((suffix.to_string(), body.data)),
                Some((baseline_suffix, baseline_data)) => {
                    assert_eq!(
                        &body.data, baseline_data,
                        "promql `{path}` on {db} differs from the {baseline_suffix} baseline"
                    );
                }
            }
        }
    }

    // Pin the baseline responses so the cross-unit equality above cannot pass
    // vacuously on wrong data: the samples are 1.0@1s and 2.0@2s, and
    // `rate(unit_gauge[1m])@2s` follows Prometheus's extrapolated rate:
    // Δv=1 over a 1s sampled interval extrapolated to 1.5s, divided by the
    // 60s window -> 1.5/60 = 0.025 per second.
    let res = client
        .get("/v1/prometheus/api/v1/query?db=promql_units_ms&query=unit_gauge&time=2")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!({
            "resultType": "vector",
            "result": [{
                "metric": {"__name__": "unit_gauge", "job": "demo"},
                "value": [2.0, "2.0"]
            }]
        }))
        .unwrap()
    );

    let res = client
        .get("/v1/prometheus/api/v1/query_range?db=promql_units_ms&query=unit_gauge&start=0&end=5&step=1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!({
            "resultType": "matrix",
            "result": [{
                "metric": {"__name__": "unit_gauge", "job": "demo"},
                "values": [
                    [1.0, "1.0"], [2.0, "2.0"], [3.0, "2.0"], [4.0, "2.0"], [5.0, "2.0"]
                ]
            }]
        }))
        .unwrap()
    );

    let res = client
        .get("/v1/prometheus/api/v1/query?db=promql_units_ms&query=unit_gauge[1m]&time=2")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!({
            "resultType": "matrix",
            "result": [{
                "metric": {"__name__": "unit_gauge", "job": "demo"},
                "values": [[1.0, "1.0"], [2.0, "2.0"]]
            }]
        }))
        .unwrap()
    );

    let res = client
        .get("/v1/prometheus/api/v1/query?db=promql_units_ms&query=avg_over_time(unit_gauge[1m])&time=2")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        body.data,
        serde_json::from_value::<PrometheusResponse>(json!({
            "resultType": "vector",
            "result": [{
                "metric": {"__name__": "unit_gauge", "job": "demo"},
                "value": [2.0, "1.5"]
            }]
        }))
        .unwrap()
    );

    let res = client
        .get("/v1/prometheus/api/v1/query?db=promql_units_ms&query=rate(unit_gauge[1m])&time=2")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    let rate = match body.data {
        PrometheusResponse::PromData(data) => match data.result {
            PromQueryResult::Vector(v) => v[0].value.as_ref().unwrap().1.clone(),
            other => panic!("expected vector, got {other:?}"),
        },
        other => panic!("expected prom data, got {other:?}"),
    };
    assert_eq!(rate, "0.025");

    guard.remove_all().await;
}

pub async fn test_metrics_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app(store_type, "metrics_api").await;
    let client = TestClient::new(app).await;

    // Send a sql
    let res = client
        .get("/v1/sql?sql=select * from numbers limit 10")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // Call metrics api
    let res = client.get("/metrics").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.text().await;
    // Comment in the metrics text.
    assert!(body.contains("# HELP"));
    guard.remove_all().await;
}

pub async fn test_splunk_health(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_splunk_health").await;
    let client = TestClient::new(app).await;

    // The HEC health endpoint returns 200 with the HEC health body.
    let res = client
        .get("/v1/splunk/services/collector/health")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let json: serde_json::Value = serde_json::from_str(&res.text().await).unwrap();
    assert_eq!(json["text"], "HEC is healthy");
    assert_eq!(json["code"], 17);

    // The versioned alias `/health/1.0` serves the same handler.
    let res = client
        .get("/v1/splunk/services/collector/health/1.0")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let json: serde_json::Value = serde_json::from_str(&res.text().await).unwrap();
    assert_eq!(json["text"], "HEC is healthy");
    assert_eq!(json["code"], 17);

    // Query parameters (e.g. `ack`, `token`) are tolerated and ignored rather
    // than rejected
    for path in [
        "/v1/splunk/services/collector/health?ack=true&token=xyz",
        "/v1/splunk/services/collector/health/1.0?ack=true&token=xyz",
    ] {
        let res = client.get(path).send().await;
        assert_eq!(StatusCode::OK, res.status());
    }

    guard.remove_all().await;
}

pub async fn test_splunk_health_is_public(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();

    let user_provider =
        user_provider_from_option("static_user_provider:cmd:greptime_user=greptime_pwd").unwrap();
    let (app, _guard) = setup_test_http_app_with_frontend_and_user_provider(
        store_type,
        "test_splunk_health_is_public",
        Some(user_provider),
    )
    .await;
    let client = TestClient::new(app).await;

    // The health endpoint is registered in `PUBLIC_API_PREFIX`, so it must
    // succeed without credentials even when a user provider is configured.
    let res = client
        .get("/v1/splunk/services/collector/health")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
}

#[apply(both_deployment_cases)]
async fn test_splunk_skip_wal(distributed: bool) {
    let cases = vec![
        HttpWalCase::new(
            "Splunk event",
            "/v1/splunk/services/collector/event",
            "application/json",
            r#"{"event":"wal test","time":1700000000,"index":"wal_splunk"}"#,
        ),
        HttpWalCase::new(
            "Splunk raw",
            "/v1/splunk/services/collector/raw?index=wal_splunk_raw&time=1700000000",
            "text/plain",
            "wal test",
        ),
    ];
    check_http_skip_wal("splunk", &cases, distributed).await;
}

pub async fn test_splunk_logs(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();

    let user_provider =
        user_provider_from_option("static_user_provider:cmd:greptime_user=greptime_pwd").unwrap();
    let (app, mut guard) = setup_test_http_app_with_frontend_and_user_provider(
        store_type,
        "test_splunk_logs",
        Some(user_provider),
    )
    .await;
    let client = TestClient::new(app).await;

    // Authenticated SQL statement (the user-provider harness requires auth on
    // /v1/sql). Posted as a form so quotes and parentheses survive.
    async fn query(client: &TestClient, sql: &str) -> String {
        let res = client
            .post("/v1/sql")
            .header("Authorization", basic_auth("greptime_user", "greptime_pwd"))
            .form(&[("sql", sql)])
            .send()
            .await;
        let status = res.status();
        let body = res.text().await;
        assert_eq!(status, StatusCode::OK, "query failed: {sql}: {body}");
        // A failing statement still comes back as 200 with an `error` envelope.
        assert!(!body.contains("\"error\""), "query failed: {sql}: {body}");
        body
    }

    // HEC `Authorization: Splunk <user:pass>` + JSON content type.
    let splunk_headers = || {
        vec![
            (
                HeaderName::from_static("authorization"),
                HeaderValue::from_static("Splunk greptime_user:greptime_pwd"),
            ),
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/json"),
            ),
        ]
    };
    let event_path = "/v1/splunk/services/collector/event";

    // 1. Ingest a HEC batch (no `index` -> default `splunk_logs` table).
    let body = concat!(
        r#"{"event":"login ok","time":1700000000,"host":"web-01","source":"auth.log","sourcetype":"syslog","fields":{"region":"us-east"}}"#,
        "\n",
        r#"{"event":"login fail","time":1700000001,"host":"web-02","source":"auth.log","sourcetype":"syslog","fields":{"region":"us-west"}}"#,
    );
    let res = send_req(
        &client,
        splunk_headers(),
        event_path,
        body.as_bytes().to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    assert!(res.text().await.contains("\"code\":0"));

    // 2. Rows landed with the right values (proves route + auth + parse + insert,
    //    and that the default table name is `splunk_logs`).
    let rows = get_rows_from_output(
        &query(
            &client,
            "select host, region, event, greptime_timestamp from splunk_logs order by host",
        )
        .await,
    );
    // `time` (epoch seconds) maps to the nanosecond timestamp column.
    assert_eq!(
        rows,
        r#"[["web-01","us-east","login ok",1700000000000000000],["web-02","us-west","login fail",1700000001000000000]]"#
    );

    // 3. host/source/sourcetype + `fields` keys are tags (i.e. primary key).
    let create = query(&client, "show create table splunk_logs").await;
    let pk = create
        .split("PRIMARY KEY")
        .nth(1)
        .expect("splunk_logs should have a PRIMARY KEY");
    for col in ["host", "source", "sourcetype", "region"] {
        assert!(
            pk.contains(col),
            "expected `{col}` in primary key: {create}"
        );
    }
    // host/source/sourcetype lead the primary key (ahead of the `fields` tag `region`),
    let pos = |col: &str| pk.find(col).expect("column missing from primary key");
    assert!(
        pos("host") < pos("source")
            && pos("source") < pos("sourcetype")
            && pos("sourcetype") < pos("region"),
        "expected host/source/sourcetype to lead the primary key: {create}"
    );

    // 4. A brand-new `fields` key on a later write also becomes a tag (dynamic
    //    tag column added to the existing table's primary key).
    let body2 = r#"{"event":"deploy","time":1700000002,"host":"web-03","source":"deploy.log","sourcetype":"syslog","fields":{"datacenter":"dc1"}}"#;
    let res = send_req(
        &client,
        splunk_headers(),
        event_path,
        body2.as_bytes().to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let create = query(&client, "show create table splunk_logs").await;
    let pk = create.split("PRIMARY KEY").nth(1).unwrap();
    assert!(
        pk.contains("datacenter"),
        "expected dynamically-added `datacenter` in primary key: {create}"
    );

    // 5. An invalid `?table=` override returns HEC code 7 ("incorrect index").
    let res = send_req(
        &client,
        splunk_headers(),
        "/v1/splunk/services/collector/event?table=bad%20name",
        br#"{"event":"x","time":1700000003}"#.to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    assert!(res.text().await.contains("\"code\":7"));

    // 6. A custom pipeline (via `x-greptime-pipeline-name`) overrides identity and
    //    owns the schema: this one keeps host/event/timestamp and drops source/sourcetype.
    let pipeline_yaml = r#"
transform:
  - field: host
    type: string
    index: tag
  - field: event
    type: string
  - field: greptime_timestamp
    type: time
    index: timestamp
"#;
    let res = client
        .post("/v1/pipelines/splunk_custom")
        .header("Content-Type", "application/x-yaml")
        .header("Authorization", basic_auth("greptime_user", "greptime_pwd"))
        .body(pipeline_yaml)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "create pipeline failed");

    let mut headers = splunk_headers();
    headers.push((
        HeaderName::from_static("x-greptime-pipeline-name"),
        HeaderValue::from_static("splunk_custom"),
    ));
    let res = send_req(
        &client,
        headers,
        "/v1/splunk/services/collector/event?table=splunk_custom_tbl",
        br#"{"event":"hi","time":1700000010,"host":"web-09","source":"s","sourcetype":"st"}"#
            .to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let create = query(&client, "show create table splunk_custom_tbl").await;
    assert!(
        create.contains("host"),
        "custom pipeline kept host: {create}"
    );
    assert!(
        !create.contains("sourcetype"),
        "custom pipeline should have dropped sourcetype (identity would keep it): {create}"
    );

    // 6b. Ingest into a pre-created table with a fulltext index, then flush: the
    // index file is only written on flush, so this is the one path that reads it
    // back. `matches_term` is the predicate the index applier recognises.
    query(
        &client,
        "create table splunk_indexed (greptime_timestamp timestamp(9) time index, \
         \"event\" string fulltext index with(backend='bloom'))",
    )
    .await;
    let res = send_req(
        &client,
        splunk_headers(),
        "/v1/splunk/services/collector/event?table=splunk_indexed",
        br#"{"event":"disk pressure on web-01","time":1700000020}"#.to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    query(&client, "ADMIN FLUSH_TABLE('splunk_indexed')").await;
    let rows = get_rows_from_output(
        &query(
            &client,
            "select \"event\" from splunk_indexed where matches_term(\"event\", 'pressure')",
        )
        .await,
    );
    assert_eq!(rows, r#"[["disk pressure on web-01"]]"#);

    // 7. Auth failures return HEC codes: missing token -> 2 (401), bad token -> 4 (403).
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/json"),
        )],
        event_path,
        br#"{"event":"x","time":1700000020}"#.to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::UNAUTHORIZED, res.status());
    assert!(res.text().await.contains("\"code\":2"));

    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("authorization"),
                HeaderValue::from_static("Splunk baduser:badpass"),
            ),
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/json"),
            ),
        ],
        event_path,
        br#"{"event":"x","time":1700000021}"#.to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::FORBIDDEN, res.status());
    assert!(res.text().await.contains("\"code\":4"));

    guard.remove_all().await;
}

pub async fn test_splunk_raw(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();

    let user_provider =
        user_provider_from_option("static_user_provider:cmd:greptime_user=greptime_pwd").unwrap();
    let (app, mut guard) = setup_test_http_app_with_frontend_and_user_provider(
        store_type,
        "test_splunk_raw",
        Some(user_provider),
    )
    .await;
    let client = TestClient::new(app).await;

    async fn query(client: &TestClient, sql: &str) -> String {
        let res = client
            .get(format!("/v1/sql?sql={sql}").as_str())
            .header("Authorization", basic_auth("greptime_user", "greptime_pwd"))
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK, "query failed: {sql}");
        res.text().await
    }

    // HEC `Authorization: Splunk <user:pass>` + plain-text content type (raw bodies).
    let splunk_headers = || {
        vec![
            (
                HeaderName::from_static("authorization"),
                HeaderValue::from_static("Splunk greptime_user:greptime_pwd"),
            ),
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/plain"),
            ),
        ]
    };
    let raw_path = "/v1/splunk/services/collector/raw";

    // 1. Explicit `?linebreaker=%0A` ("\n") splits the body into one event per
    //    line, with request-level metadata; `channel` (param AND header) is
    //    accepted and ignored (ack protocol not implemented); `?time=` sets the
    //    timestamp for every event; `index` routes the table. Blank lines are
    //    skipped; indentation inside a line is preserved verbatim.
    let mut headers = splunk_headers();
    headers.push((
        HeaderName::from_static("x-splunk-request-channel"),
        HeaderValue::from_static("FE0ECFAD-13D5-401B-847D-77833BD77131"),
    ));
    let body = "line one\nline two\n\n  indented line";
    let res = send_req(
        &client,
        headers,
        &format!(
            "{raw_path}?channel=FE0ECFAD-13D5-401B-847D-77833BD77131\
             &host=web-01&source=nginx&sourcetype=access&index=raw_main&time=1700000100\
             &linebreaker=%0A"
        ),
        body.as_bytes().to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    assert!(res.text().await.contains("\"code\":0"));

    // 2. Rows landed: `message` holds each line verbatim, metadata columns carry the
    //    query-param values, `?time=` (epoch seconds) became the timestamp.
    let rows = get_rows_from_output(
        &query(
            &client,
            "select host, source, sourcetype, message, greptime_timestamp from raw_main order by message",
        )
        .await,
    );
    assert_eq!(
        rows,
        concat!(
            r#"[["web-01","nginx","access","  indented line",1700000100000000000],"#,
            r#"["web-01","nginx","access","line one",1700000100000000000],"#,
            r#"["web-01","nginx","access","line two",1700000100000000000]]"#
        )
    );

    // 3. host/source/sourcetype are tags (primary key); `message` is not.
    let create = query(&client, "show create table raw_main").await;
    let pk = create
        .split("PRIMARY KEY")
        .nth(1)
        .expect("raw_main should have a PRIMARY KEY");
    for col in ["host", "source", "sourcetype"] {
        assert!(
            pk.contains(col),
            "expected `{col}` in primary key: {create}"
        );
    }
    assert!(
        !pk.contains("message"),
        "`message` must not be a tag: {create}"
    );

    // 4. No query params at all: default table (`splunk_logs`), only timestamp +
    //    `message` columns, ingest-time timestamp.
    let res = send_req(
        &client,
        splunk_headers(),
        raw_path,
        b"Hello World".to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let rows = get_rows_from_output(&query(&client, "select message from splunk_logs").await);
    assert_eq!(rows, r#"[["Hello World"]]"#);

    // 4b. Without `?linebreaker=`, a multiline body (e.g. a stack trace) is stored
    //     as it is.
    let stack_trace = "java.lang.NullPointerException: boom\n\tat com.example.Foo.bar(Foo.java:42)\n\tat com.example.Main.main(Main.java:7)";
    let res = send_req(
        &client,
        splunk_headers(),
        &format!("{raw_path}?table=raw_multiline"),
        stack_trace.as_bytes().to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let rows = get_rows_from_output(&query(&client, "select message from raw_multiline").await);
    assert_eq!(
        rows,
        r#"[["java.lang.NullPointerException: boom\n\tat com.example.Foo.bar(Foo.java:42)\n\tat com.example.Main.main(Main.java:7)"]]"#
    );

    // 5. gzip-compressed raw body on the versioned alias (exercises the
    //    decompression layer and the `/raw/1.0` route).
    let res = send_req(
        &client,
        splunk_headers(),
        "/v1/splunk/services/collector/raw/1.0?table=raw_gzip",
        b"compressed line".to_vec(),
        true,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let rows = get_rows_from_output(&query(&client, "select message from raw_gzip").await);
    assert_eq!(rows, r#"[["compressed line"]]"#);

    // 6. Error paths: empty body -> 5 ("No data"); unparsable `?time=` -> 6;
    //    invalid `?table=` -> 7 ("incorrect index").
    let res = send_req(
        &client,
        splunk_headers(),
        raw_path,
        b"  \n ".to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    assert!(res.text().await.contains("\"code\":5"));

    let res = send_req(
        &client,
        splunk_headers(),
        &format!("{raw_path}?time=not-a-time"),
        b"x".to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    assert!(res.text().await.contains("\"code\":6"));

    let res = send_req(
        &client,
        splunk_headers(),
        &format!("{raw_path}?table=bad%20name"),
        b"x".to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    assert!(res.text().await.contains("\"code\":7"));

    // 7. Replay of a real Vector `splunk_hec_logs` (endpoint_target = "raw") request,
    //    path and headers verbatim from a wire capture (body trimmed):
    let mut headers = splunk_headers();
    headers.push((
        HeaderName::from_static("x-splunk-request-channel"),
        HeaderValue::from_static("b408271e-51af-43c1-a99f-9c21f78df0cf"),
    ));
    let res = send_req(
        &client,
        headers,
        "/v1/splunk/services/collector/raw?source=vector%2Dsrc&sourcetype=vector%5Fdemo&index=vector_raw&host=localhost",
        b"245.158.191.1 - AnthraX [13/Jul/2026:05:10:26 +0000] \"GET /wp-admin HTTP/1.0\" 300 17922".to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let rows = get_rows_from_output(
        &query(
            &client,
            "select host, source, sourcetype, message from vector_raw",
        )
        .await,
    );
    // the percent-encoded params (`vector%2Dsrc` etc.) decode to the plain values.
    assert_eq!(
        rows,
        r#"[["localhost","vector-src","vector_demo","245.158.191.1 - AnthraX [13/Jul/2026:05:10:26 +0000] \"GET /wp-admin HTTP/1.0\" 300 17922"]]"#
    );

    // 8. An unknown `Content-Encoding` passes through the decompression layer
    //    still compressed (`pass_through_unaccepted(true)`); the handler must reject
    //    it (code 6) instead of ingesting compressed bytes. `identity` is fine.
    //    (Known-but-broken encodings, e.g. `zstd` over gzip bytes, already fail in
    //    the layer itself.)
    //    Invalid UTF-8 must also be rejected with code 6, not lossily replaced.
    let mut headers = splunk_headers();
    headers.push((
        HeaderName::from_static("content-encoding"),
        HeaderValue::from_static("snappy"),
    ));
    let res = send_req(
        &client,
        headers,
        &format!("{raw_path}?table=raw_encoding"),
        compress_vec_with_gzip(b"still compressed".to_vec()),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    assert!(res.text().await.contains("\"code\":6"));

    let mut headers = splunk_headers();
    headers.push((
        HeaderName::from_static("content-encoding"),
        HeaderValue::from_static("identity"),
    ));
    let res = send_req(
        &client,
        headers,
        &format!("{raw_path}?table=raw_encoding"),
        b"identity line".to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let rows = get_rows_from_output(&query(&client, "select message from raw_encoding").await);
    assert_eq!(rows, r#"[["identity line"]]"#);

    let res = send_req(
        &client,
        splunk_headers(),
        &format!("{raw_path}?table=raw_bad_utf8"),
        vec![b'h', b'i', 0xff, 0xfe],
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    assert!(res.text().await.contains("\"code\":6"));

    guard.remove_all().await;
}

pub async fn test_health_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, _guard) = setup_test_http_app_with_frontend(store_type, "health_api").await;
    let client = TestClient::new(app).await;

    async fn health_api(client: &TestClient, endpoint: &str) {
        // we can call health api with both `GET` and `POST` method.
        let res_post = client.post(endpoint).send().await;
        assert_eq!(res_post.status(), StatusCode::OK);
        let res_get = client.get(endpoint).send().await;
        assert_eq!(res_get.status(), StatusCode::OK);

        // both `GET` and `POST` method return same result
        let body_text = res_post.text().await;
        assert_eq!(body_text, res_get.text().await);

        // currently health api simply returns an empty json `{}`, which can be deserialized to an empty `HealthResponse`
        assert_eq!(body_text, "{}");

        let body = serde_json::from_str::<HealthResponse>(&body_text).unwrap();
        assert_eq!(body, HealthResponse {});
    }

    health_api(&client, "/health").await;
    health_api(&client, "/ready").await;
}

pub async fn test_status_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, _guard) = setup_test_http_app_with_frontend(store_type, "status_api").await;
    let client = TestClient::new(app).await;

    let res_get = client.get("/status").send().await;
    assert_eq!(res_get.status(), StatusCode::OK);

    let res_body = res_get.text().await;
    assert!(res_body.contains("\"commit\":"));
    assert!(res_body.contains("\"branch\":"));
    assert!(res_body.contains("\"rustc_version\":"));
    assert!(res_body.contains("\"hostname\":"));
    assert!(res_body.contains("\"version\":"));
}

pub async fn test_config_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, _guard) = setup_test_http_app_with_frontend(store_type, "config_api").await;
    let client = TestClient::new(app).await;

    let res_get = client.get("/config").send().await;
    assert_eq!(res_get.status(), StatusCode::OK);

    let storage = if store_type != StorageType::File {
        format!(
            r#"[storage]
type = "{}"
providers = []

[storage.http_client]
pool_max_idle_per_host = 1024
connect_timeout = "30s"
timeout = "30s"
pool_idle_timeout = "1m 30s"
skip_ssl_validation = false"#,
            store_type
        )
    } else {
        format!(
            r#"[storage]
type = "{}"
providers = []"#,
            store_type
        )
    };

    let expected_toml_str = format!(
        r#"
enable_telemetry = true
auto_create_table = true
experimental_metric_export = false
max_in_flight_write_bytes = "0KiB"
write_bytes_exhausted_policy = "wait"
init_regions_in_background = false
init_regions_parallelism = 16
heartbeat_env_vars = []

[http]
addr = "127.0.0.1:4000"
timeout = "0s"
body_limit = "64MiB"
cors_allowed_origins = []
enable_cors = true
enable_api_server = false
api_server_addr = "127.0.0.1:4006"

[grpc]
bind_addr = "127.0.0.1:4001"
server_addr = "127.0.0.1:4001"
max_recv_message_size = "512MiB"
max_send_message_size = "512MiB"
flight_compression = "arrow_ipc"
runtime_size = 8
http2_keep_alive_interval = "10s"
http2_keep_alive_timeout = "3s"

[grpc.tls]
mode = "disable"
cert_path = ""
key_path = ""
ca_cert_path = ""
watch = false

[mysql]
enable = true
addr = "127.0.0.1:4002"
runtime_size = 2
keep_alive = "0s"
prepared_stmt_cache_size = 10000

[mysql.tls]
mode = "disable"
cert_path = ""
key_path = ""
ca_cert_path = ""
watch = false

[postgres]
enable = true
addr = "127.0.0.1:4003"
runtime_size = 2
keep_alive = "0s"

[postgres.tls]
mode = "disable"
cert_path = ""
key_path = ""
ca_cert_path = ""
watch = false

[opentsdb]
enable = true

[influxdb]
enable = true
default_merge_mode = "last_non_null"

[pending_rows_batcher]
protocols = []
pending_rows_flush_interval = "0s"
max_batch_rows = 100000
max_concurrent_flushes = 256
worker_channel_capacity = 65536
max_inflight_requests = 3000
flow_notification_queue_capacity = 1024

[jaeger]
enable = true

[otlp]
enable = true
trace_ingest_chunk_size = 512
experimental_enable_resource_info = true

[prom_store]
enable = true
with_metric_engine = true
prom_validation_mode = "strict"
pending_rows_flush_interval = "0s"
max_batch_rows = 100000
max_concurrent_flushes = 256
worker_channel_capacity = 65536
max_inflight_requests = 3000
flow_notification_queue_capacity = 1024

[wal]
provider = "raft_engine"
file_size = "128MiB"
purge_threshold = "1GiB"
purge_interval = "1m"
read_batch_size = 128
sync_write = false
enable_log_recycle = true
prefill_log_files = false

{storage}

[metadata_store]
file_size = "64MiB"
purge_threshold = "256MiB"
purge_interval = "1m"

[procedure]
max_retry_times = 3
retry_delay = "500ms"
max_running_procedures = 128

[flow]

[flow.batching_mode]
query_timeout = "10m"
slow_query_threshold = "1m"
experimental_min_refresh_duration = "5s"
grpc_conn_timeout = "5s"
experimental_flight_do_get_timeout = "10s"
experimental_grpc_max_retries = 3
experimental_frontend_scan_timeout = "30s"
experimental_max_filter_num_per_query = 20
experimental_time_window_merge_threshold = 3
experimental_enable_incremental_read = false
read_preference = "Leader"

[logging]
max_log_files = 720
max_log_dir_size = "0KiB"
append_stdout = true
enable_file_logging = true
enable_otlp_tracing = false
enable_per_region_metrics = false

[[region_engine]]

[region_engine.mito]
worker_channel_size = 128
worker_request_batch_size = 64
manifest_checkpoint_distance = 10
experimental_manifest_keep_removed_file_count = 256
experimental_manifest_keep_removed_file_ttl = "1h"
compress_manifest = false
experimental_enable_series_index = false
experimental_series_index_max_size = "5GiB"
experimental_enable_range_index = false
experimental_series_index_maintenance_interval = "5m"
experimental_series_index_bucket_width = "5days"
experimental_compaction_memory_limit = "unlimited"
experimental_compaction_on_exhausted = "wait"
auto_flush_interval = "10m"
default_region_write_buffer_size = "0KiB"
enable_write_cache = false
write_cache_path = ""
write_cache_size = "5GiB"
preload_index_cache = true
index_cache_percent = 20
enable_refill_cache_on_read = true
manifest_cache_size = "256MiB"
sst_write_buffer_size = "8MiB"
max_concurrent_scan_files = 384
allow_stale_entries = false
scan_memory_on_exhausted = "fail"
min_compaction_interval = "0s"
schedule_compaction_after_edit = true
default_flat_format = true
experimental_series_scan_v2 = false

[region_engine.mito.index]
aux_path = ""
staging_size = "2GiB"
staging_ttl = "7days"
build_mode = "sync"
write_buffer_size = "8MiB"
content_cache_page_size = "64KiB"

[region_engine.mito.inverted_index]
create_on_flush = "auto"
create_on_compaction = "auto"
apply_on_query = "auto"
mem_threshold_on_create = "auto"

[region_engine.mito.fulltext_index]
create_on_flush = "auto"
create_on_compaction = "auto"
apply_on_query = "auto"
mem_threshold_on_create = "auto"
compress = true

[region_engine.mito.bloom_filter_index]
create_on_flush = "auto"
create_on_compaction = "auto"
apply_on_query = "auto"
mem_threshold_on_create = "auto"

[region_engine.mito.gc]
enable = false
lingering_time = "1h"
unknown_file_lingering_time = "1day"
max_concurrent_lister_per_gc_job = 32
max_concurrent_gc_job = 4

[[region_engine]]

[region_engine.file]

[tracing]

[slow_query]
enable = true
record_type = "system_table"
threshold = "1s"
sample_ratio = 1.0
ttl = "2months 29days 2h 52m 48s"

[query]
parallelism = 0
allow_query_fallback = false
experimental_spill_mode = "default"
experimental_spill_max_temp_directory_size = "1GiB"
experimental_spill_compression = "uncompressed"
experimental_memory_pool_policy = "greedy"

[memory]
enable_heap_profiling = true

[pipeline]
cache_ttl = "10s"

[event_recorder]
ttl = "2months 29days 2h 52m 48s"
"#,
    )
    .trim()
    .to_string();
    let body_text = drop_lines_with_inconsistent_results(res_get.text().await);
    similar_asserts::assert_eq!(expected_toml_str, body_text);
}

fn drop_lines_with_inconsistent_results(input: String) -> String {
    let inconsistent_results = [
        "dir =",
        "log_format =",
        "data_home =",
        "bucket =",
        "root =",
        "endpoint =",
        "region =",
        "enable_virtual_host_style =",
        "disable_ec2_metadata =",
        "cache_path =",
        "cache_capacity =",
        "memory_pool_size =",
        "scan_memory_limit =",
        "sas_token =",
        "scope =",
        "num_workers =",
        "scan_parallelism =",
        "global_write_buffer_size =",
        "global_write_buffer_reject_size =",
        "sst_meta_cache_size =",
        "vector_cache_size =",
        "page_cache_size =",
        "selector_result_cache_size =",
        "range_result_cache_size =",
        "prefilter_result_cache_size =",
        "metadata_cache_size =",
        "content_cache_size =",
        "result_cache_size =",
        "name =",
        "recovery_parallelism =",
        "max_background_index_builds =",
        "max_background_flushes =",
        "max_background_compactions =",
        "max_background_purges =",
        "enable_read_cache =",
        "max_total_body_memory =",
        "max_total_message_memory =",
    ];

    input
        .lines()
        .filter(|line| {
            // ignores
            let line = line.trim();
            for prefix in inconsistent_results {
                if line.starts_with(prefix) {
                    return false;
                }
            }
            true
        })
        .collect::<Vec<&str>>()
        .join(
            "
",
        )
}

pub async fn test_dynamic_tracer_toggle(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();

    let (app, mut guard) = setup_test_http_app(store_type, "test_dynamic_tracer_toggle").await;
    let client = TestClient::new(app).await;

    let disable_resp = client
        .post("/debug/enable_trace")
        .body("false")
        .send()
        .await;
    assert_eq!(disable_resp.status(), StatusCode::OK);
    assert_eq!(disable_resp.text().await, "trace disabled");

    let enable_resp = client.post("/debug/enable_trace").body("true").send().await;
    assert_eq!(enable_resp.status(), StatusCode::OK);
    assert_eq!(enable_resp.text().await, "trace enabled");

    let cleanup_resp = client
        .post("/debug/enable_trace")
        .body("false")
        .send()
        .await;
    assert_eq!(cleanup_resp.status(), StatusCode::OK);
    assert_eq!(cleanup_resp.text().await, "trace disabled");

    guard.remove_all().await;
}

#[cfg(feature = "dashboard")]
pub async fn test_dashboard_path(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, _guard) = setup_test_http_app_with_frontend(store_type, "dashboard_path").await;
    let client = TestClient::new(app).await;

    let res_get = client.get("/dashboard").send().await;
    assert_eq!(res_get.status(), StatusCode::PERMANENT_REDIRECT);
    let res_post = client.post("/dashboard/").send().await;
    assert_eq!(res_post.status(), StatusCode::OK);
    let res_get = client.get("/dashboard/").send().await;
    assert_eq!(res_get.status(), StatusCode::OK);

    // both `GET` and `POST` method return same result
    let body_text = res_post.text().await;
    assert_eq!(body_text, res_get.text().await);
}

#[cfg(not(feature = "dashboard"))]
pub async fn test_dashboard_path(_: StorageType) {}

#[cfg(feature = "dashboard")]
pub async fn test_dashboard_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "dashboard_api").await;
    let client = TestClient::new(app).await;

    // 1. List dashboards - should be empty initially
    let res = client.get("/v1/dashboards").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert!(dashboards.is_empty());

    // 2. Save a dashboard
    let dashboard_definition = r#"{"title": "My Dashboard", "panels": []}"#;
    let res = client
        .post("/v1/dashboards/test_dashboard")
        .body(dashboard_definition)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert_eq!(dashboards.len(), 1);
    assert_eq!(dashboards[0].get("name").unwrap(), "test_dashboard");

    // 3. Save another dashboard
    let res = client
        .post("/v1/dashboards/another_dashboard")
        .body(r#"{"title": "Another Dashboard"}"#)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 4. List dashboards - should have 2
    let res = client.get("/v1/dashboards").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert_eq!(dashboards.len(), 2);

    let names: Vec<&str> = dashboards
        .iter()
        .map(|d| d.get("name").unwrap().as_str().unwrap())
        .collect();
    assert!(names.contains(&"test_dashboard"));
    assert!(names.contains(&"another_dashboard"));

    // 5. Update a dashboard by posting again with new definition
    let updated_definition = r#"{"title": "Updated Dashboard", "panels": [{"id": 1}]}"#;
    let res = client
        .post("/v1/dashboards/test_dashboard")
        .body(updated_definition)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert_eq!(dashboards.len(), 1);
    assert_eq!(dashboards[0].get("name").unwrap(), "test_dashboard");

    // Verify the definition was updated by listing again
    let res = client.get("/v1/dashboards").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert_eq!(dashboards.len(), 2);

    // Find test_dashboard and verify it has updated definition
    let test_db = dashboards
        .iter()
        .find(|d| d.get("name").unwrap() == "test_dashboard")
        .unwrap();
    assert_eq!(
        test_db.get("definition").unwrap(),
        r#"{"title": "Updated Dashboard", "panels": [{"id": 1}]}"#
    );

    // 6. Delete one dashboard
    let res = client.delete("/v1/dashboards/test_dashboard").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert_eq!(dashboards.len(), 1);
    assert_eq!(dashboards[0].get("name").unwrap(), "test_dashboard");

    // 7. List dashboards - should have 1
    let res = client.get("/v1/dashboards").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert_eq!(dashboards.len(), 1);
    assert_eq!(dashboards[0].get("name").unwrap(), "another_dashboard");

    // 8. Delete the remaining dashboard
    let res = client
        .delete("/v1/dashboards/another_dashboard")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 9. List dashboards - should be empty
    let res = client.get("/v1/dashboards").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let body: Value = res.json().await;
    let dashboards = body.get("dashboards").unwrap().as_array().unwrap();
    assert!(dashboards.is_empty());

    guard.remove_all().await;
}

#[cfg(not(feature = "dashboard"))]
pub async fn test_dashboard_api(_: StorageType) {}

#[apply(both_deployment_cases)]
async fn test_prometheus_skip_wal(distributed: bool) {
    let mut cases = Vec::new();
    let prom = api::prom_store::remote::WriteRequest {
        timeseries: prom_store::mock_timeseries(),
        ..Default::default()
    };
    let mut prom_case = HttpWalCase::new(
        "Prometheus v1",
        "/v1/prometheus/write",
        "application/x-protobuf",
        prom_store::snappy_compress(&prom.encode_to_vec()).unwrap(),
    );
    prom_case.headers.push(("content-encoding", "snappy"));
    cases.push(prom_case);
    let prom_v2 = servers::prom_remote_write::v2::test_util::request_with_labels_and_samples(
        vec![
            (prom_store::METRIC_NAME_LABEL, "wal_prom_v2"),
            ("host", "a"),
        ],
        vec![api::greptime_proto::io::prometheus::write::v2::Sample {
            value: 1.0,
            timestamp: 1700000000000,
            start_timestamp: 0,
        }],
    );
    let mut prom_v2_case = HttpWalCase::new(
        "Prometheus v2",
        "/v1/prometheus/write",
        "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        prom_store::snappy_compress(&prom_v2.encode_to_vec()).unwrap(),
    );
    prom_v2_case.headers.push(("content-encoding", "snappy"));
    cases.push(prom_v2_case);
    check_http_skip_wal("prometheus", &cases, distributed).await;
}

pub async fn test_prometheus_remote_write(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "prometheus_remote_write").await;
    let client = TestClient::new(app).await;

    // write snappy encoded data
    let write_request = WriteRequest {
        timeseries: prom_store::mock_timeseries(),
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // The direct Prom remote-write handler creates tables without going through
    // the generic inserter. Its submitted events must retain this origin.
    wait_for_event_data(
        &client,
        "SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type IN ('create_table', 'create_logical_tables') AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'prometheus' AND json_get_string(event_context, 'reason') = 'auto_create'",
        r#"[["{\"protocol\":\"prometheus\",\"reason\":\"auto_create\"}"]]"#,
    )
    .await;

    let expected = "[[\"demo\"],[\"demo_metrics\"],[\"demo_metrics_with_nanos\"],[\"greptime_physical_table\"],[\"metric1\"],[\"metric2\"],[\"metric3\"],[\"mito\"],[\"multi_labels\"],[\"numbers\"],[\"phy\"],[\"phy2\"],[\"phy_ns\"]]";
    validate_data("prometheus_remote_write", &client, "show tables;", expected).await;

    let table_val = "[[1000,3.0,\"z001\",\"test_host1\"],[2000,4.0,\"z001\",\"test_host1\"]]";
    validate_data(
        "prometheus_remote_write",
        &client,
        "select * from metric2",
        table_val,
    )
    .await;

    // A metric-engine logical table stores its rows in the shared physical
    // table, so reading them back from SST files goes through a different
    // column mapping than the memtable read does.
    flush_table(&client, "metric2").await;
    validate_data(
        "prometheus_remote_write_after_flush",
        &client,
        "select * from metric2",
        table_val,
    )
    .await;

    // Prom RW tables carry the metric identity; type is inferred from naming.
    validate_data(
        "prometheus_remote_write_semantic_identity",
        &client,
        "select count(*) from information_schema.tables where table_name = 'metric2' \
         and create_options like '%greptime.semantic.signal_type=metric%' \
         and create_options like '%greptime.semantic.source=prometheus%' \
         and create_options like '%greptime.semantic.metric.metadata_quality=inferred%';",
        "[[1]]",
    )
    .await;

    // Write snappy encoded data with new labels
    let write_request = WriteRequest {
        timeseries: mock_timeseries_new_label(),
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // Adding a label is handled by PendingRowsSchemaAlterer, which is another
    // direct DDL path outside the generic inserter.
    wait_for_event_data(
        &client,
        "SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type IN ('alter_table', 'alter_logical_tables') AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'prometheus' AND json_get_string(event_context, 'reason') = 'auto_alter'",
        r#"[["{\"protocol\":\"prometheus\",\"reason\":\"auto_alter\"}"]]"#,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_prometheus_remote_write_v2(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "prometheus_remote_write_v2").await;
    let client = TestClient::new(app).await;

    let write_request = remote_write_v2::request_with_labels_and_samples(
        vec![
            (prom_store::METRIC_NAME_LABEL, "remote_write_v2_total"),
            ("job", "api"),
            ("instance", "localhost:9090"),
        ],
        vec![
            RemoteWriteV2Sample {
                value: 42.0,
                timestamp: 1000,
                start_timestamp: 0,
            },
            RemoteWriteV2Sample {
                value: 43.0,
                timestamp: 2000,
                start_timestamp: 0,
            },
        ],
    );
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .header(
            "Content-Type",
            "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        )
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);
    let headers = res.headers();
    assert_remote_write_v2_written_headers(&headers, "2");
    assert!(res.text().await.is_empty());

    validate_data(
        "prometheus_remote_write_v2_rows",
        &client,
        "select greptime_timestamp, greptime_value, job, instance from remote_write_v2_total order by greptime_timestamp;",
        "[[1000,42.0,\"api\",\"localhost:9090\"],[2000,43.0,\"api\",\"localhost:9090\"]]",
    )
        .await;

    validate_data(
        "prometheus_remote_write_v2_semantic_identity",
        &client,
        "select count(*) from information_schema.tables where table_name = 'remote_write_v2_total' \
         and create_options like '%greptime.semantic.signal_type=metric%' \
         and create_options like '%greptime.semantic.source=prometheus%' \
         and create_options like '%greptime.semantic.metric.metadata_quality=inferred%';",
        "[[1]]",
    )
    .await;

    // A series carrying inline metadata (v2 senders SHOULD) upgrades its
    // table's quality to declared, with the metric type and the unit
    // canonicalised from the OpenMetrics word to UCUM.
    let mut write_request = remote_write_v2::request_with_labels_and_samples(
        vec![
            (prom_store::METRIC_NAME_LABEL, "remote_write_v2_typed_total"),
            ("job", "api"),
        ],
        vec![RemoteWriteV2Sample {
            value: 1.0,
            timestamp: 1000,
            start_timestamp: 0,
        }],
    );
    let unit_ref = write_request.symbols.len() as u32;
    write_request.symbols.push("seconds".to_string());
    write_request.timeseries[0].metadata = Some(RemoteWriteV2Metadata {
        r#type: RemoteWriteV2MetricType::Counter as i32,
        help_ref: 0,
        unit_ref,
    });
    let compressed_request = prom_store::snappy_compress(&write_request.encode_to_vec())
        .expect("failed to encode snappy");
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .header(
            "Content-Type",
            "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        )
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    validate_data(
        "prometheus_remote_write_v2_declared_metadata",
        &client,
        "select count(*) from information_schema.tables where table_name = 'remote_write_v2_typed_total' \
         and create_options like '%greptime.semantic.metric.type=counter%' \
         and create_options like '%greptime.semantic.metric.unit=s%' \
         and create_options like '%greptime.semantic.metric.metadata_quality=declared%';",
        "[[1]]",
    )
    .await;

    let res = client
        .get("/v1/prometheus/api/v1/metadata?metric=remote_write_v2_typed_total")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        serde_json::to_string_pretty(&body).unwrap(),
        r#"{
  "status": "success",
  "data": {
    "remote_write_v2_typed_total": [
      {
        "type": "counter",
        "unit": "seconds",
        "help": ""
      }
    ]
  }
}"#
    );

    guard.remove_all().await;
}

pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_prom_app_with_frontend(
        store_type,
        "prometheus_remote_write_v2_native_histogram",
    )
    .await;
    let client = TestClient::new(app).await;

    let mut write_request = remote_write_v2::request_with_labels_and_samples(
        vec![
            (
                prom_store::METRIC_NAME_LABEL,
                "remote_write_v2_sample_seconds",
            ),
            ("job", "api"),
            ("instance", "localhost:9090"),
        ],
        vec![RemoteWriteV2Sample {
            value: 5.0,
            timestamp: 2500,
            start_timestamp: 0,
        }],
    );
    let name_ref = write_request
        .symbols
        .iter()
        .position(|symbol| symbol == prom_store::METRIC_NAME_LABEL)
        .unwrap() as u32;
    let job_ref = write_request
        .symbols
        .iter()
        .position(|symbol| symbol == "job")
        .unwrap() as u32;
    let api_ref = write_request
        .symbols
        .iter()
        .position(|symbol| symbol == "api")
        .unwrap() as u32;
    let instance_ref = write_request
        .symbols
        .iter()
        .position(|symbol| symbol == "instance")
        .unwrap() as u32;
    let localhost_ref = write_request
        .symbols
        .iter()
        .position(|symbol| symbol == "localhost:9090")
        .unwrap() as u32;
    let histogram_metric_ref = write_request.symbols.len() as u32;
    write_request
        .symbols
        .push("remote_write_v2_latency_seconds".to_string());
    write_request.timeseries.push(RemoteWriteV2TimeSeries {
        labels_refs: vec![
            name_ref,
            histogram_metric_ref,
            job_ref,
            api_ref,
            instance_ref,
            localhost_ref,
        ],
        histograms: vec![
            Histogram {
                count: Some(Count::CountInt(8)),
                sum: 10.0,
                schema: 1,
                zero_threshold: 0.001,
                zero_count: Some(ZeroCount::ZeroCountInt(1)),
                negative_spans: vec![BucketSpan {
                    offset: -2,
                    length: 1,
                }],
                negative_deltas: vec![1],
                positive_spans: vec![BucketSpan {
                    offset: 0,
                    length: 3,
                }],
                positive_deltas: vec![1, 2, -1],
                reset_hint: 2,
                timestamp: 3000,
                start_timestamp: 1500,
                ..Default::default()
            },
            Histogram {
                count: Some(Count::CountFloat(6.0)),
                sum: 20.0,
                schema: 2,
                zero_threshold: 0.002,
                zero_count: Some(ZeroCount::ZeroCountFloat(0.5)),
                positive_spans: vec![BucketSpan {
                    offset: 3,
                    length: 2,
                }],
                positive_counts: vec![2.0, 3.5],
                reset_hint: 3,
                timestamp: 4000,
                start_timestamp: 2500,
                ..Default::default()
            },
        ],
        ..Default::default()
    });
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .header(
            "Content-Type",
            "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        )
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);
    let headers = res.headers();
    assert_remote_write_v2_written_headers_with_histograms(&headers, "1", "2");
    assert!(res.text().await.is_empty());

    validate_data(
        "prometheus_remote_write_v2_native_histogram_sample_rows",
        &client,
        "select greptime_timestamp, greptime_value, job, instance from remote_write_v2_sample_seconds order by greptime_timestamp;",
        "[[2500,5.0,\"api\",\"localhost:9090\"]]",
    )
    .await;

    let histogram_sql = "select greptime_timestamp, greptime_native_histogram, job, instance from remote_write_v2_latency_seconds order by greptime_timestamp;";
    let histogram_rows = "[[3000,{\"count_f64\":null,\"count_i64\":8,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[1],\"negative_span_lengths\":[1],\"negative_span_offsets\":[-2],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,3,2],\"positive_span_lengths\":[3],\"positive_span_offsets\":[0],\"reset_hint\":2,\"schema\":1,\"start_timestamp\":1500,\"sum\":10.0,\"zero_count_f64\":null,\"zero_count_i64\":1,\"zero_threshold\":0.001},\"api\",\"localhost:9090\"],[4000,{\"count_f64\":6.0,\"count_i64\":null,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[2.0,3.5],\"positive_buckets_i64\":[],\"positive_span_lengths\":[2],\"positive_span_offsets\":[3],\"reset_hint\":3,\"schema\":2,\"start_timestamp\":2500,\"sum\":20.0,\"zero_count_f64\":0.5,\"zero_count_i64\":null,\"zero_threshold\":0.002},\"api\",\"localhost:9090\"]]";
    validate_data(
        "prometheus_remote_write_v2_native_histogram_rows",
        &client,
        histogram_sql,
        histogram_rows,
    )
    .await;

    // The native histogram column is a struct of nested lists, so its SST
    // encoding differs from its memtable representation.
    flush_table(&client, "remote_write_v2_latency_seconds").await;
    validate_data(
        "prometheus_remote_write_v2_native_histogram_rows_after_flush",
        &client,
        histogram_sql,
        histogram_rows,
    )
    .await;

    let res = client
        .get("/v1/prometheus/api/v1/metadata?metric=remote_write_v2_latency_seconds")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        serde_json::to_string_pretty(&body).unwrap(),
        r#"{
  "status": "success",
  "data": {
    "remote_write_v2_latency_seconds": [
      {
        "type": "histogram",
        "unit": "",
        "help": ""
      }
    ]
  }
}"#
    );

    let res = client
        .get("/v1/prometheus/api/v1/query?query=histogram_count(remote_write_v2_latency_seconds)&time=4")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    // Finite vector sample values use the HTTP response's ryu contract, so
    // integral f64 values retain their explicit `.0`; timestamps remain JSON numbers.
    assert_eq!(
        serde_json::to_string_pretty(&body).unwrap(),
        r#"{
  "status": "success",
  "data": {
    "resultType": "vector",
    "result": [
      {
        "metric": {
          "__name__": "remote_write_v2_latency_seconds",
          "instance": "localhost:9090",
          "job": "api"
        },
        "value": [
          4.0,
          "6.0"
        ]
      }
    ]
  }
}"#
    );

    let res = client
        .get("/v1/prometheus/api/v1/query?query=remote_write_v2_latency_seconds&time=4")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        serde_json::to_string_pretty(&body).unwrap(),
        r#"{
  "status": "success",
  "data": {
    "resultType": "vector",
    "result": [
      {
        "metric": {
          "__name__": "remote_write_v2_latency_seconds",
          "instance": "localhost:9090",
          "job": "api"
        },
        "histogram": [
          4.0,
          {
            "count": "6",
            "sum": "20",
            "buckets": [
              [
                3,
                "-0.002",
                "0.002",
                "0.5"
              ],
              [
                0,
                "1.4142135623730951",
                "1.681792830507429",
                "2"
              ],
              [
                0,
                "1.681792830507429",
                "2",
                "3.5"
              ]
            ]
          }
        ]
      }
    ]
  }
}"#
    );

    let res = client
        .get("/v1/prometheus/api/v1/query_range?query=remote_write_v2_latency_seconds&start=3&end=4&step=1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let body = res.json::<PrometheusJsonResponse>().await;
    assert_eq!(
        serde_json::to_string_pretty(&body).unwrap(),
        r#"{
  "status": "success",
  "data": {
    "resultType": "matrix",
    "result": [
      {
        "metric": {
          "__name__": "remote_write_v2_latency_seconds",
          "instance": "localhost:9090",
          "job": "api"
        },
        "histograms": [
          [
            3.0,
            {
              "count": "8",
              "sum": "10",
              "buckets": [
                [
                  1,
                  "-0.5",
                  "-0.3535533905932738",
                  "1"
                ],
                [
                  3,
                  "-0.001",
                  "0.001",
                  "1"
                ],
                [
                  0,
                  "0.7071067811865476",
                  "1",
                  "1"
                ],
                [
                  0,
                  "1",
                  "1.4142135623730951",
                  "3"
                ],
                [
                  0,
                  "1.4142135623730951",
                  "2",
                  "2"
                ]
              ]
            }
          ],
          [
            4.0,
            {
              "count": "6",
              "sum": "20",
              "buckets": [
                [
                  3,
                  "-0.002",
                  "0.002",
                  "0.5"
                ],
                [
                  0,
                  "1.4142135623730951",
                  "1.681792830507429",
                  "2"
                ],
                [
                  0,
                  "1.681792830507429",
                  "2",
                  "3.5"
                ]
              ]
            }
          ]
        ]
      }
    ]
  }
}"#
    );

    validate_data(
        "prometheus_remote_write_v2_native_histogram_table_created",
        &client,
        "select count(*) from information_schema.tables where table_name = 'remote_write_v2_latency_seconds';",
        "[[1]]",
    )
    .await;

    validate_data(
        "prometheus_remote_write_v2_native_histogram_suffix_table_not_created",
        &client,
        "select count(*) from information_schema.tables where table_name = 'remote_write_v2_latency_seconds_native_histogram';",
        "[[0]]",
    )
    .await;

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_prometheus_remote_write_batched_skip_wal(distributed: bool) {
    check_prometheus_remote_write_batched_skip_wal(distributed, false).await;
}

#[apply(both_deployment_cases)]
async fn test_prometheus_remote_write_v2_batched_skip_wal(distributed: bool) {
    check_prometheus_remote_write_batched_skip_wal(distributed, true).await;
}

async fn check_prometheus_remote_write_batched_skip_wal(distributed: bool, v2: bool) {
    common_telemetry::init_default_ut_logging();
    let mut instance =
        MockInstanceImpl::new(&format!("prom_bulk_skip_wal_v2_{v2}"), distributed).await;
    let server = build_test_prom_server(instance.frontend(), true, None).build();
    let client = TestClient::new(server.build(server.make_app()).unwrap()).await;

    write_prometheus_skip_wal_sample(&client, v2, 1000, None).await;
    wait_for_data(&client, "SELECT count(*) FROM wal_prom_bulk", "[[1]]").await;
    let mut before = instance.flush_and_snapshot_wal().await;
    // Reuse the same batcher across policy transitions to detect leaked policy.
    for (index, hint) in [None, Some("true"), Some("false"), None]
        .into_iter()
        .enumerate()
    {
        write_prometheus_skip_wal_sample(&client, v2, (index as i64 + 2) * 1000, hint).await;
        wait_for_data(
            &client,
            "SELECT count(*) FROM wal_prom_bulk",
            &format!("[[{}]]", index + 2),
        )
        .await;
        let after = instance.flush_and_snapshot_wal().await;
        assert_wal_delta(&before, &after, hint == Some("true"));
        before = after;
    }
    instance.shutdown().await;
}

async fn write_prometheus_skip_wal_sample(
    client: &TestClient,
    v2: bool,
    timestamp: i64,
    hint: Option<&str>,
) {
    let payload = if v2 {
        remote_write_v2::request_with_labels_and_samples(
            vec![(prom_store::METRIC_NAME_LABEL, "wal_prom_bulk")],
            vec![RemoteWriteV2Sample {
                value: 1.0,
                timestamp,
                start_timestamp: 0,
            }],
        )
        .encode_to_vec()
    } else {
        WriteRequest {
            timeseries: vec![TimeSeries {
                labels: vec![Label {
                    name: prom_store::METRIC_NAME_LABEL.to_string(),
                    value: "wal_prom_bulk".to_string(),
                }],
                samples: vec![Sample {
                    value: 1.0,
                    timestamp,
                }],
                ..Default::default()
            }],
            ..Default::default()
        }
        .encode_to_vec()
    };
    let mut request = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(prom_store::snappy_compress(&payload).unwrap());
    if v2 {
        request = request.header(
            "Content-Type",
            "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        );
    }
    if let Some(hint) = hint {
        request = request.header("x-greptime-insert-skip-wal", hint);
    }
    assert_eq!(request.send().await.status(), StatusCode::NO_CONTENT);
}

/// Batched remote write against a logical table bound to a non-millisecond
/// physical table: the bulk guard must reject the destination (the bulk
/// encode only produces millisecond batches) and fall back to the ordinary
/// insert path, which converts the requests to the table's unit.
pub async fn test_prometheus_remote_write_batched_mixed_time_index_units(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend_batched(store_type, "prom_rw_batched_mixed_units").await;
    let client = TestClient::new(app).await;

    let res = client
        .get("/v1/sql?db=public&sql=CREATE TABLE phy_us (greptime_timestamp TIMESTAMP(6) NOT NULL, greptime_value DOUBLE NULL, TIME INDEX (greptime_timestamp)) ENGINE = metric WITH ('physical_metric_table' = 'true')")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let write = |metric: &str, value: f64, timestamp: i64| {
        let write_request = WriteRequest {
            timeseries: vec![TimeSeries {
                labels: vec![
                    Label {
                        name: prom_store::METRIC_NAME_LABEL.to_string(),
                        value: metric.to_string(),
                    },
                    Label {
                        name: "job".to_string(),
                        value: "demo".to_string(),
                    },
                ],
                samples: vec![Sample { value, timestamp }],
                ..Default::default()
            }],
            ..Default::default()
        };
        prom_store::snappy_compress(&write_request.encode_to_vec()).unwrap()
    };

    // Create the logical table on the microsecond physical table. The bulk
    // path handles the microsecond selected physical table: the new table
    // is created on it and the samples are widened to its unit during
    // batch alignment.
    let res = client
        .post("/v1/prometheus/write?physical_table=phy_us")
        .header("Content-Encoding", "snappy")
        .body(write("us_metric", 2.5, 1500))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // Write the same existing table again while selecting the default
    // (millisecond) physical table: the destination is bound to another
    // physical table, so the bulk eligibility check must reject it and fall
    // back to the ordinary insert path — without the destination binding
    // check the bulk flush would write the rows through the selected
    // physical's regions, silently misplacing them.
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(write("us_metric", 3.5, 2000))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // Both samples are stored on the microsecond time index.
    validate_data(
        "prom_rw_batched_mixed_units",
        &client,
        "SELECT COUNT(*), MAX(greptime_value) FROM us_metric",
        "[[2,3.5]]",
    )
    .await;

    guard.remove_all().await;
}

/// A Prometheus write interceptor that redirects every remote write to the
/// `tenant_redirect` schema, like a per-tenant redirection: the incoming
/// context targets `public` while the destination tables live elsewhere.
struct PromSchemaRedirectInterceptor;

impl servers::interceptor::PromStoreProtocolInterceptor for PromSchemaRedirectInterceptor {
    type Error = servers::error::Error;

    fn pre_write(
        &self,
        _write_req: &RowInsertRequests,
        ctx: QueryContextRef,
    ) -> servers::error::Result<()> {
        ctx.set_current_schema("tenant_redirect");
        Ok(())
    }
}

async fn setup_redirecting_batched_prom_app(
    store_type: StorageType,
    name: &str,
) -> (TestClient, TestGuard) {
    let plugins = Plugins::default();
    plugins.insert::<servers::interceptor::PromStoreProtocolInterceptorRef<servers::error::Error>>(
        Arc::new(PromSchemaRedirectInterceptor),
    );
    let standalone = tests_integration::standalone::GreptimeDbStandaloneBuilder::new(name)
        .with_default_store_type(store_type)
        .with_plugin(plugins)
        .build()
        .await;
    let server = build_test_prom_server(standalone.fe_instance().clone(), true, None)
        .with_greptime_config_options(standalone.opts.datanode_options().to_toml().unwrap())
        .build();
    let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
    (client, standalone.guard)
}

async fn create_redirect_schema_with_microsecond_table(client: &TestClient, table: &str) {
    for sql in [
        "create database if not exists tenant_redirect",
        "CREATE TABLE tenant_redirect.phy_us \
         (ts timestamp(6) time index, val double, host string primary key) \
         engine=metric with ('physical_metric_table' = 'true')",
        &format!(
            "CREATE TABLE tenant_redirect.{table} \
             (ts timestamp(6) time index, val double, host string primary key) \
             engine=metric with ('on_physical_table' = 'phy_us')"
        ),
    ] {
        let res = client
            .get(format!("/v1/sql?sql={sql}").as_str())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK, "setup: {sql}");
    }
}

/// Regression test: with batching enabled and a pre_write interceptor that
/// redirects the context to a schema holding same-named non-millisecond
/// tables, the bulk eligibility must be evaluated against the redirected
/// destinations (after preflight) and fall back to the ordinary insert path
/// instead of failing in the bulk encode.
pub async fn test_prometheus_remote_write_batched_interceptor_time_index_units(
    store_type: StorageType,
) {
    common_telemetry::init_default_ut_logging();
    let (client, mut guard) =
        setup_redirecting_batched_prom_app(store_type, "prom_rw_batched_interceptor_units").await;
    create_redirect_schema_with_microsecond_table(&client, "intercept_metric").await;

    let write_request = WriteRequest {
        timeseries: vec![TimeSeries {
            labels: vec![
                Label {
                    name: prom_store::METRIC_NAME_LABEL.to_string(),
                    value: "intercept_metric".to_string(),
                },
                Label {
                    name: "job".to_string(),
                    value: "demo".to_string(),
                },
            ],
            samples: vec![Sample {
                value: 1.0,
                timestamp: 1000,
            }],
            ..Default::default()
        }],
        ..Default::default()
    };
    let compressed = prom_store::snappy_compress(&write_request.encode_to_vec()).unwrap();
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    validate_data(
        "prom_rw_batched_interceptor_units",
        &client,
        "SELECT COUNT(*) FROM tenant_redirect.intercept_metric",
        "[[1]]",
    )
    .await;

    guard.remove_all().await;
}

/// The v2 variant of the interceptor regression test: a mixed samples-plus-
/// histograms request whose existing sample destination lives in the
/// redirected schema with a microsecond time index must fall back to the
/// ordinary insert path (both series land), not fail in the bulk encode.
pub async fn test_prometheus_remote_write_v2_batched_interceptor_time_index_units(
    store_type: StorageType,
) {
    common_telemetry::init_default_ut_logging();
    let (client, mut guard) =
        setup_redirecting_batched_prom_app(store_type, "prom_rw_v2_batched_interceptor_units")
            .await;
    create_redirect_schema_with_microsecond_table(&client, "v2_mixed_sample").await;

    // One sample series targeting the existing microsecond table, plus one
    // histogram series for a new (millisecond) table.
    let mut symbols = vec![String::new()];
    let mut symbol = |value: &str| {
        symbols.push(value.to_string());
        (symbols.len() - 1) as u32
    };
    let name_ref = symbol("__name__");
    let sample_ref = symbol("v2_mixed_sample");
    let job_ref = symbol("job");
    let demo_ref = symbol("demo");
    let name_ref2 = symbol("__name__");
    let histo_ref = symbol("v2_mixed_histo");
    let request = RemoteWriteV2Request {
        symbols,
        timeseries: vec![
            RemoteWriteV2TimeSeries {
                labels_refs: vec![name_ref, sample_ref, job_ref, demo_ref],
                samples: vec![RemoteWriteV2Sample {
                    value: 1.0,
                    timestamp: 1000,
                    start_timestamp: 0,
                }],
                ..Default::default()
            },
            RemoteWriteV2TimeSeries {
                labels_refs: vec![name_ref2, histo_ref, job_ref, demo_ref],
                histograms: vec![Histogram {
                    count: Some(Count::CountInt(1)),
                    sum: 1.0,
                    positive_spans: vec![BucketSpan {
                        offset: 0,
                        length: 1,
                    }],
                    positive_deltas: vec![1],
                    timestamp: 1000,
                    start_timestamp: 1000,
                    ..Default::default()
                }],
                ..Default::default()
            },
        ],
    };
    let compressed = prom_store::snappy_compress(&request.encode_to_vec()).unwrap();
    let res = client
        .post("/v1/prometheus/write")
        .header(
            "Content-Type",
            "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        )
        .header("Content-Encoding", "snappy")
        .body(compressed)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    validate_data(
        "prom_rw_v2_batched_interceptor_units",
        &client,
        "SELECT COUNT(*) FROM tenant_redirect.v2_mixed_sample",
        "[[1]]",
    )
    .await;
    validate_data(
        "prom_rw_v2_batched_interceptor_units",
        &client,
        "SELECT COUNT(*) FROM tenant_redirect.v2_mixed_histo",
        "[[1]]",
    )
    .await;

    guard.remove_all().await;
}

/// Covers the batched (pending-rows-batcher) Prometheus remote write path, which
/// bypasses `PromStoreProtocolHandler::write`. Verifies the metric table is created
/// asynchronously and still carries the Prometheus semantic identity stamped on the
/// shared request context.
/// Regression test for <https://github.com/GreptimeTeam/greptimedb/issues/9342>:
/// prometheus remote write uses the logical batcher against a physical
/// metric table pre-created with a microsecond time index; the millisecond
/// samples are widened to the physical table's unit during batch alignment.
pub async fn test_prometheus_remote_write_batched_microsecond_physical_table(
    store_type: StorageType,
) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_prom_app_with_frontend_batched(
        store_type,
        "prometheus_remote_write_batched_us_physical",
    )
    .await;
    let client = TestClient::new(app).await;

    // Pre-create the default physical metric table with a microsecond time
    // index before any remote write, so the batched bulk path must handle it.
    let res = client
        .get(
            "/v1/sql?db=public&sql=CREATE TABLE greptime_physical_table \
             (greptime_timestamp TIMESTAMP(6) NOT NULL, greptime_value DOUBLE NULL, \
             TIME INDEX (greptime_timestamp)) \
             ENGINE = metric WITH ('physical_metric_table' = 'true')",
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let write_request = WriteRequest {
        timeseries: vec![prom_store::mock_timeseries()[0].clone()],
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    // submit_build_and_align counts every batcher submission in both
    // acknowledgement modes, so the stored-value assertions below cannot be
    // satisfied by a silent fallback to ordinary insertion.
    let submissions = servers::metrics::PENDING_ROWS_BATCH_INGEST_STAGE_ELAPSED
        .with_label_values(&["submit_build_and_align"]);
    let before = submissions.get_sample_count();

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);
    assert_eq!(
        submissions.get_sample_count() - before,
        1,
        "non-millisecond physical tables must use the logical batcher"
    );

    // metric1 samples are 1.0@1000ms and 2.0@2000ms; on the microsecond
    // physical table they must be stored as 1_000_000us and 2_000_000us.
    wait_for_data(
        &client,
        "select greptime_timestamp, greptime_value from metric1 order by greptime_timestamp",
        "[[1000000,1.0],[2000000,2.0]]",
    )
    .await;

    guard.remove_all().await;
}

/// Regression test for the per-series physical-table selection conflict:
/// one remote-write request containing the same metric under two different
/// physical-table selections must fall back to the ordinary insert path —
/// both when the destination table already exists (bound to one of the two
/// physicals) and when it is missing (conflicting creations). A consistent
/// selection must still use the batcher.
pub async fn test_prometheus_remote_write_batched_conflicting_physical_selections(
    store_type: StorageType,
) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_prom_app_with_frontend_batched(
        store_type,
        "prom_rw_batched_conflicting_physicals",
    )
    .await;
    let client = TestClient::new(app).await;

    let res = client
        .get(
            "/v1/sql?db=public&sql=CREATE TABLE p1              (greptime_timestamp TIMESTAMP(6) NOT NULL, greptime_value DOUBLE NULL,              TIME INDEX (greptime_timestamp))              ENGINE = metric WITH ('physical_metric_table' = 'true')",
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    // An existing logical table bound to p1.
    let res = client
        .get(
            "/v1/sql?db=public&sql=CREATE TABLE conflict_existing              (greptime_timestamp TIMESTAMP(6) NOT NULL, greptime_value DOUBLE NULL,              \"job\" STRING NULL, TIME INDEX (greptime_timestamp), PRIMARY KEY (\"job\"))              ENGINE = metric WITH ('on_physical_table' = 'p1')",
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let write = |metric: &str, job: &str, physical: Option<&str>, value: f64| {
        let mut labels = vec![
            Label {
                name: prom_store::METRIC_NAME_LABEL.to_string(),
                value: metric.to_string(),
            },
            Label {
                name: "job".to_string(),
                value: job.to_string(),
            },
        ];
        if let Some(physical) = physical {
            labels.push(Label {
                name: "x_greptime_physical_table".to_string(),
                value: physical.to_string(),
            });
        }
        TimeSeries {
            labels,
            samples: vec![Sample {
                value,
                timestamp: 1000,
            }],
            ..Default::default()
        }
    };
    let send = |timeseries: Vec<TimeSeries>| {
        let write_request = WriteRequest {
            timeseries,
            ..Default::default()
        };
        prom_store::snappy_compress(&write_request.encode_to_vec()).unwrap()
    };

    let submissions = servers::metrics::PENDING_ROWS_BATCH_INGEST_STAGE_ELAPSED
        .with_label_values(&["submit_build_and_align"]);

    // Existing destination under two different selections: the batch must
    // fall back (the batch for the unbound selection would otherwise write
    // through the default physical's regions), and both series land on the
    // bound physical.
    let before = submissions.get_sample_count();
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(send(vec![
            write("conflict_existing", "a", Some("p1"), 1.0),
            write("conflict_existing", "b", None, 2.0),
        ]))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);
    assert_eq!(
        submissions.get_sample_count() - before,
        0,
        "conflicting selections for an existing table must fall back"
    );
    wait_for_data(
        &client,
        "select count(*), sum(greptime_value) from conflict_existing",
        "[[2,3.0]]",
    )
    .await;

    // Missing destination under two different selections: conflicting
    // creations must also fall back, and both series must be visible
    // through the created table.
    let before = submissions.get_sample_count();
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(send(vec![
            write("conflict_fresh", "a", Some("p1"), 1.0),
            write("conflict_fresh", "b", None, 2.0),
        ]))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);
    assert_eq!(
        submissions.get_sample_count() - before,
        0,
        "conflicting selections for a missing table must fall back"
    );
    wait_for_data(
        &client,
        "select count(*), sum(greptime_value) from conflict_fresh",
        "[[2,3.0]]",
    )
    .await;

    // A consistent selection still uses the batcher.
    let before = submissions.get_sample_count();
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(send(vec![
            write("consistent_fresh", "a", Some("p1"), 1.0),
            write("consistent_fresh", "b", Some("p1"), 2.0),
        ]))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);
    assert_eq!(
        submissions.get_sample_count() - before,
        1,
        "a consistent selection must use the logical batcher"
    );
    wait_for_data(
        &client,
        "select count(*), sum(greptime_value) from consistent_fresh",
        "[[2,3.0]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_prometheus_remote_write_batched(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend_batched(store_type, "prometheus_remote_write_batched")
            .await;
    let client = TestClient::new(app).await;

    let write_request = WriteRequest {
        timeseries: prom_store::mock_timeseries(),
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // The batcher flushes asynchronously, so poll until the table exists and carries
    // the semantic identity (signal_type/source/metadata_quality).
    wait_for_data(
        &client,
        "select count(*) from information_schema.tables where table_name = 'metric2' \
         and create_options like '%greptime.semantic.signal_type=metric%' \
         and create_options like '%greptime.semantic.source=prometheus%' \
         and create_options like '%greptime.semantic.metric.metadata_quality=inferred%'",
        "[[1]]",
    )
    .await;

    // The batched path bypasses the operator's auto-create: v2 per-series
    // metadata must reach its table options through the batch create too.
    let mut write_request = remote_write_v2::request_with_labels_and_samples(
        vec![
            (
                prom_store::METRIC_NAME_LABEL,
                "remote_write_batched_typed_total",
            ),
            ("job", "api"),
        ],
        vec![RemoteWriteV2Sample {
            value: 1.0,
            timestamp: 1000,
            start_timestamp: 0,
        }],
    );
    let unit_ref = write_request.symbols.len() as u32;
    write_request.symbols.push("bytes".to_string());
    write_request.timeseries[0].metadata = Some(RemoteWriteV2Metadata {
        r#type: RemoteWriteV2MetricType::Gauge as i32,
        help_ref: 0,
        unit_ref,
    });
    let compressed_request = prom_store::snappy_compress(&write_request.encode_to_vec())
        .expect("failed to encode snappy");
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .header(
            "Content-Type",
            "application/x-protobuf;proto=io.prometheus.write.v2.Request",
        )
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    wait_for_data(
        &client,
        "select count(*) from information_schema.tables where table_name = 'remote_write_batched_typed_total' \
         and create_options like '%greptime.semantic.metric.type=gauge%' \
         and create_options like '%greptime.semantic.metric.unit=By%' \
         and create_options like '%greptime.semantic.metric.metadata_quality=declared%'",
        "[[1]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_prometheus_remote_special_labels(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "test_prometheus_remote_special_labels")
            .await;
    let client = TestClient::new(app).await;

    // write snappy encoded data
    let write_request = WriteRequest {
        timeseries: prom_store::mock_timeseries_special_labels(),
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    // create databases
    let res = client
        .post("/v1/sql?sql=create database idc3")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .post("/v1/sql?sql=create database idc4")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // write data
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // test idc3
    let expected = "[[\"f1\"],[\"idc3_lo_table\"]]";
    validate_data(
        "test_prometheus_remote_special_labels_idc3",
        &client,
        "show tables from idc3;",
        expected,
    )
    .await;
    let expected = "[[\"idc3_lo_table\",\"CREATE TABLE IF NOT EXISTS \\\"idc3_lo_table\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"greptime_value\\\" DOUBLE NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n  \'comment\' = 'Created on insertion',\\n  'greptime.semantic.metric.metadata_quality' = 'inferred',\\n  'greptime.semantic.signal_type' = 'metric',\\n  'greptime.semantic.source' = 'prometheus',\\n  'greptime.semantic.source_version' = '1.0',\\n  on_physical_table = 'f1'\\n)\"]]";
    validate_data(
        "test_prometheus_remote_special_labels_idc3_show_create_table",
        &client,
        "show create table idc3.idc3_lo_table",
        expected,
    )
    .await;
    let expected = "[[3000,42.0]]";
    validate_data(
        "test_prometheus_remote_special_labels_idc3_select",
        &client,
        "select * from idc3.idc3_lo_table",
        expected,
    )
    .await;

    // test idc4
    let expected = "[[\"f2\"],[\"idc4_local_table\"]]";
    validate_data(
        "test_prometheus_remote_special_labels_idc4",
        &client,
        "show tables from idc4;",
        expected,
    )
    .await;
    let expected = "[[\"idc4_local_table\",\"CREATE TABLE IF NOT EXISTS \\\"idc4_local_table\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"greptime_value\\\" DOUBLE NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n  \'comment\' = 'Created on insertion',\\n  'greptime.semantic.metric.metadata_quality' = 'inferred',\\n  'greptime.semantic.signal_type' = 'metric',\\n  'greptime.semantic.source' = 'prometheus',\\n  'greptime.semantic.source_version' = '1.0',\\n  on_physical_table = 'f2'\\n)\"]]";
    validate_data(
        "test_prometheus_remote_special_labels_idc4_show_create_table",
        &client,
        "show create table idc4.idc4_local_table",
        expected,
    )
    .await;
    let expected = "[[4000,99.0]]";
    validate_data(
        "test_prometheus_remote_special_labels_idc4_select",
        &client,
        "select * from idc4.idc4_local_table",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_prometheus_remote_write_with_pipeline(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "prometheus_remote_write_with_pipeline")
            .await;
    let client = TestClient::new(app).await;

    // write snappy encoded data
    let write_request = WriteRequest {
        timeseries: prom_store::mock_timeseries(),
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .header("x-greptime-log-pipeline-name", "greptime_identity")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    let expected = "[[\"demo\"],[\"demo_metrics\"],[\"demo_metrics_with_nanos\"],[\"greptime_physical_table\"],[\"metric1\"],[\"metric2\"],[\"metric3\"],[\"mito\"],[\"multi_labels\"],[\"numbers\"],[\"phy\"],[\"phy2\"],[\"phy_ns\"]]";
    validate_data(
        "prometheus_remote_write_pipeline",
        &client,
        "show tables;",
        expected,
    )
    .await;

    let table_val = "[[1000,3.0,\"z001\",\"test_host1\"],[2000,4.0,\"z001\",\"test_host1\"]]";
    validate_data(
        "prometheus_remote_write_pipeline",
        &client,
        "select * from metric2",
        table_val,
    )
    .await;

    guard.remove_all().await;
}

fn schema_series(schema: &str, value: f64) -> TimeSeries {
    TimeSeries {
        labels: vec![
            Label {
                name: "__name__".to_string(),
                value: "schema_metric".to_string(),
            },
            Label {
                name: "__schema__".to_string(),
                value: schema.to_string(),
            },
            Label {
                name: "instance".to_string(),
                value: "host1".to_string(),
            },
        ],
        samples: vec![Sample {
            value,
            timestamp: 1000,
        }],
        ..Default::default()
    }
}

fn schema_query(schema: Option<&str>) -> Query {
    let mut matchers = vec![LabelMatcher {
        name: "__name__".to_string(),
        value: "schema_metric".to_string(),
        r#type: MatcherType::Eq as i32,
    }];
    if let Some(schema) = schema {
        matchers.push(LabelMatcher {
            name: "__schema__".to_string(),
            value: schema.to_string(),
            r#type: MatcherType::Eq as i32,
        });
    }

    Query {
        start_timestamp_ms: 500,
        end_timestamp_ms: 1500,
        matchers,
        ..Default::default()
    }
}

async fn post_remote_read(client: &TestClient, queries: Vec<Query>) -> ReadResponse {
    let read_request = ReadRequest {
        queries,
        ..Default::default()
    };
    let compressed_request = prom_store::snappy_compress(&read_request.encode_to_vec())
        .expect("failed to encode snappy");

    let response = client
        .post("/v1/prometheus/read")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(response.status(), StatusCode::OK);

    let response_body = response.bytes().await;
    let decompressed_response = prom_store::snappy_decompress(&response_body).unwrap();
    ReadResponse::decode(decompressed_response.as_slice()).unwrap()
}

async fn post_remote_read_from_schema_a(client: &TestClient, queries: Vec<Query>) -> ReadResponse {
    let read_request = ReadRequest {
        queries,
        ..Default::default()
    };
    let compressed_request = prom_store::snappy_compress(&read_request.encode_to_vec())
        .expect("failed to encode snappy");

    let response = client
        .post("/v1/prometheus/read")
        .header(GREPTIME_DB_HEADER_NAME.clone(), "schema_a")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(response.status(), StatusCode::OK);

    let response_body = response.bytes().await;
    let decompressed_response = prom_store::snappy_decompress(&response_body).unwrap();
    ReadResponse::decode(decompressed_response.as_slice()).unwrap()
}

fn read_sample_values(read_response: ReadResponse) -> Vec<f64> {
    read_response
        .results
        .into_iter()
        .map(|result| {
            assert_eq!(result.timeseries.len(), 1);
            let timeseries = &result.timeseries[0];
            assert_eq!(timeseries.samples.len(), 1);
            assert_eq!(timeseries.samples[0].timestamp, 1000);
            timeseries.samples[0].value
        })
        .collect()
}

pub async fn test_prometheus_remote_schema_labels(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "test_prometheus_remote_schema_labels").await;
    let client = TestClient::new(app).await;

    const SCHEMA_A: &str = "schema_a";
    const SCHEMA_B: &str = "schema_b";

    for schema in [SCHEMA_A, SCHEMA_B] {
        let res = client
            .post(&format!("/v1/sql?sql=create database {schema}"))
            .header("Content-Type", "application/x-www-form-urlencoded")
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);
    }

    // Both series have the same metric, non-schema labels, and timestamp.
    let write_request = WriteRequest {
        timeseries: vec![
            schema_series(SCHEMA_A, 101.0),
            schema_series(SCHEMA_B, 202.0),
        ],
        ..Default::default()
    };
    let compressed_request = prom_store::snappy_compress(&write_request.encode_to_vec())
        .expect("failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // Independent controls establish that each selector resolves its own schema.
    assert_eq!(
        read_sample_values(post_remote_read(&client, vec![schema_query(Some(SCHEMA_A))]).await,),
        vec![101.0]
    );
    assert_eq!(
        read_sample_values(post_remote_read(&client, vec![schema_query(Some(SCHEMA_B))]).await,),
        vec![202.0]
    );

    // Results must stay aligned with the request query order.
    let forward_values = read_sample_values(
        post_remote_read(
            &client,
            vec![schema_query(Some(SCHEMA_A)), schema_query(Some(SCHEMA_B))],
        )
        .await,
    );
    let reverse_values = read_sample_values(
        post_remote_read(
            &client,
            vec![schema_query(Some(SCHEMA_B)), schema_query(Some(SCHEMA_A))],
        )
        .await,
    );
    assert_eq!(
        (forward_values, reverse_values),
        (vec![101.0, 202.0], vec![202.0, 101.0])
    );

    // With request database A, unqualified queries must resolve to A independently.
    let explicit_then_default_values = read_sample_values(
        post_remote_read_from_schema_a(
            &client,
            vec![schema_query(Some(SCHEMA_B)), schema_query(None)],
        )
        .await,
    );
    let default_then_explicit_values = read_sample_values(
        post_remote_read_from_schema_a(
            &client,
            vec![schema_query(None), schema_query(Some(SCHEMA_B))],
        )
        .await,
    );
    assert_eq!(
        (explicit_then_default_values, default_then_explicit_values),
        (vec![202.0, 101.0], vec![101.0, 202.0])
    );

    // write data to unknown schema
    let unknown_schema_series = TimeSeries {
        labels: vec![
            Label {
                name: "__name__".to_string(),
                value: "metric_unknown_schema".to_string(),
            },
            Label {
                name: "__schema__".to_string(),
                value: "unknown_schema".to_string(),
            },
            Label {
                name: "instance".to_string(),
                value: "host2".to_string(),
            },
        ],
        samples: vec![Sample {
            value: 200.0,
            timestamp: 2000,
        }],
        ..Default::default()
    };

    let unknown_write_request = WriteRequest {
        timeseries: vec![unknown_schema_series],
        ..Default::default()
    };
    let serialized_unknown_request = unknown_write_request.encode_to_vec();
    let compressed_unknown_request =
        prom_store::snappy_compress(&serialized_unknown_request).expect("failed to encode snappy");

    // Write data to unknown schema
    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_unknown_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    // Read data from unknown schema
    let unknown_read_request = ReadRequest {
        queries: vec![Query {
            start_timestamp_ms: 1500,
            end_timestamp_ms: 2500,
            matchers: vec![
                LabelMatcher {
                    name: "__name__".to_string(),
                    value: "metric_unknown_schema".to_string(),
                    r#type: MatcherType::Eq as i32,
                },
                LabelMatcher {
                    name: "__schema__".to_string(),
                    value: "unknown_schema".to_string(),
                    r#type: MatcherType::Eq as i32,
                },
            ],
            ..Default::default()
        }],
        ..Default::default()
    };

    let serialized_unknown_read_request = unknown_read_request.encode_to_vec();
    let compressed_unknown_read_request =
        prom_store::snappy_compress(&serialized_unknown_read_request)
            .expect("failed to encode snappy");

    let unknown_result = client
        .post("/v1/prometheus/read")
        .body(compressed_unknown_read_request)
        .send()
        .await;
    assert_eq!(unknown_result.status(), StatusCode::BAD_REQUEST);

    guard.remove_all().await;
}

pub async fn test_vm_proto_remote_write(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_prom_app_with_frontend(store_type, "vm_proto_remote_write").await;

    // handshake
    let client = TestClient::new(app).await;
    let res = client
        .post("/v1/prometheus/write?get_vm_proto_version=1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    assert_eq!(res.text().await, "1");

    // write zstd encoded data
    let write_request = WriteRequest {
        timeseries: prom_store::mock_timeseries(),
        ..Default::default()
    };
    let serialized_request = write_request.encode_to_vec();
    let compressed_request =
        zstd::stream::encode_all(&serialized_request[..], 1).expect("Failed to encode zstd");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "zstd")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // also test fallback logic, vmagent could sent data in wrong content-type
    // we encode it as zstd but send it as snappy
    let compressed_request =
        zstd::stream::encode_all(&serialized_request[..], 1).expect("Failed to encode zstd");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "snappy")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    // reversed
    let compressed_request =
        prom_store::snappy_compress(&serialized_request[..]).expect("Failed to encode snappy");

    let res = client
        .post("/v1/prometheus/write")
        .header("Content-Encoding", "zstd")
        .body(compressed_request)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::NO_CONTENT);

    guard.remove_all().await;
}

pub async fn test_pipeline_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_pipeline_api").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline_body = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

transform:
  - fields:
      - id1
      - id2
    type: int32
    index: inverted
  - fields:
      - logger
    type: string
  - field: type
    type: string
    index: skipping
    tag: true
  - field: log
    type: string
    index: fulltext
    tag: true
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/pipelines/greptime_guagua")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    assert_eq!(
        res.json::<serde_json::Value>().await["error"]
            .as_str()
            .unwrap(),
        "Invalid request parameter: pipeline_name cannot start with greptime_"
    );

    let res = client
        .post("/v1/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    let content = res.text().await;

    let content = serde_json::from_str(&content);
    assert!(content.is_ok());
    //  {"execution_time_ms":13,"pipelines":[{"name":"test","version":"2024-07-04 08:31:00.987136"}]}
    let content: Value = content.unwrap();

    let execution_time = content.get("execution_time_ms");
    assert!(execution_time.unwrap().is_number());
    let pipelines = content.get("pipelines");
    let pipelines = pipelines.unwrap().as_array().unwrap();
    assert_eq!(pipelines.len(), 1);
    let pipeline = pipelines.first().unwrap();
    assert_eq!(pipeline.get("name").unwrap(), "test");

    let version_str = pipeline
        .get("version")
        .unwrap()
        .as_str()
        .unwrap()
        .to_string();
    let encoded_ver_str: String =
        url::form_urlencoded::byte_serialize(version_str.as_bytes()).collect();

    // get pipeline
    let res = client
        .get(format!("/v1/pipelines/test?version={}", encoded_ver_str).as_str())
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    let content = res.text().await;
    let content = serde_json::from_str(&content);
    let content: Value = content.unwrap();
    let pipeline_yaml = content
        .get("pipelines")
        .unwrap()
        .as_array()
        .unwrap()
        .first()
        .unwrap()
        .get("pipeline")
        .unwrap()
        .as_str()
        .unwrap();
    let docs = YamlLoader::load_from_str(pipeline_yaml).unwrap();
    let body_yaml = YamlLoader::load_from_str(pipeline_body).unwrap();
    assert_eq!(docs, body_yaml);

    // Do not specify version, get the latest version
    let res = client.get("/v1/pipelines/test").send().await;
    assert_eq!(res.status(), StatusCode::OK);

    let content = res.text().await;
    let content = serde_json::from_str(&content);
    let content: Value = content.unwrap();
    let pipeline_yaml = content
        .get("pipelines")
        .unwrap()
        .as_array()
        .unwrap()
        .first()
        .unwrap()
        .get("pipeline")
        .unwrap()
        .as_str()
        .unwrap();
    let docs = YamlLoader::load_from_str(pipeline_yaml).unwrap();
    assert_eq!(docs, body_yaml);

    // 2. write data
    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "I",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/ingest?db=public&table=logs1&pipeline_name=test")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 3. check schema
    let expected_schema = "[[\"logs1\",\"CREATE TABLE IF NOT EXISTS \\\"logs1\\\" (\\n  \\\"id1\\\" INT NULL INVERTED INDEX,\\n  \\\"id2\\\" INT NULL INVERTED INDEX,\\n  \\\"logger\\\" STRING NULL,\\n  \\\"type\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n  \\\"log\\\" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false', false_positive_rate = '0.01', granularity = '10240'),\\n  \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n  TIME INDEX (\\\"time\\\"),\\n  PRIMARY KEY (\\\"type\\\", \\\"log\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  \'comment\' = 'Created on insertion',\\n  append_mode = 'true'\\n)\"]]";
    validate_data(
        "pipeline_schema",
        &client,
        "show create table logs1",
        expected_schema,
    )
    .await;

    // 4. cross-ref pipeline
    // create database test_db
    let res = client
        .post("/v1/sql?sql=create database test_db")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // check test_db created
    validate_data(
        "pipeline_db_schema",
        &client,
        "show databases",
        "[[\"greptime_private\"],[\"information_schema\"],[\"public\"],[\"test_db\"]]",
    )
    .await;

    // cross ref using public's pipeline
    let res = client
        .post("/v1/ingest?db=test_db&table=logs1&pipeline_name=test")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // check write success
    validate_data(
        "pipeline_db_schema",
        &client,
        "select * from test_db.logs1",
        "[[2436,2528,\"INTERACT.MANAGER\",\"I\",\"ClusterAdapter:enter sendTextDataToCluster\\\\n\",1716668197217000000]]",
    )
    .await;

    // The inverted, skipping and fulltext index files are only written on flush,
    // so the queries below are the first thing that reads them back.
    flush_table(&client, "test_db.logs1").await;
    validate_data(
        "pipeline_db_inverted_index",
        &client,
        "select id2 from test_db.logs1 where id1 = 2436",
        "[[2528]]",
    )
    .await;
    validate_data(
        "pipeline_db_skipping_index",
        &client,
        "select id1 from test_db.logs1 where type = 'I'",
        "[[2436]]",
    )
    .await;
    validate_data(
        "pipeline_db_fulltext_index",
        &client,
        "select id1 from test_db.logs1 where matches(log, 'ClusterAdapter')",
        "[[2436]]",
    )
    .await;

    // 5. remove pipeline
    let encoded_ver_str: String =
        url::form_urlencoded::byte_serialize(version_str.as_bytes()).collect();
    let res = client
        .delete(format!("/v1/pipelines/test?version={}", encoded_ver_str).as_str())
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // {"pipelines":[{"name":"test","version":"2024-07-04 08:55:29.038347"}],"execution_time_ms":22}
    let content = res.text().await;
    let content: Value = serde_json::from_str(&content).unwrap();
    assert!(content.get("execution_time_ms").unwrap().is_number());

    assert_eq!(
        content.get("pipelines").unwrap().to_string(),
        format!(r#"[{{"name":"test","version":"{}"}}]"#, version_str).as_str()
    );

    // 6. write data failed
    let res = client
        .post("/v1/ingest?db=public&table=logs1&pipeline_name=test")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    // todo(shuiyisong): refactor http error handling
    assert_ne!(res.status(), StatusCode::OK);

    guard.remove_all().await;
}

pub async fn test_pipeline_name_in_header(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_name_in_header").await;

    let client = TestClient::new(app).await;

    let pipeline_body = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true
transform:
  - field: message
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    // create pipeline named `test`
    let res = client
        .post("/v1/events/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let data_body = r#"
[
  {
    "time": "2024-05-25 20:16:37.217",
    "message": "hello header"
  }
]
"#;

    // 1. pipeline name provided via the `x-greptime-pipeline-name` header,
    //    without the `pipeline_name` query parameter.
    let res = client
        .post("/v1/events/logs?db=public&table=logs1")
        .header("Content-Type", "application/json")
        .header("x-greptime-pipeline-name", "test")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    validate_data(
        "pipeline_name_header",
        &client,
        "select message from logs1",
        "[[\"hello header\"]]",
    )
    .await;

    // 2. the deprecated `x-greptime-log-pipeline-name` header is also accepted.
    let res = client
        .post("/v1/events/logs?db=public&table=logs2")
        .header("Content-Type", "application/json")
        .header("x-greptime-log-pipeline-name", "test")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    validate_data(
        "pipeline_name_log_header",
        &client,
        "select message from logs2",
        "[[\"hello header\"]]",
    )
    .await;

    // 3. the header takes precedence over the query parameter. A bogus
    //    `pipeline_name` query param is overridden by the valid header.
    let res = client
        .post("/v1/events/logs?db=public&table=logs3&pipeline_name=does_not_exist")
        .header("Content-Type", "application/json")
        .header("x-greptime-pipeline-name", "test")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    validate_data(
        "pipeline_name_header_priority",
        &client,
        "select message from logs3",
        "[[\"hello header\"]]",
    )
    .await;

    // 4. when both headers are present, the non-deprecated
    //    `x-greptime-pipeline-name` wins over the deprecated
    //    `x-greptime-log-pipeline-name`.
    let res = client
        .post("/v1/events/logs?db=public&table=logs4")
        .header("Content-Type", "application/json")
        .header("x-greptime-pipeline-name", "test")
        .header("x-greptime-log-pipeline-name", "does_not_exist")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    validate_data(
        "pipeline_name_header_non_deprecated_priority",
        &client,
        "select message from logs4",
        "[[\"hello header\"]]",
    )
    .await;

    // 5. missing pipeline name in both header and query param is rejected.
    let res = client
        .post("/v1/events/logs?db=public&table=logs5")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_json_logs_skip_wal(distributed: bool) {
    let cases = vec![
        HttpWalCase::new(
            "JSON logs",
            "/v1/ingest?table=wal_logs&pipeline_name=greptime_identity",
            "application/json",
            r#"[{"message":"wal test"}]"#,
        ),
        HttpWalCase::new(
            "Events logs",
            "/v1/events/logs?table=wal_events&pipeline_name=greptime_identity",
            "application/json",
            r#"[{"message":"wal test"}]"#,
        ),
    ];
    check_http_skip_wal("json_logs", &cases, distributed).await;
}

pub async fn test_identity_pipeline(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_identity_pipeline").await;

    // handshake
    let client = TestClient::new(app).await;
    let body = r#"{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java", "json_object": {"a":1,"b":2}, "json_array":[1,2,3]}
{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java","hasagei":"hasagei","dongdongdong":"guaguagua", "json_object": {"a":1,"b":2}, "json_array":[1,2,3]}"#;
    let res = client
        .post("/v1/ingest?db=public&table=logs&pipeline_name=greptime_identity")
        .header("Content-Type", "application/json")
        .body(body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    let body: serde_json::Value = res.json().await;

    assert!(body.get("execution_time_ms").unwrap().is_number());
    assert_eq!(body["output"][0]["affectedrows"], 2);

    let res = client.get("/v1/sql?sql=select * from logs").send().await;

    assert_eq!(res.status(), StatusCode::OK);

    let line1_expected = r#"[null,"10.170.***.***",1453809242,"","10.200.**.***","[1,2,3]",1,2,"200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","aliyun-sdk-java",null,null]"#;
    let line2_expected = r#"[null,"10.170.***.***",1453809242,"","10.200.**.***","[1,2,3]",1,2,"200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","aliyun-sdk-java","guaguagua","hasagei"]"#;
    let res = client.get("/v1/sql?sql=select * from logs").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp: serde_json::Value = res.json().await;
    let result = resp["output"][0]["records"]["rows"].as_array().unwrap();
    assert_eq!(result.len(), 2);
    let mut line1 = result[0].as_array().unwrap().clone();
    let mut line2 = result[1].as_array().unwrap().clone();
    assert!(line1.first().unwrap().is_i64());
    assert!(line2.first().unwrap().is_i64());
    // set time index to null for assertion
    *line1.first_mut().unwrap() = serde_json::Value::Null;
    *line2.first_mut().unwrap() = serde_json::Value::Null;

    assert_eq!(
        line1,
        serde_json::from_str::<Vec<Value>>(line1_expected).unwrap()
    );
    assert_eq!(
        line2,
        serde_json::from_str::<Vec<Value>>(line2_expected).unwrap()
    );

    let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["json_array","String","","YES","","FIELD"],["json_object.a","Int64","","YES","","FIELD"],["json_object.b","Int64","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"],["dongdongdong","String","","YES","","FIELD"],["hasagei","String","","YES","","FIELD"]]"#;
    validate_data("identity_schema", &client, "desc logs", expected).await;

    guard.remove_all().await;
}

/// Test for identity pipeline with null values for new columns.
/// This test verifies that null values for columns not yet in the schema
/// are handled correctly - they should NOT push to the row without updating schema.
/// Regression test for: https://github.com/GreptimeTeam/greptimedb/issues/7654
///
/// The bug was that when a null value appeared for a column not in schema:
/// 1. The schema was not updated (correct)
/// 2. But row.push() was still called (BUG!)
///
/// This caused row.len() > schema.len(), leading to integer underflow in the
/// padding loop: `let diff = column_count - row.values.len()` where column_count < row.values.len()
/// causes usize underflow, making the loop try to push billions of elements → OOM.
///
/// To trigger this bug:
/// - Row 1 must have ONLY null values for NEW columns (pushes to row without schema update)
/// - Row 2 must have FEWER new non-null columns than Row 1's null columns
///
/// Example that triggers the bug:
/// - Row 1: {"a": null, "b": null} → row has [ts, null, null] (3 elements), schema has [ts] (1 col)
/// - Row 2: {"c": "value"} → row has [ts, "value"] (2 elements), schema has [ts, c] (2 cols)
/// - Padding: column_count=2, Row1.len()=3 → diff = 2-3 = underflow!
///
/// This test also covers resolve_value function branches:
/// - VrlValue::Null (null values)
/// - VrlValue::Integer (integer values)
/// - VrlValue::Float (floating point values)
/// - VrlValue::Boolean (boolean values)
/// - VrlValue::Bytes (string values)
///
/// Note: VrlValue::Array and VrlValue::Object branches are NOT covered because
/// the identity pipeline flattens nested objects and stringifies arrays before
/// resolve_value is called. Testing those branches would require a custom pipeline.
pub async fn test_identity_pipeline_with_null_column(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_identity_pipeline_with_null_column")
            .await;

    let client = TestClient::new(app).await;

    // Row 1 has 2 null values for new columns (a, b) - these push to row but don't add to schema
    // Row 2 has 1 non-null value for new column (c) - this adds to schema
    // Result: Row 1 has 3 elements [ts, null, null], schema has 2 columns [ts, c]
    // The padding loop: diff = 2 - 3 = usize underflow → OOM
    let body = r#"{"a":null,"b":null}
{"c":"value"}"#;

    let res = client
        .post("/v1/ingest?db=public&table=null_test&pipeline_name=greptime_identity")
        .header("Content-Type", "application/json")
        .body(body)
        .send()
        .await;

    // The buggy code would OOM or panic here due to the integer underflow causing
    // billions of push operations. If we get here, the bug is fixed.
    assert_eq!(res.status(), StatusCode::OK);

    let body: serde_json::Value = res.json().await;
    assert_eq!(body["output"][0]["affectedrows"], 2);

    // Verify the schema only contains columns that had non-null values
    // a and b should NOT be in the schema since they only had null values
    let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["c","String","","YES","","FIELD"]]"#;
    validate_data(
        "test_identity_pipeline_null_column_schema",
        &client,
        "desc null_test",
        expected,
    )
    .await;

    // Verify the data is correct
    let res = client
        .get(
            "/v1/sql?sql=select c from null_test order by case when c is null then 0 else 1 end, c",
        )
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp: serde_json::Value = res.json().await;
    let rows = resp["output"][0]["records"]["rows"].as_array().unwrap();
    assert_eq!(rows.len(), 2);

    // Row 1 has null for c (column didn't exist when row 1 was processed)
    assert!(rows[0][0].is_null());

    // Row 2 has "value" for c
    assert_eq!(rows[1][0], "value");

    // Test resolve_value function with various data types
    // Covers: VrlValue::Null, Integer, Float, Boolean, Bytes (string)
    // Note: Array and Object are flattened/stringified by identity pipeline before resolve_value
    let body_all_types = r#"{"null_col":null,"int_col":42,"float_col":3.10,"bool_col":true,"str_col":"hello","array_col":[1,2,3],"obj_col":{"nested":"value"}}"#;

    let res = client
        .post("/v1/ingest?db=public&table=all_types_test&pipeline_name=greptime_identity")
        .header("Content-Type", "application/json")
        .body(body_all_types)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body: serde_json::Value = res.json().await;
    assert_eq!(body["output"][0]["affectedrows"], 1);

    // Verify the schema contains all non-null column types
    // null_col should NOT be in schema since it only has null value
    // Note: arrays are converted to strings, nested objects are flattened (obj_col.nested)
    let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["array_col","String","","YES","","FIELD"],["bool_col","Boolean","","YES","","FIELD"],["float_col","Float64","","YES","","FIELD"],["int_col","Int64","","YES","","FIELD"],["obj_col.nested","String","","YES","","FIELD"],["str_col","String","","YES","","FIELD"]]"#;
    validate_data(
        "test_identity_pipeline_all_types_schema",
        &client,
        "desc all_types_test",
        expected,
    )
    .await;

    // Verify the data values are correct
    let res = client
        .get("/v1/sql?sql=select int_col, float_col, bool_col, str_col from all_types_test")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp: serde_json::Value = res.json().await;
    let rows = resp["output"][0]["records"]["rows"].as_array().unwrap();
    assert_eq!(rows.len(), 1);
    assert_eq!(rows[0][0], 42); // int_col
    assert_eq!(rows[0][1], 3.10); // float_col
    assert_eq!(rows[0][2], true); // bool_col
    assert_eq!(rows[0][3], "hello"); // str_col

    // Test with multiple rows containing different combinations of types
    // This ensures the schema evolution works correctly across rows
    let body_multi_rows = r#"{"int_val":100,"str_val":"first"}
{"int_val":200,"float_val":2.5,"bool_val":false}
{"str_val":"third","array_val":[4,5,6],"nested_obj":{"key":"val"}}"#;

    let res = client
        .post("/v1/ingest?db=public&table=multi_types_test&pipeline_name=greptime_identity")
        .header("Content-Type", "application/json")
        .body(body_multi_rows)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let body: serde_json::Value = res.json().await;
    assert_eq!(body["output"][0]["affectedrows"], 3);

    // Verify all columns from all rows are in the schema
    // Note: arrays are converted to strings, nested objects are flattened (nested_obj.key)
    // Columns appear in order they were first encountered: Row1(int_val,str_val), Row2(float_val,bool_val), Row3(array_val,nested_obj.key)
    let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["int_val","Int64","","YES","","FIELD"],["str_val","String","","YES","","FIELD"],["bool_val","Boolean","","YES","","FIELD"],["float_val","Float64","","YES","","FIELD"],["array_val","String","","YES","","FIELD"],["nested_obj.key","String","","YES","","FIELD"]]"#;
    validate_data(
        "test_identity_pipeline_multi_types_schema",
        &client,
        "desc multi_types_test",
        expected,
    )
    .await;

    // Verify data with nulls for columns not present in each row
    // ORDER BY with explicit NULLS LAST
    let res = client
        .get("/v1/sql?sql=select int_val, str_val, float_val, bool_val from multi_types_test order by case when int_val is null then 1 else 0 end, int_val")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp: serde_json::Value = res.json().await;
    let rows = resp["output"][0]["records"]["rows"].as_array().unwrap();
    assert_eq!(rows.len(), 3);

    // Row 1: int_val=100, str_val="first", float_val=null, bool_val=null
    assert_eq!(rows[0][0], 100);
    assert_eq!(rows[0][1], "first");
    assert!(rows[0][2].is_null());
    assert!(rows[0][3].is_null());

    // Row 2: int_val=200, str_val=null, float_val=2.5, bool_val=false
    assert_eq!(rows[1][0], 200);
    assert!(rows[1][1].is_null());
    assert_eq!(rows[1][2], 2.5);
    assert_eq!(rows[1][3], false);

    // Row 3: int_val=null, str_val="third", float_val=null, bool_val=null (nulls sorted last)
    assert!(rows[2][0].is_null());
    assert_eq!(rows[2][1], "third");
    assert!(rows[2][2].is_null());
    assert!(rows[2][3].is_null());

    guard.remove_all().await;
}

pub async fn test_pipeline_skip_error(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_skip_error").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline_body = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true
transform:
  - field: message
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    // 2. write data
    let data_body = r#"
[
  {
    "time": "2024-05-25 20:16:37.217",
      "message": "good message"
  },
  {
    "time": "2024-05-25",
    "message": "bad message"
  },
  {
    "message": "another bad message"
  }
]
"#;

    // write data without skip error
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=test")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    // 3. check data
    // without skip error, all data should be dropped
    validate_data(
        "pipeline_skip_error",
        &client,
        "show tables",
        "[[\"demo\"],[\"numbers\"]]",
    )
    .await;

    // write data with skip error
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=test&skip_error=true")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 4. check data
    // with skip error, only the first row should be written
    validate_data(
        "pipeline_skip_error",
        &client,
        "select message, time from logs1",
        "[[\"good message\",1716668197217000000]]",
    )
    .await;

    // write data with skip error on header
    // header has a higher priority than params
    // write one row
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=test&skip_error=false")
        .header("Content-Type", "application/json")
        .header("x-greptime-pipeline-params", "skip_error=true")
        .body(data_body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // write one row
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=test&skip_error=true")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // write zero row
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=test&skip_error=true")
        .header("Content-Type", "application/json")
        .header("x-greptime-pipeline-params", "skip_error=false")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    // 5. check data
    // it will be three rows with the same ts
    validate_data(
        "pipeline_skip_error",
        &client,
        "select message, time from logs1",
        "[[\"good message\",1716668197217000000],[\"good message\",1716668197217000000],[\"good message\",1716668197217000000]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_filter(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_filter").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline_body = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
  - filter:
      field: name
      targets:
        - John
transform:
  - field: name
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "time": "2024-05-25 20:16:37.217",
    "name": "John"
  },
  {
    "time": "2024-05-25 20:16:37.218",
    "name": "JoHN"
  },
  {
    "time": "2024-05-25 20:16:37.328",
    "name": "Jane"
  }
]
"#;

    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=test")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    validate_data(
        "pipeline_filter",
        &client,
        "select * from logs1",
        "[[\"Jane\",1716668197328000000]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_create_table(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_create_table").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline_body = r#"
processors:
- dissect:
    fields:
      - message
    patterns:
      - '%{ip_address} - %{username} [%{timestamp}] "%{http_method} %{request_line} %{protocol}" %{status_code} %{response_size}'
    ignore_missing: true
- date:
    fields:
      - timestamp
    formats:
      - "%d/%b/%Y:%H:%M:%S %z"

transform:
  - fields:
      - timestamp
    type: time
    index: timestamp
  - fields:
      - ip_address
    type: string
    index: skipping
  - fields:
      - username
    type: string
    tag: true
  - fields:
      - http_method
    type: string
    index: inverted
  - fields:
      - request_line
    type: string
    index: fulltext
  - fields:
      - protocol
    type: string
  - fields:
      - status_code
    type: int32
    index: inverted
    tag: true
  - fields:
      - response_size
    type: int64
    on_failure: default
    default: 0
  - fields:
      - message
    type: string
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let res = client
        .get("/v1/pipelines/test/ddl?table=logs1")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp: serde_json::Value = res.json().await;
    let sql = resp
        .get("sql")
        .unwrap()
        .get("sql")
        .unwrap()
        .as_str()
        .unwrap();

    assert_eq!(
        "CREATE TABLE IF NOT EXISTS `logs1` (\n  `timestamp` TIMESTAMP(9) NOT NULL,\n  `ip_address` STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  `username` STRING NULL,\n  `http_method` STRING NULL INVERTED INDEX,\n  `request_line` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false', false_positive_rate = '0.01', granularity = '10240'),\n  `protocol` STRING NULL,\n  `status_code` INT NULL INVERTED INDEX,\n  `response_size` BIGINT NULL,\n  `message` STRING NULL,\n  TIME INDEX (`timestamp`),\n  PRIMARY KEY (`username`, `status_code`)\n)\nENGINE=mito\nWITH(\n  append_mode = 'true'\n)",
        sql
    );

    guard.remove_all().await;
}

pub async fn test_pipeline_index_options(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_index_options").await;

    let client = TestClient::new(app).await;

    let pipeline_body = r#"
processors:
  - date:
      field: ts
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

transform:
  - field: message
    type: string
    index:
      type: fulltext
      options:
        analyzer: Chinese
        case_sensitive: true
        backend: bloom
        granularity: 2048
        false_positive_rate: 0.02
  - field: trace_id
    type: int64
    index:
      type: skipping
      options:
        granularity: 1024
        false_positive_rate: 0.05
        type: BLOOM
  - field: ts
    type: time
    index: timestamp
"#;

    let res = client
        .post("/v1/pipelines/index_options")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let data_body = r#"
[
  {
    "message": "hello greptime",
    "trace_id": 42,
    "ts": "2024-05-25 20:16:37.217"
  }
]
"#;
    let res = client
        .post("/v1/ingest?db=public&table=pipeline_index_options&pipeline_name=index_options")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let expected_schema = "[[\"pipeline_index_options\",\"CREATE TABLE IF NOT EXISTS \\\"pipeline_index_options\\\" (\\n  \\\"message\\\" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'bloom', case_sensitive = 'true', false_positive_rate = '0.02', granularity = '2048'),\\n  \\\"trace_id\\\" BIGINT NULL SKIPPING INDEX WITH(false_positive_rate = '0.05', granularity = '1024', type = 'BLOOM'),\\n  \\\"ts\\\" TIMESTAMP(9) NOT NULL,\\n  TIME INDEX (\\\"ts\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true'\\n)\"]]";
    validate_data(
        "pipeline_index_options_schema",
        &client,
        "show create table pipeline_index_options",
        expected_schema,
    )
    .await;

    // The fulltext and skipping index files are only written on flush, so the
    // queries below are the first thing that reads them back.
    flush_table(&client, "pipeline_index_options").await;
    validate_data(
        "pipeline_index_options_fulltext",
        &client,
        "select trace_id from pipeline_index_options where matches(message, 'greptime')",
        "[[42]]",
    )
    .await;
    validate_data(
        "pipeline_index_options_skipping",
        &client,
        "select message from pipeline_index_options where trace_id = 42",
        r#"[["hello greptime"]]"#,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_dispatcher(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_pipeline_dispatcher").await;

    // handshake
    let client = TestClient::new(app).await;

    let root_pipeline = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

dispatcher:
  field: type
  rules:
    - value: http
      table_suffix: _http
      pipeline: http
    - value: db
      table_suffix: _db
    - value: not_found
      table_suffix: _not_found
      pipeline: not_found

transform:
  - fields:
      - id1, id1_root
      - id2, id2_root
    type: int32
  - fields:
      - type
      - log
      - logger
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    let http_pipeline = r#"
processors:

transform:
  - fields:
      - id1, id1_http
      - id2, id2_http
    type: int32
  - fields:
      - log
      - logger
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/root")
        .header("Content-Type", "application/x-yaml")
        .body(root_pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    let res = client
        .post("/v1/events/pipelines/http")
        .header("Content-Type", "application/x-yaml")
        .body(http_pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "http",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "db",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "api",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "not_found",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=logs1&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::BAD_REQUEST);

    // 3. verify data
    let expected = "[[2436]]";
    validate_data(
        "test_dispatcher_pipeline default table",
        &client,
        "select id1_root from logs1",
        expected,
    )
    .await;

    let expected = "[[2436]]";
    validate_data(
        "test_dispatcher_pipeline http table",
        &client,
        "select id1_http from logs1_http",
        expected,
    )
    .await;

    let expected = "[[\"2436\"]]";
    validate_data(
        "test_dispatcher_pipeline db table",
        &client,
        "select id1 from logs1_db",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_suffix_template(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_pipeline_suffix_template").await;

    // handshake
    let client = TestClient::new(app).await;

    let root_pipeline = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

transform:
  - fields:
      - id1, id1_root
      - id2, id2_root
    type: int32
  - fields:
      - type
      - log
      - logger
    type: string
  - field: time
    type: time
    index: timestamp
table_suffix: _${type}
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/root")
        .header("Content-Type", "application/x-yaml")
        .body(root_pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "http",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  },
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "http",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  },
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "db",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  },
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "http",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  },
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=d_table&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 3. check table list
    validate_data(
        "test_pipeline_suffix_template_table_list",
        &client,
        "show tables",
        "[[\"d_table\"],[\"d_table_db\"],[\"d_table_http\"],[\"demo\"],[\"numbers\"]]",
    )
    .await;

    // 4. check each table's data
    validate_data(
        "test_pipeline_suffix_template_default",
        &client,
        "select * from d_table",
        "[[2436,2528,null,\"ClusterAdapter:enter sendTextDataToCluster\\\\n\",\"INTERACT.MANAGER\",1716668197217000000]]",
    )
    .await;

    validate_data(
        "test_pipeline_name_template_db",
        &client,
        "select * from d_table_db",
        "[[2436,2528,\"db\",\"ClusterAdapter:enter sendTextDataToCluster\\\\n\",\"INTERACT.MANAGER\",1716668197217000000]]",
    )
    .await;

    validate_data(
        "test_pipeline_name_template_http",
        &client,
        "select * from d_table_http",
        "[[2436,2528,\"http\",\"ClusterAdapter:enter sendTextDataToCluster\\\\n\",\"INTERACT.MANAGER\",1716668197217000000],[2436,2528,\"http\",\"ClusterAdapter:enter sendTextDataToCluster\\\\n\",\"INTERACT.MANAGER\",1716668197217000000],[2436,2528,\"http\",\"ClusterAdapter:enter sendTextDataToCluster\\\\n\",\"INTERACT.MANAGER\",1716668197217000000]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_context(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_pipeline_context").await;

    // handshake
    let client = TestClient::new(app).await;

    let root_pipeline = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

transform:
  - fields:
      - id1, id1_root
      - id2, id2_root
    type: int32
  - fields:
      - type
      - log
      - logger
    type: string
  - field: time
    type: time
    index: timestamp
table_suffix: _${type}
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/root")
        .header("Content-Type", "application/x-yaml")
        .body(root_pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "http",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n",
    "greptime_ttl": "1d",
    "greptime_skip_wal": "true"
  },
  {
    "id1": "2436",
    "id2": "2528",
    "logger": "INTERACT.MANAGER",
    "type": "db",
    "time": "2024-05-25 20:16:37.217",
    "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=d_table&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 3. check table list
    validate_data(
        "test_pipeline_context_table_list",
        &client,
        "show tables",
        "[[\"d_table_db\"],[\"d_table_http\"],[\"demo\"],[\"numbers\"]]",
    )
    .await;

    // 4. check each table's data
    // CREATE TABLE IF NOT EXISTS "d_table_db" (
    //     ... ignore
    //     )
    //   ENGINE=mito
    //   WITH(
    //     'comment' = 'Created on insertion',
    //     append_mode = 'true'
    //     )
    let expected = "[[\"d_table_db\",\"CREATE TABLE IF NOT EXISTS \\\"d_table_db\\\" (\\n  \\\"id1_root\\\" INT NULL,\\n  \\\"id2_root\\\" INT NULL,\\n  \\\"type\\\" STRING NULL,\\n  \\\"log\\\" STRING NULL,\\n  \\\"logger\\\" STRING NULL,\\n  \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n  TIME INDEX (\\\"time\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true'\\n)\"]]";

    validate_data(
        "test_pipeline_context_db",
        &client,
        "show create table d_table_db",
        expected,
    )
    .await;

    // CREATE TABLE IF NOT EXISTS "d_table_http" (
    //     ... ignore
    //     )
    //     ENGINE=mito
    //   WITH(
    //     'comment' = 'Created on insertion',
    //     append_mode = 'true',
    //     skip_wal = 'true',
    //     ttl = '1day'
    //     )
    let expected = "[[\"d_table_http\",\"CREATE TABLE IF NOT EXISTS \\\"d_table_http\\\" (\\n  \\\"id1_root\\\" INT NULL,\\n  \\\"id2_root\\\" INT NULL,\\n  \\\"type\\\" STRING NULL,\\n  \\\"log\\\" STRING NULL,\\n  \\\"logger\\\" STRING NULL,\\n  \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n  TIME INDEX (\\\"time\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  \'comment\' = 'Created on insertion',\\n  append_mode = 'true',\\n  skip_wal = 'true',\\n  ttl = '1day'\\n)\"]]";
    validate_data(
        "test_pipeline_context_http",
        &client,
        "show create table d_table_http",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_with_vrl(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_pipeline_with_vrl").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true
  - vrl:
      source: |
        .from_source = "channel_2"
        cond, err = .id1 > .id2
        if (cond) {
            .from_source = "channel_1"
        }
        del(.id1)
        del(.id2)
        .

transform:
  - fields:
      - from_source
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/root")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "id1": 2436,
    "id2": 123,
    "time": "2024-05-25 20:16:37.217"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=d_table&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    validate_data(
        "test_pipeline_with_vrl",
        &client,
        "select * from d_table",
        "[[\"channel_1\",1716668197217000000]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_with_hint_vrl(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_pipeline_with_hint_vrl").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true
  - vrl:
      source: |
        .greptime_table_suffix, err = "_" + .id
        .

transform:
  - fields:
      - id
    type: int32
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/root")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "id": "2436",
    "time": "2024-05-25 20:16:37.217"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=d_table&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    validate_data(
        "test_pipeline_with_hint_vrl",
        &client,
        "show tables",
        "[[\"d_table_2436\"],[\"demo\"],[\"numbers\"]]",
    )
    .await;

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_influxdb_skip_wal(distributed: bool) {
    let cases = vec![
        HttpWalCase::new(
            "InfluxDB v1",
            "/v1/influxdb/write?db=public",
            "text/plain",
            "wal_influx,host=a value=1 1700000000000000000",
        ),
        HttpWalCase::new(
            "InfluxDB v2",
            "/v1/influxdb/api/v2/write?bucket=public",
            "text/plain",
            "wal_influx_v2,host=a value=1 1700000000000000000",
        ),
    ];
    check_http_skip_wal("influxdb", &cases, distributed).await;
}

pub async fn test_influxdb_write_with_hints(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_influxdb_write_with_hints").await;

    let client = TestClient::new(app).await;

    let result = client
        .post("/v1/influxdb/write?db=public")
        .header("x-greptime-hints", "sst_format=flat,ttl=30d,skip_wal=true")
        .body("sst_fmt_table,host=host1 cpu=1.2 1664370459457010101")
        .send()
        .await;
    assert_eq!(result.status(), 204);

    let res = client
        .get("/v1/sql?sql=show create table sst_fmt_table")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp = res.text().await;
    assert!(
        resp.contains("sst_format = 'flat'"),
        "expected sst_format = 'flat' in SHOW CREATE TABLE output, got: {resp}"
    );
    assert!(
        resp.contains("ttl = '30days'"),
        "expected ttl = '30days' in SHOW CREATE TABLE output, got: {resp}"
    );
    assert!(
        resp.contains("skip_wal = 'true'"),
        "expected skip_wal = 'true' in SHOW CREATE TABLE output, got: {resp}"
    );

    guard.remove_all().await;
}

pub async fn test_influxdb_write_with_append_mode_hint(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(
        storage_type,
        "test_influxdb_write_with_append_mode_hint",
    )
    .await;

    let client = TestClient::new(app).await;

    let result = client
        .post("/v1/influxdb/write?db=public")
        .header("x-greptime-hint-append_mode", "true")
        .body("append_mode_table,host=host1 cpu=1.2 1664370459457010101")
        .send()
        .await;
    assert_eq!(result.status(), 204);

    let res = client
        .get("/v1/sql?sql=show create table append_mode_table")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp = res.text().await;
    assert!(
        resp.contains("append_mode = 'true'"),
        "expected append_mode = 'true' in SHOW CREATE TABLE output, got: {resp}"
    );
    assert!(
        resp.contains("merge_mode = 'last_row'"),
        "expected merge_mode = 'last_row' in SHOW CREATE TABLE output, got: {resp}"
    );

    guard.remove_all().await;
}

/// Test one-to-many VRL pipeline expansion.
/// This test verifies that a VRL processor can return an array, which results in
/// multiple output rows from a single input row.
pub async fn test_pipeline_one_to_many_vrl(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(storage_type, "test_pipeline_one_to_many_vrl").await;

    let client = TestClient::new(app).await;

    // Pipeline that expands events array into multiple rows
    let pipeline = r#"
processors:
  - date:
      field: timestamp
      formats:
        - "%Y-%m-%d %H:%M:%S"
      ignore_missing: true
  - vrl:
      source: |
        # Extract events array and expand each event into a separate row
        events = del(.events)
        base_host = del(.host)
        base_timestamp = del(.timestamp)

        # Map each event to a complete row object
        map_values(array!(events)) -> |event| {
            {
                "host": base_host,
                "event_type": event.type,
                "event_value": event.value,
                "timestamp": base_timestamp
            }
        }

transform:
  - field: host
    type: string
  - field: event_type
    type: string
  - field: event_value
    type: int32
  - field: timestamp
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/one_to_many")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data - single input with multiple events
    let data_body = r#"
[
  {
    "host": "server1",
    "timestamp": "2024-05-25 20:16:37",
    "events": [
      {"type": "cpu", "value": 80},
      {"type": "memory", "value": 60},
      {"type": "disk", "value": 45}
    ]
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=metrics&pipeline_name=one_to_many")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 3. verify: one input row should produce three output rows
    validate_data(
        "test_pipeline_one_to_many_vrl_count",
        &client,
        "select count(*) from metrics",
        "[[3]]",
    )
    .await;

    // 4. verify the actual data
    validate_data(
        "test_pipeline_one_to_many_vrl_data",
        &client,
        "select host, event_type, event_value from metrics order by event_type",
        "[[\"server1\",\"cpu\",80],[\"server1\",\"disk\",45],[\"server1\",\"memory\",60]]",
    )
    .await;

    // 5. Test with multiple input rows, each producing multiple output rows
    let data_body2 = r#"
[
  {
    "host": "server2",
    "timestamp": "2024-05-25 20:17:00",
    "events": [
      {"type": "cpu", "value": 90},
      {"type": "memory", "value": 70}
    ]
  },
  {
    "host": "server3",
    "timestamp": "2024-05-25 20:18:00",
    "events": [
      {"type": "cpu", "value": 50}
    ]
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=metrics&pipeline_name=one_to_many")
        .header("Content-Type", "application/json")
        .body(data_body2)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 6. verify total count: 3 (from first batch) + 2 + 1 = 6 rows
    validate_data(
        "test_pipeline_one_to_many_vrl_total_count",
        &client,
        "select count(*) from metrics",
        "[[6]]",
    )
    .await;

    // 7. verify rows per host
    validate_data(
        "test_pipeline_one_to_many_vrl_per_host",
        &client,
        "select host, count(*) as cnt from metrics group by host order by host",
        "[[\"server1\",3],[\"server2\",2],[\"server3\",1]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_2(storage_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(storage_type, "test_pipeline_2").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline = r#"
version: 2
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"

transform:
  - field: id1
    type: int32
    index: inverted
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/events/pipelines/root")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
[
  {
    "id1": "123",
    "id2": "2436",
    "time": "2024-05-25 20:16:37.217"
  }
]
"#;
    let res = client
        .post("/v1/events/logs?db=public&table=d_table&pipeline_name=root")
        .header("Content-Type", "application/json")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // CREATE TABLE IF NOT EXISTS "d_table" (
    //     "id1" INT NULL INVERTED INDEX,
    //     "time" TIMESTAMP(9) NOT NULL,
    //     "id2" STRING NULL,
    //     TIME INDEX ("time")
    //   )
    //   ENGINE=mito
    //   WITH(
    //     'comment' = 'Created on insertion',
    //     append_mode = 'true'
    //   )
    validate_data(
        "test_pipeline_2_schema",
        &client,
        "show create table d_table",
        "[[\"d_table\",\"CREATE TABLE IF NOT EXISTS \\\"d_table\\\" (\\n  \\\"id1\\\" INT NULL INVERTED INDEX,\\n  \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n  \\\"id2\\\" STRING NULL,\\n  TIME INDEX (\\\"time\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true'\\n)\"]]",
    )
    .await;

    validate_data(
        "test_pipeline_2_data",
        &client,
        "select * from d_table",
        "[[123,1716668197217000000,\"2436\"]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_identity_pipeline_with_flatten(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_identity_pipeline_with_flatten").await;

    let client = TestClient::new(app).await;
    let body = r#"{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java","custom_map":{"value_a":["a","b","c"],"value_b":"b"}}"#;

    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/json"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-params"),
                HeaderValue::from_static("flatten_json_object=true"),
            ),
        ],
        "/v1/ingest?table=logs&pipeline_name=greptime_identity",
        body.as_bytes().to_vec(),
        false,
    )
    .await;

    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["custom_map.value_a","String","","YES","","FIELD"],["custom_map.value_b","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"]]"#;
    validate_data(
        "test_identity_pipeline_with_flatten_desc_logs",
        &client,
        "desc logs",
        expected,
    )
    .await;

    let expected = "[[\"[\\\"a\\\",\\\"b\\\",\\\"c\\\"]\"]]";
    validate_data(
        "test_identity_pipeline_with_flatten_select_json",
        &client,
        "select `custom_map.value_a` from logs",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_identity_pipeline_with_custom_ts(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_identity_pipeline_with_custom_ts")
            .await;

    let client = TestClient::new(app).await;
    let body = r#"
    [{"__time__":1453809242,"__source__":"10.170.***.***", "__name__":"hello"},
    {"__time__":1453809252,"__source__":"10.170.***.***"}]
    "#;

    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/json"),
        )],
        "/v1/ingest?table=logs&pipeline_name=greptime_identity&custom_time_index=__time__;epoch;s",
        body.as_bytes().to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"[["__time__","TimestampSecond","PRI","NO","","TIMESTAMP"],["__name__","String","","YES","","FIELD"],["__source__","String","","YES","","FIELD"]]"#;
    validate_data(
        "test_identity_pipeline_with_custom_ts_desc_logs",
        &client,
        "desc logs",
        expected,
    )
    .await;

    let expected = r#"[[1453809242,"hello","10.170.***.***"],[1453809252,null,"10.170.***.***"]]"#;
    validate_data(
        "test_identity_pipeline_with_custom_ts_data",
        &client,
        "select * from logs",
        expected,
    )
    .await;

    // drop table
    let res = client.get("/v1/sql?sql=drop table logs").send().await;
    assert_eq!(res.status(), StatusCode::OK);

    let body = r#"
    [{"__time__":"2019-01-16 02:42:01+08:00","__source__":"10.170.***.***"},
    {"__time__":"2019-01-16 02:42:04+08:00","__source__":"10.170.***.***", "__name__":"hello"}]
    "#;

    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/json"),
        )],
        "/v1/ingest?table=logs&pipeline_name=greptime_identity&custom_time_index=__time__;datestr;%Y-%m-%d %H:%M:%S%z",
        body.as_bytes().to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"[["__time__","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__name__","String","","YES","","FIELD"]]"#;
    validate_data(
        "test_identity_pipeline_with_custom_ts_desc_logs",
        &client,
        "desc logs",
        expected,
    )
    .await;

    let expected = r#"[[1547577721000000000,"10.170.***.***",null],[1547577724000000000,"10.170.***.***","hello"]]"#;
    validate_data(
        "test_identity_pipeline_with_custom_ts_data",
        &client,
        "select * from logs",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_test_pipeline_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_pipeline_api").await;

    // handshake
    let client = TestClient::new(app).await;

    let pipeline_content = r#"
processors:
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

transform:
  - fields:
      - id1
      - id2
    type: int32
  - fields:
      - type
      - log
      - logger
    type: string
  - field: time
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(pipeline_content)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    let content = res.text().await;

    let content = serde_json::from_str(&content);
    assert!(content.is_ok());
    //  {"execution_time_ms":13,"pipelines":[{"name":"test","version":"2024-07-04 08:31:00.987136"}]}
    let content: Value = content.unwrap();

    let execution_time = content.get("execution_time_ms");
    assert!(execution_time.unwrap().is_number());
    let pipelines = content.get("pipelines");
    let pipelines = pipelines.unwrap().as_array().unwrap();
    assert_eq!(pipelines.len(), 1);
    let pipeline = pipelines.first().unwrap();
    assert_eq!(pipeline.get("name").unwrap(), "test");

    let dryrun_schema = json!([
        {
            "column_type": "FIELD",
            "data_type": "INT32",
            "fulltext": false,
            "name": "id1"
        },
        {
            "column_type": "FIELD",
            "data_type": "INT32",
            "fulltext": false,
            "name": "id2"
        },
        {
            "column_type": "FIELD",
            "data_type": "STRING",
            "fulltext": false,
            "name": "type"
        },
        {
            "column_type": "FIELD",
            "data_type": "STRING",
            "fulltext": false,
            "name": "log"
        },
        {
            "column_type": "FIELD",
            "data_type": "STRING",
            "fulltext": false,
            "name": "logger"
        },
        {
            "column_type": "TIMESTAMP",
            "data_type": "TIMESTAMP_NANOSECOND",
            "fulltext": false,
            "name": "time"
        }
    ]);
    let dryrun_rows = json!([
        [
            {
                "data_type": "INT32",
                "key": "id1",
                "semantic_type": "FIELD",
                "value": 2436
            },
            {
                "data_type": "INT32",
                "key": "id2",
                "semantic_type": "FIELD",
                "value": 2528
            },
            {
                "data_type": "STRING",
                "key": "type",
                "semantic_type": "FIELD",
                "value": "I"
            },
            {
                "data_type": "STRING",
                "key": "log",
                "semantic_type": "FIELD",
                "value": "ClusterAdapter:enter sendTextDataToCluster"
            },
            {
                "data_type": "STRING",
                "key": "logger",
                "semantic_type": "FIELD",
                "value": "INTERACT.MANAGER"
            },
            {
                "data_type": "TIMESTAMP_NANOSECOND",
                "key": "time",
                "semantic_type": "TIMESTAMP",
                "value": 1716668197217000000i64
            }
        ],
        [
            {
                "data_type": "INT32",
                "key": "id1",
                "semantic_type": "FIELD",
                "value": 1111
            },
            {
                "data_type": "INT32",
                "key": "id2",
                "semantic_type": "FIELD",
                "value": 2222
            },
            {
                "data_type": "STRING",
                "key": "type",
                "semantic_type": "FIELD",
                "value": "D"
            },
            {
                "data_type": "STRING",
                "key": "log",
                "semantic_type": "FIELD",
                "value": "ClusterAdapter:enter sendTextDataToCluster ggg"
            },
            {
                "data_type": "STRING",
                "key": "logger",
                "semantic_type": "FIELD",
                "value": "INTERACT.MANAGER"
            },
            {
                "data_type": "TIMESTAMP_NANOSECOND",
                "key": "time",
                "semantic_type": "TIMESTAMP",
                "value": 1716668198217000000i64
            }
        ]
    ]);
    {
        // test original api
        let data_body = r#"
        [
          {
            "id1": "2436",
            "id2": "2528",
            "logger": "INTERACT.MANAGER",
            "type": "I",
            "time": "2024-05-25 20:16:37.217",
            "log": "ClusterAdapter:enter sendTextDataToCluster"
          },
         {
            "id1": "1111",
            "id2": "2222",
            "logger": "INTERACT.MANAGER",
            "type": "D",
            "time": "2024-05-25 20:16:38.217",
            "log": "ClusterAdapter:enter sendTextDataToCluster ggg"
          }
        ]
        "#;
        let res = client
            .post("/v1/pipelines/_dryrun?pipeline_name=test")
            .header("Content-Type", "application/json")
            .body(data_body)
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);
        let body: Value = res.json().await;
        let schema = &body[0]["schema"];
        let rows = &body[0]["rows"];
        assert_eq!(schema, &dryrun_schema);
        assert_eq!(rows, &dryrun_rows);
    }
    {
        // test new api specify pipeline via pipeline_name
        let data = r#"[
                {
                "id1": "2436",
                "id2": "2528",
                "logger": "INTERACT.MANAGER",
                "type": "I",
                "time": "2024-05-25 20:16:37.217",
                "log": "ClusterAdapter:enter sendTextDataToCluster"
                },
                {
                "id1": "1111",
                "id2": "2222",
                "logger": "INTERACT.MANAGER",
                "type": "D",
                "time": "2024-05-25 20:16:38.217",
                "log": "ClusterAdapter:enter sendTextDataToCluster ggg"
                }
            ]"#;
        let body = json!({"pipeline_name":"test","data":data});
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);
        let body: Value = res.json().await;
        let schema = &body[0]["schema"];
        let rows = &body[0]["rows"];
        assert_eq!(schema, &dryrun_schema);
        assert_eq!(rows, &dryrun_rows);
    }
    {
        let pipeline_content_for_text = r#"
processors:
  - dissect:
      fields:
        - message
      patterns:
        - "%{id1} %{id2} %{logger} %{type} \"%{time}\" \"%{log}\""
  - date:
      field: time
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
      ignore_missing: true

transform:
  - fields:
      - id1
      - id2
    type: int32
  - fields:
      - type
      - log
      - logger
    type: string
  - field: time
    type: time
    index: timestamp
"#;

        // test new api specify pipeline via pipeline raw data
        let data = r#"[
            {
            "id1": "2436",
            "id2": "2528",
            "logger": "INTERACT.MANAGER",
            "type": "I",
            "time": "2024-05-25 20:16:37.217",
            "log": "ClusterAdapter:enter sendTextDataToCluster"
            },
            {
            "id1": "1111",
            "id2": "2222",
            "logger": "INTERACT.MANAGER",
            "type": "D",
            "time": "2024-05-25 20:16:38.217",
            "log": "ClusterAdapter:enter sendTextDataToCluster ggg"
            }
        ]"#;
        let mut body = json!({
        "data": data
        });
        body["pipeline"] = json!(pipeline_content);
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);
        let body: Value = res.json().await;
        let schema = &body[0]["schema"];
        let rows = &body[0]["rows"];
        assert_eq!(schema, &dryrun_schema);
        assert_eq!(rows, &dryrun_rows);
        let mut body_for_text = json!({
            "data": r#"2436 2528 INTERACT.MANAGER I "2024-05-25 20:16:37.217" "ClusterAdapter:enter sendTextDataToCluster"
1111 2222 INTERACT.MANAGER D "2024-05-25 20:16:38.217" "ClusterAdapter:enter sendTextDataToCluster ggg"
"#,
        });
        body_for_text["pipeline"] = json!(pipeline_content_for_text);
        body_for_text["data_type"] = json!("text/plain");
        let ndjson_content = r#"{"id1":"2436","id2":"2528","logger":"INTERACT.MANAGER","type":"I","time":"2024-05-25 20:16:37.217","log":"ClusterAdapter:enter sendTextDataToCluster"}
{"id1":"1111","id2":"2222","logger":"INTERACT.MANAGER","type":"D","time":"2024-05-25 20:16:38.217","log":"ClusterAdapter:enter sendTextDataToCluster ggg"}
"#;
        let body_for_ndjson = json!({
            "pipeline":pipeline_content,
            "data_type": "application/x-ndjson",
            "data": ndjson_content,
        });
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body_for_ndjson.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);
        let body: Value = res.json().await;
        let schema = &body[0]["schema"];
        let rows = &body[0]["rows"];
        assert_eq!(schema, &dryrun_schema);
        assert_eq!(rows, &dryrun_rows);

        body_for_text["data_type"] = json!("application/yaml");
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body_for_text.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::BAD_REQUEST);
        let body: Value = res.json().await;
        assert_eq!(
            body["error"],
            json!(
                "Invalid request parameter: invalid content type: application/yaml, expected: one of application/json, application/x-ndjson, text/plain"
            )
        );

        body_for_text["data_type"] = json!("application/json");
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body_for_text.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::BAD_REQUEST);
        let body: Value = res.json().await;
        assert_eq!(
            body["error"],
            json!(
                "Invalid request parameter: json format error, please check the date is valid JSON."
            )
        );

        body_for_text["data_type"] = json!("text/plain");
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body_for_text.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);
        let body: Value = res.json().await;

        let schema = &body[0]["schema"];
        let rows = &body[0]["rows"];
        assert_eq!(schema, &dryrun_schema);
        assert_eq!(rows, &dryrun_rows);
    }
    {
        // failback to old version api
        // not pipeline and pipeline_name in the body
        let body = json!({
        "data": [
            {
            "id1": "2436",
            "id2": "2528",
            "logger": "INTERACT.MANAGER",
            "type": "I",
            "time": "2024-05-25 20:16:37.217",
            "log": "ClusterAdapter:enter sendTextDataToCluster\\n"
            },
            {
            "id1": "1111",
            "id2": "2222",
            "logger": "INTERACT.MANAGER",
            "type": "D",
            "time": "2024-05-25 20:16:38.217",
            "log": "ClusterAdapter:enter sendTextDataToCluster ggg"
            }
        ]
        });
        let res = client
            .post("/v1/pipelines/_dryrun")
            .header("Content-Type", "application/json")
            .body(body.to_string())
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::BAD_REQUEST);
    }
    guard.remove_all().await;
}

pub async fn test_plain_text_ingestion(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_pipeline_api").await;

    // handshake
    let client = TestClient::new(app).await;

    let body = r#"
processors:
  - dissect:
      fields:
        - message
      patterns:
        - "%{+ts} %{+ts} %{content}"
  - date:
      fields:
        - ts
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"

transform:
  - fields:
      - content
    type: string
  - field: ts
    type: time
    index: timestamp
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    let content = res.text().await;

    let content = serde_json::from_str(&content);
    assert!(content.is_ok());
    //  {"execution_time_ms":13,"pipelines":[{"name":"test","version":"2024-07-04 08:31:00.987136"}]}
    let content: Value = content.unwrap();

    let version_str = content
        .get("pipelines")
        .unwrap()
        .as_array()
        .unwrap()
        .first()
        .unwrap()
        .get("version")
        .unwrap()
        .as_str()
        .unwrap()
        .to_string();
    assert!(!version_str.is_empty());

    // 2. write data
    let data_body = r#"
2024-05-25 20:16:37.217 hello
2024-05-25 20:16:37.218 hello world
"#;
    let res = client
        .post("/v1/ingest?db=public&table=logs1&pipeline_name=test")
        .header("Content-Type", "text/plain")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 3. select data
    let res = client.get("/v1/sql?sql=select * from logs1").send().await;
    assert_eq!(res.status(), StatusCode::OK);
    let resp = res.text().await;

    let v = get_rows_from_output(&resp);

    assert_eq!(
        v,
        r#"[["hello",1716668197217000000],["hello world",1716668197218000000]]"#,
    );

    guard.remove_all().await;
}

pub async fn test_pipeline_auto_transform(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_auto_transform").await;

    // handshake
    let client = TestClient::new(app).await;

    let body = r#"
processors:
  - dissect:
      fields:
        - message
      patterns:
        - "%{+ts} %{+ts} %{http_status_code} %{content}"
  - date:
      fields:
        - ts
      formats:
        - "%Y-%m-%d %H:%M:%S%.3f"
"#;

    // 1. create pipeline
    let res = client
        .post("/v1/pipelines/test")
        .header("Content-Type", "application/x-yaml")
        .body(body)
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK);

    // 2. write data
    let data_body = r#"
2024-05-25 20:16:37.217 404 hello
2024-05-25 20:16:37.218 200 hello world
"#;
    let res = client
        .post("/v1/ingest?db=public&table=logs1&pipeline_name=test")
        .header("Content-Type", "text/plain")
        .body(data_body)
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // 3. select data
    let expected = "[[1716668197217000000,\"hello\",\"404\",\"2024-05-25 20:16:37.217 404 hello\"],[1716668197218000000,\"hello world\",\"200\",\"2024-05-25 20:16:37.218 200 hello world\"]]";
    validate_data(
        "test_pipeline_auto_transform",
        &client,
        "select * from logs1",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_pipeline_auto_transform_with_select(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_pipeline_auto_transform_with_select")
            .await;

    // handshake
    let client = TestClient::new(app).await;
    let data_body = r#"
2024-05-25 20:16:37.217 404 hello
2024-05-25 20:16:37.218 200 hello world"#;

    // select include
    {
        let body = r#"
        processors:
          - dissect:
              fields:
                - message
              patterns:
                - "%{+ts} %{+ts} %{http_status_code} %{content}"
          - date:
              fields:
                - ts
              formats:
                - "%Y-%m-%d %H:%M:%S%.3f"
          - select:
              fields:
                - ts
                - http_status_code
        "#;

        // 1. create pipeline
        let res = client
            .post("/v1/pipelines/test")
            .header("Content-Type", "application/x-yaml")
            .body(body)
            .send()
            .await;

        assert_eq!(res.status(), StatusCode::OK);

        // 2. write data
        let res = client
            .post("/v1/ingest?db=public&table=logs1&pipeline_name=test")
            .header("Content-Type", "text/plain")
            .body(data_body)
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);

        // 3. select data
        let expected = "[[1716668197217000000,\"404\"],[1716668197218000000,\"200\"]]";
        validate_data(
            "test_pipeline_auto_transform_with_select",
            &client,
            "select * from logs1",
            expected,
        )
        .await;
    }

    // select include rename
    {
        let body = r#"
        processors:
          - dissect:
              fields:
                - message
              patterns:
                - "%{+ts} %{+ts} %{http_status_code} %{content}"
          - date:
              fields:
                - ts
              formats:
                - "%Y-%m-%d %H:%M:%S%.3f"
          - select:
              fields:
                - ts
                - key: http_status_code
                  rename_to: s_code
        "#;

        // 1. create pipeline
        let res = client
            .post("/v1/pipelines/test2")
            .header("Content-Type", "application/x-yaml")
            .body(body)
            .send()
            .await;

        assert_eq!(res.status(), StatusCode::OK);

        // 2. write data
        let res = client
            .post("/v1/ingest?db=public&table=logs2&pipeline_name=test2")
            .header("Content-Type", "text/plain")
            .body(data_body)
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);

        // 3. check schema
        let expected = "[[\"ts\",\"TimestampNanosecond\",\"PRI\",\"NO\",\"\",\"TIMESTAMP\"],[\"s_code\",\"String\",\"\",\"YES\",\"\",\"FIELD\"]]";
        validate_data(
            "test_pipeline_auto_transform_with_select_rename",
            &client,
            "desc table logs2",
            expected,
        )
        .await;

        // 4. check data
        let expected = "[[1716668197217000000,\"404\"],[1716668197218000000,\"200\"]]";
        validate_data(
            "test_pipeline_auto_transform_with_select_rename",
            &client,
            "select * from logs2",
            expected,
        )
        .await;
    }

    // select exclude
    {
        let body = r#"
            processors:
              - dissect:
                  fields:
                    - message
                  patterns:
                    - "%{+ts} %{+ts} %{http_status_code} %{content}"
              - date:
                  fields:
                    - ts
                  formats:
                    - "%Y-%m-%d %H:%M:%S%.3f"
              - select:
                  type: exclude
                  fields:
                    - http_status_code
            "#;

        // 1. create pipeline
        let res = client
            .post("/v1/pipelines/test3")
            .header("Content-Type", "application/x-yaml")
            .body(body)
            .send()
            .await;

        assert_eq!(res.status(), StatusCode::OK);

        // 2. write data
        let res = client
            .post("/v1/ingest?db=public&table=logs3&pipeline_name=test3")
            .header("Content-Type", "text/plain")
            .body(data_body)
            .send()
            .await;
        assert_eq!(res.status(), StatusCode::OK);

        // 3. check schema
        let expected = "[[\"ts\",\"TimestampNanosecond\",\"PRI\",\"NO\",\"\",\"TIMESTAMP\"],[\"content\",\"String\",\"\",\"YES\",\"\",\"FIELD\"],[\"message\",\"String\",\"\",\"YES\",\"\",\"FIELD\"]]";
        validate_data(
            "test_pipeline_auto_transform_with_select_rename",
            &client,
            "desc table logs3",
            expected,
        )
        .await;

        // 4. check data
        let expected = "[[1716668197217000000,\"hello\",\"2024-05-25 20:16:37.217 404 hello\"],[1716668197218000000,\"hello world\",\"2024-05-25 20:16:37.218 200 hello world\"]]";
        validate_data(
            "test_pipeline_auto_transform_with_select_rename",
            &client,
            "select * from logs3",
            expected,
        )
        .await;
    }

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_otlp_metrics_skip_wal(distributed: bool) {
    let mut cases = Vec::new();
    let metrics: ExportMetricsServiceRequest = serde_json::from_value(json!({
        "resourceMetrics":[{"scopeMetrics":[{"metrics":[{"name":"wal_otlp", "gauge":{"dataPoints":[{"timeUnixNano":"1700000000000000000","asDouble":1.0}]}}]}]}]
    })).unwrap();
    cases.push(HttpWalCase::new(
        "OTLP metrics",
        "/v1/otlp/v1/metrics",
        "application/x-protobuf",
        metrics.encode_to_vec(),
    ));
    check_http_skip_wal("otlp_metrics", &cases, distributed).await;
}

pub async fn test_otlp_metrics_new(store_type: StorageType) {
    // init
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_otlp_metrics_new").await;

    let content = r#"
{"resourceMetrics":[{"resource":{"attributes":[{"key":"host.arch","value":{"stringValue":"arm64"}},{"key":"os.type","value":{"stringValue":"darwin"}},{"key":"os.version","value":{"stringValue":"25.0.0"}},{"key":"service.name","value":{"stringValue":"claude-code"}},{"key":"service.version","value":{"stringValue":"1.0.62"}}],"droppedAttributesCount":0},"scopeMetrics":[{"scope":{"name":"com.anthropic.claude_code","version":"1.0.62","attributes":[],"droppedAttributesCount":0},"metrics":[{"name":"claude_code.cost.usage","description":"Cost of the Claude Code session","unit":"USD","metadata":[],"sum":{"dataPoints":[{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"model","value":{"stringValue":"claude-3-5-haiku-20241022"}}],"startTimeUnixNano":"1753780502453000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":0.0052544},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"model","value":{"stringValue":"claude-sonnet-4-20250514"}}],"startTimeUnixNano":"1753780513420000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":2.244618}],"aggregationTemporality":1,"isMonotonic":true}},{"name":"claude_code.token.usage","description":"Number of tokens used","unit":"tokens","metadata":[],"sum":{"dataPoints":[{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"input"}},{"key":"model","value":{"stringValue":"claude-3-5-haiku-20241022"}}],"startTimeUnixNano":"1753780502453000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":6208.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"output"}},{"key":"model","value":{"stringValue":"claude-3-5-haiku-20241022"}}],"startTimeUnixNano":"1753780502453000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":72.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"cacheRead"}},{"key":"model","value":{"stringValue":"claude-3-5-haiku-20241022"}}],"startTimeUnixNano":"1753780502453000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":0.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"cacheCreation"}},{"key":"model","value":{"stringValue":"claude-3-5-haiku-20241022"}}],"startTimeUnixNano":"1753780502453000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":0.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"input"}},{"key":"model","value":{"stringValue":"claude-sonnet-4-20250514"}}],"startTimeUnixNano":"1753780513420000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":743056.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"output"}},{"key":"model","value":{"stringValue":"claude-sonnet-4-20250514"}}],"startTimeUnixNano":"1753780513420000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":1030.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"cacheRead"}},{"key":"model","value":{"stringValue":"claude-sonnet-4-20250514"}}],"startTimeUnixNano":"1753780513420000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":0.0},{"attributes":[{"key":"user.id","value":{"stringValue":"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4"}},{"key":"session.id","value":{"stringValue":"736525A3-F5D4-496B-933E-827AF23A5B97"}},{"key":"terminal.type","value":{"stringValue":"ghostty"}},{"key":"type","value":{"stringValue":"cacheCreation"}},{"key":"model","value":{"stringValue":"claude-sonnet-4-20250514"}}],"startTimeUnixNano":"1753780513420000000","timeUnixNano":"1753780559836000000","exemplars":[],"flags":0,"asDouble":0.0}],"aggregationTemporality":1,"isMonotonic":true}}],"schemaUrl":""}],"schemaUrl":""}]}
    "#;

    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let body = req.encode_to_vec();

    // handshake
    let client = TestClient::new(app).await;

    // write metrics data
    // with scope attrs and all resource attrs
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-promote-scope-attrs"),
                HeaderValue::from_static("true"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-promote-all-resource-attrs"),
                HeaderValue::from_static("true"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-ignore-resource-attrs"),
                HeaderValue::from_static("os.type"),
            ),
        ],
        "/v1/otlp/v1/metrics",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = "[[\"claude_code_cost_usage_USD_total\"],[\"claude_code_token_usage_tokens_total\"],[\"demo\"],[\"greptime_otel_resource_info\"],[\"greptime_physical_table\"],[\"numbers\"]]";
    validate_data("otlp_metrics_all_tables", &client, "show tables;", expected).await;

    // Metric-engine logical table carries the semantic identity. Match substrings
    // because extra_options ordering is not stable.
    validate_data(
        "otlp_metrics_semantic_identity",
        &client,
        "select count(*) from information_schema.tables where table_name = 'claude_code_cost_usage_USD_total' \
         and create_options like '%greptime.semantic.signal_type=metric%' \
         and create_options like '%greptime.semantic.source=opentelemetry%';",
        "[[1]]",
    )
    .await;
    // OTLP metric type is declared, so Phase 2 stamps `metadata_quality=declared`.
    validate_data(
        "otlp_metrics_metadata_quality_declared",
        &client,
        "select count(*) from information_schema.tables where table_name = 'claude_code_cost_usage_USD_total' \
         and create_options like '%greptime.semantic.metric.metadata_quality=declared%';",
        "[[1]]",
    )
    .await;

    // CREATE TABLE IF NOT EXISTS "claude_code_cost_usage_USD_total" (
    //   "greptime_timestamp" TIMESTAMP(3) NOT NULL,
    //   "greptime_value" DOUBLE NULL,
    //   "host_arch" STRING NULL,
    //   "job" STRING NULL,
    //   "model" STRING NULL,
    //   "os_version" STRING NULL,
    //   "otel_scope_name" STRING NULL,
    //   "otel_scope_schema_url" STRING NULL,
    //   "otel_scope_version" STRING NULL,
    //   "otlp_aggregation_temporality" STRING NULL,
    //   "service_name" STRING NULL,
    //   "service_version" STRING NULL,
    //   "session_id" STRING NULL,
    //   "terminal_type" STRING NULL,
    //   "user_id" STRING NULL,
    //   TIME INDEX ("greptime_timestamp"),
    //   PRIMARY KEY ("host_arch", "job", "model", "os_version", "otel_scope_name", "otel_scope_schema_url", "otel_scope_version", "otlp_aggregation_temporality", "service_name", "service_version", "session_id", "terminal_type", "user_id")
    //   )
    // ENGINE=metric
    // WITH(
    //   'comment' = 'Created on insertion',
    //   on_physical_table = 'greptime_physical_table',
    //   otlp_metric_compat = 'prom'
    // )
    let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"greptime_value\\\" DOUBLE NULL,\\n  \\\"host_arch\\\" STRING NULL,\\n  \\\"job\\\" STRING NULL,\\n  \\\"model\\\" STRING NULL,\\n  \\\"os_version\\\" STRING NULL,\\n  \\\"otel_scope_name\\\" STRING NULL,\\n  \\\"otel_scope_schema_url\\\" STRING NULL,\\n  \\\"otel_scope_version\\\" STRING NULL,\\n  \\\"otlp_aggregation_temporality\\\" STRING NULL,\\n  \\\"service_name\\\" STRING NULL,\\n  \\\"service_version\\\" STRING NULL,\\n  \\\"session_id\\\" STRING NULL,\\n  \\\"terminal_type\\\" STRING NULL,\\n  \\\"user_id\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\"),\\n  PRIMARY KEY (\\\"host_arch\\\", \\\"job\\\", \\\"model\\\", \\\"os_version\\\", \\\"otel_scope_name\\\", \\\"otel_scope_schema_url\\\", \\\"otel_scope_version\\\", \\\"otlp_aggregation_temporality\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n  'comment' = 'Created on insertion',\\n  'greptime.semantic.metric.metadata_quality' = 'declared',\\n  'greptime.semantic.metric.original_name' = 'claude_code.cost.usage',\\n  'greptime.semantic.metric.temporality' = 'delta',\\n  'greptime.semantic.metric.type' = 'counter',\\n  'greptime.semantic.metric.unit' = 'USD',\\n  'greptime.semantic.signal_type' = 'metric',\\n  'greptime.semantic.source' = 'opentelemetry',\\n  on_physical_table = 'greptime_physical_table',\\n  otlp_metric_compat = 'prom'\\n)\"]]";
    validate_data(
        "otlp_metrics_all_show_create_table",
        &client,
        "show create table `claude_code_cost_usage_USD_total`;",
        expected,
    )
    .await;

    // select metrics data
    let expected = "[[1753780559836,2.244618,\"arm64\",\"claude-code\",\"claude-sonnet-4-20250514\",\"25.0.0\",\"com.anthropic.claude_code\",\"\",\"1.0.62\",\"delta\",\"claude-code\",\"1.0.62\",\"736525A3-F5D4-496B-933E-827AF23A5B97\",\"ghostty\",\"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4\"],[1753780559836,0.0052544,\"arm64\",\"claude-code\",\"claude-3-5-haiku-20241022\",\"25.0.0\",\"com.anthropic.claude_code\",\"\",\"1.0.62\",\"delta\",\"claude-code\",\"1.0.62\",\"736525A3-F5D4-496B-933E-827AF23A5B97\",\"ghostty\",\"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4\"]]";
    validate_data(
        "otlp_metrics_all_select",
        &client,
        "select * from `claude_code_cost_usage_USD_total` order by model desc;",
        expected,
    )
    .await;

    // A metric-engine logical table stores its rows in the shared physical
    // table, so reading them back from SST files goes through a different
    // column mapping than the memtable read does.
    flush_table(&client, "claude_code_cost_usage_USD_total").await;
    validate_data(
        "otlp_metrics_all_select_after_flush",
        &client,
        "select * from `claude_code_cost_usage_USD_total` order by model desc;",
        expected,
    )
    .await;

    // The synthesized resource descriptor: a plain mito info table keyed by
    // the raw OTel attribute keys (service.name is the only allowlisted
    // resource attribute in this payload), independent of the promote/ignore
    // headers. The timestamp is the newest data-point time.
    validate_data(
        "otlp_metrics_resource_info",
        &client,
        "select job, \"service.name\", greptime_value, greptime_timestamp from greptime_otel_resource_info;",
        "[[\"claude-code\",\"claude-code\",1.0,1753780559836]]",
    )
    .await;
    // All four ownership markers must land on auto-create: the descriptor
    // write path refuses tables that miss any of them, so a missing stamp
    // here would silently stop every follow-up descriptor write.
    validate_data(
        "otlp_metrics_resource_info_options",
        &client,
        "select count(*) from information_schema.tables where table_name = 'greptime_otel_resource_info' \
         and engine = 'mito' \
         and create_options like '%greptime.semantic.metric.type=info%' \
         and create_options like '%greptime.semantic.metric.metadata_quality=declared%' \
         and create_options like '%greptime.semantic.signal_type=metric%' \
         and create_options like '%greptime.semantic.source=opentelemetry%';",
        "[[1]]",
    )
    .await;

    // drop table
    let res = client
        .get("/v1/sql?sql=drop table `claude_code_cost_usage_USD_total`;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?sql=drop table claude_code_token_usage_tokens_total;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?sql=drop table greptime_otel_resource_info;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // write metrics data
    // with scope attrs
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-promote-resource-attrs"),
                HeaderValue::from_static("os.type;os.version"),
            ),
        ],
        "/v1/otlp/v1/metrics",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // CREATE TABLE IF NOT EXISTS "claude_code_cost_usage_USD_total" (
    //     "greptime_timestamp" TIMESTAMP(3) NOT NULL,
    //     "greptime_value" DOUBLE NULL,
    //     "job" STRING NULL,
    //     "model" STRING NULL,
    //     "os_type" STRING NULL,
    //     "os_version" STRING NULL,
    //     "otlp_aggregation_temporality" STRING NULL,
    //     "service_name" STRING NULL,
    //     "service_version" STRING NULL,
    //     "session_id" STRING NULL,
    //     "terminal_type" STRING NULL,
    //     "user_id" STRING NULL,
    //     TIME INDEX ("greptime_timestamp"),
    //     PRIMARY KEY ("job", "model", "os_type", "os_version", "otlp_aggregation_temporality", "service_name", "service_version", "session_id", "terminal_type", "user_id")
    //     )
    //   ENGINE=metric
    //   WITH(
    //     'comment' = 'Created on insertion',
    //     on_physical_table = 'greptime_physical_table',
    //     otlp_metric_compat = 'prom'
    //   )
    let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"greptime_value\\\" DOUBLE NULL,\\n  \\\"job\\\" STRING NULL,\\n  \\\"model\\\" STRING NULL,\\n  \\\"os_type\\\" STRING NULL,\\n  \\\"os_version\\\" STRING NULL,\\n  \\\"otlp_aggregation_temporality\\\" STRING NULL,\\n  \\\"service_name\\\" STRING NULL,\\n  \\\"service_version\\\" STRING NULL,\\n  \\\"session_id\\\" STRING NULL,\\n  \\\"terminal_type\\\" STRING NULL,\\n  \\\"user_id\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\"),\\n  PRIMARY KEY (\\\"job\\\", \\\"model\\\", \\\"os_type\\\", \\\"os_version\\\", \\\"otlp_aggregation_temporality\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n  'comment' = 'Created on insertion',\\n  'greptime.semantic.metric.metadata_quality' = 'declared',\\n  'greptime.semantic.metric.original_name' = 'claude_code.cost.usage',\\n  'greptime.semantic.metric.temporality' = 'delta',\\n  'greptime.semantic.metric.type' = 'counter',\\n  'greptime.semantic.metric.unit' = 'USD',\\n  'greptime.semantic.signal_type' = 'metric',\\n  'greptime.semantic.source' = 'opentelemetry',\\n  on_physical_table = 'greptime_physical_table',\\n  otlp_metric_compat = 'prom'\\n)\"]]";
    validate_data(
        "otlp_metrics_show_create_table",
        &client,
        "show create table `claude_code_cost_usage_USD_total`;",
        expected,
    )
    .await;

    // select metrics data
    let expected = "[[1753780559836,2.244618,\"claude-code\",\"claude-sonnet-4-20250514\",\"darwin\",\"25.0.0\",\"delta\",\"claude-code\",\"1.0.62\",\"736525A3-F5D4-496B-933E-827AF23A5B97\",\"ghostty\",\"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4\"],[1753780559836,0.0052544,\"claude-code\",\"claude-3-5-haiku-20241022\",\"darwin\",\"25.0.0\",\"delta\",\"claude-code\",\"1.0.62\",\"736525A3-F5D4-496B-933E-827AF23A5B97\",\"ghostty\",\"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4\"]]";
    validate_data(
        "otlp_metrics_select",
        &client,
        "select * from `claude_code_cost_usage_USD_total` order by model desc;",
        expected,
    )
    .await;

    // drop table
    let res = client
        .get("/v1/sql?sql=drop table `claude_code_cost_usage_USD_total`;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?sql=drop table claude_code_token_usage_tokens_total;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // write metrics data
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )],
        "/v1/otlp/v1/metrics",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // CREATE TABLE IF NOT EXISTS "claude_code_cost_usage_USD_total" (
    //     "greptime_timestamp" TIMESTAMP(3) NOT NULL,
    //     "greptime_value" DOUBLE NULL,
    //     "job" STRING NULL,
    //     "model" STRING NULL,
    //     "otlp_aggregation_temporality" STRING NULL,
    //     "service_name" STRING NULL,
    //     "service_version" STRING NULL,
    //     "session_id" STRING NULL,
    //     "terminal_type" STRING NULL,
    //     "user_id" STRING NULL,
    //     TIME INDEX ("greptime_timestamp"),
    //     PRIMARY KEY ("job", "model", "otlp_aggregation_temporality", "service_name", "service_version", "session_id", "terminal_type", "user_id")
    //     )
    //   ENGINE=metric
    //   WITH(
    //    'comment' = 'Created on insertion',
    //     on_physical_table = 'greptime_physical_table',
    //     otlp_metric_compat = 'prom'
    //   )
    let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"greptime_value\\\" DOUBLE NULL,\\n  \\\"job\\\" STRING NULL,\\n  \\\"model\\\" STRING NULL,\\n  \\\"otlp_aggregation_temporality\\\" STRING NULL,\\n  \\\"service_name\\\" STRING NULL,\\n  \\\"service_version\\\" STRING NULL,\\n  \\\"session_id\\\" STRING NULL,\\n  \\\"terminal_type\\\" STRING NULL,\\n  \\\"user_id\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\"),\\n  PRIMARY KEY (\\\"job\\\", \\\"model\\\", \\\"otlp_aggregation_temporality\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n  'comment' = 'Created on insertion',\\n  'greptime.semantic.metric.metadata_quality' = 'declared',\\n  'greptime.semantic.metric.original_name' = 'claude_code.cost.usage',\\n  'greptime.semantic.metric.temporality' = 'delta',\\n  'greptime.semantic.metric.type' = 'counter',\\n  'greptime.semantic.metric.unit' = 'USD',\\n  'greptime.semantic.signal_type' = 'metric',\\n  'greptime.semantic.source' = 'opentelemetry',\\n  on_physical_table = 'greptime_physical_table',\\n  otlp_metric_compat = 'prom'\\n)\"]]";
    validate_data(
        "otlp_metrics_show_create_table_none",
        &client,
        "show create table `claude_code_cost_usage_USD_total`;",
        expected,
    )
    .await;

    // select metrics data
    let expected = "[[1753780559836,2.244618,\"claude-code\",\"claude-sonnet-4-20250514\",\"delta\",\"claude-code\",\"1.0.62\",\"736525A3-F5D4-496B-933E-827AF23A5B97\",\"ghostty\",\"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4\"],[1753780559836,0.0052544,\"claude-code\",\"claude-3-5-haiku-20241022\",\"delta\",\"claude-code\",\"1.0.62\",\"736525A3-F5D4-496B-933E-827AF23A5B97\",\"ghostty\",\"6DA02FD9-B5C5-4E61-9355-9FE8EC9A0CF4\"]]";
    validate_data(
        "otlp_metrics_select_none",
        &client,
        "select * from `claude_code_cost_usage_USD_total` order by model desc;",
        expected,
    )
    .await;

    // the descriptor was auto-created again after the earlier drop
    validate_data(
        "otlp_metrics_resource_info_recreated",
        &client,
        "select count(*) from information_schema.tables \
         where table_name = 'greptime_otel_resource_info' and engine = 'mito';",
        "[[1]]",
    )
    .await;

    // drop table
    let res = client
        .get("/v1/sql?sql=drop table `claude_code_cost_usage_USD_total`;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?sql=drop table claude_code_token_usage_tokens_total;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);
    let res = client
        .get("/v1/sql?sql=drop table greptime_otel_resource_info;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // job composition end-to-end: service.namespace folds into "<ns>/<name>"
    // on both the descriptor and the metric tables
    let content = r#"
{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"api"}},{"key":"service.namespace","value":{"stringValue":"shop"}},{"key":"host.id","value":{"stringValue":"h-1"}}]},"scopeMetrics":[{"scope":{"name":"s"},"metrics":[{"name":"ns_gauge","gauge":{"dataPoints":[{"timeUnixNano":"1753780559836000000","asDouble":1.0}]}}]}]}]}
    "#;
    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )],
        "/v1/otlp/v1/metrics",
        req.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    validate_data(
        "otlp_metrics_namespace_job_descriptor",
        &client,
        "select job, \"service.name\", \"service.namespace\", \"host.id\" from greptime_otel_resource_info;",
        "[[\"shop/api\",\"api\",\"shop\",\"h-1\"]]",
    )
    .await;
    validate_data(
        "otlp_metrics_namespace_job_metric_table",
        &client,
        "select job from ns_gauge;",
        "[[\"shop/api\"]]",
    )
    .await;

    let content = r#"
{"resourceMetrics":[{"scopeMetrics":[{"metrics":[{"name":"http.stale.sum","sum":{"dataPoints":[{"timeUnixNano":"1753780559836000000","flags":1,"asDouble":99.0}],"aggregationTemporality":1,"isMonotonic":true}},{"name":"http.stale.histogram","histogram":{"dataPoints":[{"timeUnixNano":"1753780559835000000","count":"3","sum":4.0,"bucketCounts":["1","1","1"],"explicitBounds":[1.0,2.0]},{"timeUnixNano":"1753780559836000000","flags":1,"count":"99","sum":99.0,"bucketCounts":["99"],"explicitBounds":[1.0,2.0]}],"aggregationTemporality":1}}]}]}]}
    "#;
    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )],
        "/v1/otlp/v1/metrics",
        req.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let body = ExportMetricsServiceResponse::decode(res.bytes().await).unwrap();
    assert!(body.partial_success.is_none());
    validate_data(
        "otlp_http_stale_sum_identity",
        &client,
        "select otlp_aggregation_temporality, count(*) from http_stale_sum_total \
         group by otlp_aggregation_temporality;",
        "[[\"delta\",1]]",
    )
    .await;
    for (table, expected) in [
        ("http_stale_histogram_bucket", "[[\"delta\",6]]"),
        ("http_stale_histogram_sum", "[[\"delta\",2]]"),
        ("http_stale_histogram_count", "[[\"delta\",2]]"),
    ] {
        validate_data(
            "otlp_http_stale_histogram_identity",
            &client,
            &format!(
                "select otlp_aggregation_temporality, count(*) from {table} \
                 group by otlp_aggregation_temporality;"
            ),
            expected,
        )
        .await;
    }

    let content = r#"
{"resourceMetrics":[{"scopeMetrics":[{"metrics":[{"name":"reserved_label_gauge","gauge":{"dataPoints":[{"timeUnixNano":"1753780559836000000","asDouble":1.0,"attributes":[{"key":"otlp_aggregation_temporality","value":{"stringValue":"user"}}]}]}}]}]}]}
    "#;
    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )],
        "/v1/otlp/v1/metrics",
        req.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    let status = GoogleRpcStatus::decode(res.bytes().await.as_ref()).unwrap();
    assert_eq!(tonic::Code::InvalidArgument as i32, status.code);
    assert!(status.message.contains("reserved label"));
    validate_data(
        "otlp_metrics_reserved_label_stores_nothing",
        &client,
        "select count(*) from information_schema.tables where table_name = 'reserved_label_gauge';",
        "[[0]]",
    )
    .await;

    let content = r#"
{"resourceMetrics":[{"scopeMetrics":[{"metrics":[{"name":"malformed_delta_histogram","histogram":{"dataPoints":[{"timeUnixNano":"1753780559836000000","count":"1","sum":1.0,"bucketCounts":["1"],"explicitBounds":[1.0,2.0]}],"aggregationTemporality":1}}]}]}]}
    "#;
    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )],
        "/v1/otlp/v1/metrics",
        req.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    let status = GoogleRpcStatus::decode(res.bytes().await.as_ref()).unwrap();
    assert_eq!(tonic::Code::InvalidArgument as i32, status.code);
    assert!(status.message.contains("bucket_counts length"));
    validate_data(
        "otlp_metrics_malformed_delta_histogram_stores_nothing",
        &client,
        "select count(*) from information_schema.tables where table_name in \
         ('malformed_delta_histogram_bucket', 'malformed_delta_histogram_sum', \
          'malformed_delta_histogram_count');",
        "[[0]]",
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_otlp_exponential_histogram(store_type: StorageType) {
    use opentelemetry_proto::tonic::metrics::v1::{
        AggregationTemporality, ExponentialHistogram, ExponentialHistogramDataPoint, Metric,
        ResourceMetrics, ScopeMetrics, exponential_histogram_data_point, metric,
    };

    common_telemetry::init_default_ut_logging();
    let req = ExportMetricsServiceRequest {
        resource_metrics: vec![ResourceMetrics {
            scope_metrics: vec![ScopeMetrics {
                metrics: vec![Metric {
                    name: "otlp.exponential.latency".to_string(),
                    data: Some(metric::Data::ExponentialHistogram(ExponentialHistogram {
                        data_points: vec![
                            ExponentialHistogramDataPoint {
                                start_time_unix_nano: 1_000_000_000,
                                time_unix_nano: 3_000_000_000,
                                count: 4,
                                sum: Some(8.0),
                                scale: 0,
                                zero_count: 1,
                                positive: Some(exponential_histogram_data_point::Buckets {
                                    offset: -1,
                                    bucket_counts: vec![1, 2],
                                }),
                                ..Default::default()
                            },
                            ExponentialHistogramDataPoint {
                                start_time_unix_nano: 1_000_000_000,
                                time_unix_nano: 4_000_000_000,
                                scale: -5,
                                ..Default::default()
                            },
                        ],
                        aggregation_temporality: AggregationTemporality::Cumulative as i32,
                    })),
                    ..Default::default()
                }],
                ..Default::default()
            }],
            ..Default::default()
        }],
    };
    let body = req.encode_to_vec();
    let headers = || {
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )]
    };

    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_otlp_exponential_histogram").await;
    let client = TestClient::new(app).await;
    let res = send_req(&client, headers(), "/v1/otlp/v1/metrics", body, false).await;
    assert_eq!(StatusCode::OK, res.status());
    let response = ExportMetricsServiceResponse::decode(res.bytes().await).unwrap();
    let partial_success = response.partial_success.unwrap();
    assert_eq!(1, partial_success.rejected_data_points);
    assert!(partial_success.error_message.contains("scale -5"));

    validate_data(
        "otlp_exponential_histogram_row",
        &client,
        "select greptime_timestamp, greptime_native_histogram from otlp_exponential_latency;",
        "[[3000,{\"count_f64\":null,\"count_i64\":4,\"custom_values\":[],\"negative_buckets_f64\":[],\"negative_buckets_i64\":[],\"negative_span_lengths\":[],\"negative_span_offsets\":[],\"positive_buckets_f64\":[],\"positive_buckets_i64\":[1,2],\"positive_span_lengths\":[2],\"positive_span_offsets\":[0],\"reset_hint\":0,\"schema\":0,\"start_timestamp\":1000,\"sum\":8.0,\"zero_count_f64\":null,\"zero_count_i64\":1,\"zero_threshold\":0.0}]]",
    )
    .await;
    validate_data(
        "otlp_exponential_histogram_metadata",
        &client,
        "select count(*) from information_schema.tables where table_name = 'otlp_exponential_latency' and create_options like '%greptime.semantic.metric.type=histogram%' and create_options like '%greptime.semantic.metric.temporality=cumulative%';",
        "[[1]]",
    )
    .await;

    for query in [
        "histogram_count(otlp_exponential_latency)",
        "histogram_sum(otlp_exponential_latency)",
        "histogram_quantile(0.5,otlp_exponential_latency)",
        "histogram_count(rate(otlp_exponential_latency[5s]))",
    ] {
        let res = client
            .get(&format!(
                "/v1/prometheus/api/v1/query?query={}&time=3",
                encode(query)
            ))
            .send()
            .await;
        assert_eq!(StatusCode::OK, res.status(), "query: {query}");
        let body = res.text().await;
        assert!(body.contains("otlp_exponential_latency"), "{query}: {body}");
    }

    guard.remove_all().await;
}

pub async fn test_otlp_metric_translation_strategies(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_otlp_metric_translation_strategies")
            .await;

    let content = r#"
{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"strategy-service"}},{"key":"resource.attr","value":{"stringValue":"resource-a"}}]},"scopeMetrics":[{"scope":{"name":"test.scope","version":"1.0.0","attributes":[{"key":"scope.attr","value":{"stringValue":"scope-a"}}]},"schemaUrl":"https://example.com/schema","metrics":[{"name":"otel.strategy.duration","description":"strategy test metric","unit":"ms","sum":{"dataPoints":[{"attributes":[{"key":"data.point.label","value":{"stringValue":"duration-a"}}],"startTimeUnixNano":"1000000","timeUnixNano":"2000000","asDouble":42.0}],"aggregationTemporality":2,"isMonotonic":true}}]}]}]}
    "#;

    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let body = req.encode_to_vec();
    let client = TestClient::new(app).await;

    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-translation-strategy"),
                HeaderValue::from_static("NoTranslation"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-promote-scope-attrs"),
                HeaderValue::from_static("true"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-promote-all-resource-attrs"),
                HeaderValue::from_static("true"),
            ),
        ],
        "/v1/otlp/v1/metrics",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = "[[\"otel.strategy.duration\"]]";
    validate_data(
        "otlp_metric_translation_strategy_tables",
        &client,
        "select table_name from information_schema.tables where table_schema = 'public' and table_name = 'otel.strategy.duration';",
        expected,
    )
    .await;

    let expected = "[[42.0,\"duration-a\",\"scope-a\",\"resource-a\",\"strategy-service\"]]";
    validate_data(
        "otlp_metric_translation_strategy_ingestion",
        &client,
        "select greptime_value, \"data.point.label\", \"otel_scope_scope.attr\", \"resource.attr\", job from \"otel.strategy.duration\";",
        expected,
    )
    .await;

    // the descriptor's columns are the fixed raw keys regardless of the
    // translation strategy, and only allowlisted attributes are projected
    validate_data(
        "otlp_metric_translation_strategy_resource_info",
        &client,
        "select job, \"service.name\" from greptime_otel_resource_info;",
        "[[\"strategy-service\",\"strategy-service\"]]",
    )
    .await;
    validate_data(
        "otlp_metric_translation_strategy_resource_info_allowlist",
        &client,
        "select count(*) from information_schema.columns \
         where table_name = 'greptime_otel_resource_info' and column_name = 'resource.attr';",
        "[[0]]",
    )
    .await;

    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-otlp-metric-translation-strategy"),
                HeaderValue::from_static("no_translation"),
            ),
        ],
        "/v1/otlp/v1/metrics",
        body,
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());

    guard.remove_all().await;
}

pub async fn test_otlp_metrics_resource_info_conflicts(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_otlp_metrics_resource_info_conflicts")
            .await;
    let client = TestClient::new(app).await;

    let content_type = || {
        (
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )
    };

    // A metric named like the descriptor wins the table name: it goes through
    // the normal metric-engine path and no descriptor is synthesized.
    let content = r#"
{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"api"}},{"key":"host.id","value":{"stringValue":"h-1"}}]},"scopeMetrics":[{"scope":{"name":"s"},"metrics":[{"name":"greptime_otel_resource_info","gauge":{"dataPoints":[{"timeUnixNano":"1753780559836000000","asDouble":1.0}]}}]}]}]}
    "#;
    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let res = send_req(
        &client,
        vec![content_type()],
        "/v1/otlp/v1/metrics",
        req.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let body = ExportMetricsServiceResponse::decode(res.bytes().await).unwrap();
    assert!(body.partial_success.is_none());
    validate_data(
        "otlp_metrics_name_collision_engine",
        &client,
        "select count(*) from information_schema.tables \
         where table_name = 'greptime_otel_resource_info' and engine = 'metric';",
        "[[1]]",
    )
    .await;
    let res = execute_sql(&client, "drop table greptime_otel_resource_info;").await;
    assert_eq!(res.status(), StatusCode::OK);

    // A pre-existing table with an incompatible schema fails the descriptor
    // write, which degrades to an OTLP partial-success warning while the
    // metric data itself is accepted.
    let res = execute_sql(
        &client,
        "create table greptime_otel_resource_info (ts timestamp time index, job double, greptime_value double);",
    )
    .await;
    assert_eq!(res.status(), StatusCode::OK);

    let content = r#"
{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"api"}}]},"scopeMetrics":[{"scope":{"name":"s"},"metrics":[{"name":"conflict_gauge","gauge":{"dataPoints":[{"timeUnixNano":"1753780559836000000","asDouble":7.0}]}}]}]}]}
    "#;
    let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
    let res = send_req(
        &client,
        vec![content_type()],
        "/v1/otlp/v1/metrics",
        req.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let body = ExportMetricsServiceResponse::decode(res.bytes().await).unwrap();
    let partial_success = body.partial_success.as_ref().unwrap();
    assert_eq!(partial_success.rejected_data_points, 0);
    assert!(
        partial_success
            .error_message
            .contains("greptime_otel_resource_info"),
        "unexpected warning: {}",
        partial_success.error_message
    );
    validate_data(
        "otlp_metrics_conflict_main_data_accepted",
        &client,
        "select greptime_value from conflict_gauge;",
        "[[7.0]]",
    )
    .await;

    // The descriptor is auto-created again once the conflicting table is
    // gone, and a follow-up write into it does not degrade.
    let res = execute_sql(&client, "drop table greptime_otel_resource_info;").await;
    assert_eq!(res.status(), StatusCode::OK);
    for round in 0..2 {
        let req: ExportMetricsServiceRequest = serde_json::from_str(content).unwrap();
        let res = send_req(
            &client,
            vec![content_type()],
            "/v1/otlp/v1/metrics",
            req.encode_to_vec(),
            false,
        )
        .await;
        assert_eq!(StatusCode::OK, res.status());
        let body = ExportMetricsServiceResponse::decode(res.bytes().await).unwrap();
        assert!(
            body.partial_success.is_none(),
            "round {round} unexpectedly degraded: {:?}",
            body.partial_success
        );
    }
    validate_data(
        "otlp_metrics_owned_descriptor_keeps_accepting",
        &client,
        "select count(*) from greptime_otel_resource_info;",
        "[[1]]",
    )
    .await;

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_otlp_traces_skip_wal(distributed: bool) {
    let mut cases = Vec::new();
    let traces = make_trace_v1_request(
        "wal-test",
        vec![json!({
            "traceId":"0102030405060708090a0b0c0d0e0f10",
            "spanId":"0102030405060708", "name":"wal test",
            "startTimeUnixNano":"1700000000000000000",
            "endTimeUnixNano":"1700000001000000000"
        })],
    );
    let mut traces_case = HttpWalCase::new(
        "OTLP traces",
        "/v1/otlp/v1/traces",
        "application/x-protobuf",
        traces.encode_to_vec(),
    );
    traces_case
        .headers
        .push(("x-greptime-pipeline-name", "greptime_trace_v0"));
    cases.push(traces_case);
    let mut traces_v1 = HttpWalCase::new(
        "OTLP traces v1",
        "/v1/otlp/v1/traces",
        "application/x-protobuf",
        traces.encode_to_vec(),
    );
    traces_v1.headers = vec![
        ("x-greptime-pipeline-name", "greptime_trace_v1"),
        ("x-greptime-trace-table-name", "wal_traces_v1"),
    ];
    cases.push(traces_v1);
    check_http_skip_wal("otlp_traces", &cases, distributed).await;
}

pub async fn test_otlp_traces_v0(store_type: StorageType) {
    // init
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_traces").await;

    let content = r#"
{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"telemetrygen"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"telemetrygen","version":"","attributes":[],"droppedAttributesCount":0},"spans":[{"traceId":"c05d7a4ec8e1f231f02ed6e8da8655b4","spanId":"9630f2916e2f7909","traceState":"","parentSpanId":"d24f921c75f68e23","flags":256,"name":"okey-dokey-0","kind":2,"startTimeUnixNano":"1736480942444376000","endTimeUnixNano":"1736480942444499000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-client"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"c05d7a4ec8e1f231f02ed6e8da8655b4","spanId":"d24f921c75f68e23","traceState":"","parentSpanId":"","flags":256,"name":"lets-go","kind":3,"startTimeUnixNano":"1736480942444376000","endTimeUnixNano":"1736480942444499000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-server"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"cc9e0991a2e63d274984bd44ee669203","spanId":"8f847259b0f6e1ab","traceState":"","parentSpanId":"eba7be77e3558179","flags":256,"name":"okey-dokey-0","kind":2,"startTimeUnixNano":"1736480942444589000","endTimeUnixNano":"1736480942444712000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-client"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"cc9e0991a2e63d274984bd44ee669203","spanId":"eba7be77e3558179","traceState":"","parentSpanId":"","flags":256,"name":"lets-go","kind":3,"startTimeUnixNano":"1736480942444589000","endTimeUnixNano":"1736480942444712000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-server"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.4.0"}]}
"#;

    let req: ExportTraceServiceRequest = serde_json::from_str(content).unwrap();
    let body = req.encode_to_vec();

    // handshake
    let client = TestClient::new(app).await;

    // write traces data
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static("greptime_trace_v0"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"[["telemetrygen"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        &format!(
            "select service_name from {};",
            trace_services_table_name(TRACE_TABLE_NAME)
        ),
        expected,
    )
    .await;

    // Validate operations table
    let expected = r#"[["telemetrygen","SPAN_KIND_CLIENT","lets-go"],["telemetrygen","SPAN_KIND_SERVER","okey-dokey-0"]]"#;
    validate_data(
        "otlp_traces_operations",
        &client,
        &format!(
            "select service_name, span_kind, span_name from {} order by span_kind, span_name;",
            trace_operations_table_name(TRACE_TABLE_NAME)
        ),
        expected,
    )
    .await;

    // select traces data
    let expected = r#"[[1736480942444376000,1736480942444499000,123000,"c05d7a4ec8e1f231f02ed6e8da8655b4","d24f921c75f68e23",null,"SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen",{"net.peer.ip":"1.2.3.4","peer.service":"telemetrygen-server"},[],[],"telemetrygen","",{},{"service.name":"telemetrygen"}],[1736480942444376000,1736480942444499000,123000,"c05d7a4ec8e1f231f02ed6e8da8655b4","9630f2916e2f7909","d24f921c75f68e23","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen",{"net.peer.ip":"1.2.3.4","peer.service":"telemetrygen-client"},[],[],"telemetrygen","",{},{"service.name":"telemetrygen"}],[1736480942444589000,1736480942444712000,123000,"cc9e0991a2e63d274984bd44ee669203","eba7be77e3558179",null,"SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen",{"net.peer.ip":"1.2.3.4","peer.service":"telemetrygen-server"},[],[],"telemetrygen","",{},{"service.name":"telemetrygen"}],[1736480942444589000,1736480942444712000,123000,"cc9e0991a2e63d274984bd44ee669203","8f847259b0f6e1ab","eba7be77e3558179","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen",{"net.peer.ip":"1.2.3.4","peer.service":"telemetrygen-client"},[],[],"telemetrygen","",{},{"service.name":"telemetrygen"}]]"#;
    validate_data(
        "otlp_traces",
        &client,
        "select * from opentelemetry_traces;",
        expected,
    )
    .await;

    // The v0 data model is frozen on the unsigned `duration_nano`: pin the
    // schema so the unsigned→signed flip of the built-in models (which flips
    // only v1) cannot leak into v0 and break writes into pre-existing v0
    // tables.
    validate_data(
        "otlp_traces_v0_schema",
        &client,
        "select column_name, lower(data_type) from information_schema.columns \
         where table_name = 'opentelemetry_traces' and column_name = 'duration_nano';",
        r#"[["duration_nano","bigint unsigned"]]"#,
    )
    .await;

    // drop table
    let res = client
        .get("/v1/sql?sql=drop table opentelemetry_traces;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // write traces data with gzip
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static("greptime_trace_v0"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        true,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // write traces data without pipeline
    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        )],
        "/v1/otlp/v1/traces",
        body.clone(),
        true,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());

    // select traces data again
    validate_data(
        "otlp_traces_with_gzip",
        &client,
        "select * from opentelemetry_traces;",
        expected,
    )
    .await;

    // The attribute columns are JSON and the event/link columns are lists, whose
    // SST encodings differ from their memtable representations.
    flush_table(&client, "opentelemetry_traces").await;
    validate_data(
        "otlp_traces_after_flush",
        &client,
        "select * from opentelemetry_traces;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

/// Exercises v2 protobuf ingestion and JSON2 paths.
pub(crate) async fn test_otlp_traces_v2(
    store_type: StorageType,
) -> Result<(), Box<dyn std::error::Error>> {
    common_telemetry::init_default_ut_logging();

    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_otlp_traces_v2").await;
    let client = TestClient::new(app).await;
    let table_name = "trace_v2_spans";

    let request: ExportTraceServiceRequest = serde_json::from_value(json!({
        "resourceSpans": [{
            "resource": {
                "attributes": [
                    make_string_attr("service.name", "frontend"),
                    make_string_attr("deployment.environment", "production")
                ]
            },
            "scopeSpans": [{
                "scope": {
                    "name": "trace-v2-test",
                    "version": "1.0.0",
                    "attributes": [make_bool_attr("enabled", true)]
                },
                "spans": [{
                    "traceId": "c05d7a4ec8e1f231f02ed6e8da8655b4",
                    "spanId": "9630f2916e2f7909",
                    "name": "GET /api",
                    "kind": 2,
                    "startTimeUnixNano": "1736480942444376000",
                    "endTimeUnixNano": "1736480942444499000",
                    "attributes": [
                        make_int_attr("http.status_code", 200),
                        make_string_attr("http.route", "/api"),
                        {"key": "ratio", "value": {"doubleValue": 1.5}},
                        make_string_attr("a\"b", "quoted"),
                        make_string_attr("a\\b", "backslash")
                    ],
                    "events": [{
                        "timeUnixNano": "1736480942444400000",
                        "name": "cache.hit",
                        "attributes": [make_int_attr("event.code", 7)]
                    }],
                    "links": [{
                        "traceId": "cc9e0991a2e63d274984bd44ee669203",
                        "spanId": "8f847259b0f6e1ab",
                        "attributes": [make_string_attr("link.type", "follows_from")]
                    }],
                    "status": { "message": "", "code": 1 }
                }]
            }],
            "schemaUrl": "https://opentelemetry.io/schemas/1.4.0"
        }]
    }))?;

    let response = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V2_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(table_name),
            ),
        ],
        "/v1/otlp/v1/traces",
        request.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, response.status());

    validate_data(
        "otlp_traces_v2_json2",
        &client,
        "select service_name, span_attributes.\"http.status_code\"::BIGINT, \
         resource_attributes.\"deployment.environment\"::STRING, \
         scope_attributes.enabled::BOOLEAN from trace_v2_spans;",
        r#"[["frontend",200,"production",true]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v2_fixed_schema",
        &client,
        "select count(*) from information_schema.columns where table_name = 'trace_v2_spans' \
         and column_name like 'span_attributes.%';",
        "[[0]]",
    )
    .await;
    validate_data(
        "otlp_traces_v2_events_links",
        &client,
        r#"select json_get_string(span_events, '$[0].name'),
                  json_get_int(span_events, '$[0].attributes."event.code"'),
                  json_get_string(span_links, '$[0].trace_id'),
                  json_get_string(span_links, '$[0].attributes."link.type"')
           from trace_v2_spans;"#,
        r#"[["cache.hit",7,"cc9e0991a2e63d274984bd44ee669203","follows_from"]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v2_escaped_keys",
        &client,
        r#"select span_attributes."a""b"::STRING, span_attributes."a\b"::STRING from trace_v2_spans;"#,
        r#"[["quoted","backslash"]]"#,
    ).await;
    validate_data(
        "otlp_traces_v2_semantics",
        &client,
        "select count(*) from information_schema.tables where table_name = 'trace_v2_spans' \
         and create_options like '%table_data_model=greptime_trace_v2%' \
         and create_options like '%greptime.semantic.pipeline=greptime_trace_v2%';",
        "[[1]]",
    )
    .await;

    // JSON2 path extraction has to resolve the same way against SST files as it
    // does against the memtable.
    flush_table(&client, "trace_v2_spans").await;
    validate_data(
        "otlp_traces_v2_json2_after_flush",
        &client,
        "select service_name, span_attributes.\"http.status_code\"::BIGINT, \
         resource_attributes.\"deployment.environment\"::STRING, \
         scope_attributes.enabled::BOOLEAN from trace_v2_spans;",
        r#"[["frontend",200,"production",true]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v2_events_links_after_flush",
        &client,
        r#"select json_get_string(span_events, '$[0].name'),
                  json_get_int(span_events, '$[0].attributes."event.code"'),
                  json_get_string(span_links, '$[0].trace_id'),
                  json_get_string(span_links, '$[0].attributes."link.type"')
           from trace_v2_spans;"#,
        r#"[["cache.hit",7,"cc9e0991a2e63d274984bd44ee669203","follows_from"]]"#,
    )
    .await;

    guard.remove_all().await;
    Ok(())
}

/// One real OTLP export must come out of `semantic_relationships` as the
/// zero-configuration chain; resources missing an identity column derive
/// nothing extra.
pub async fn test_otlp_traces_v1_entity_graph(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_otlp_traces_entity_graph").await;
    let client = TestClient::new(app).await;

    let span = |trace_id: &str, span_id: &str, parent: &str, kind: u32| {
        json!({
            "traceId": trace_id,
            "spanId": span_id,
            "parentSpanId": parent,
            "name": "op",
            "kind": kind,
            "startTimeUnixNano": "1736480942444376000",
            "endTimeUnixNano": "1736480942444499000",
            "attributes": [],
            "status": { "message": "", "code": 0 }
        })
    };
    let resource_spans = |attributes: Vec<Value>, spans: Vec<Value>| {
        json!({
            "resource": { "attributes": attributes },
            "scopeSpans": [{ "scope": { "name": "graph-tests" }, "spans": spans }],
            "schemaUrl": "https://opentelemetry.io/schemas/1.4.0"
        })
    };

    // frontend: full identity (instance id + pod uid); cart: bare service —
    // no instance entity; batch: instance id but no pod uid — no runs_on.
    let req: ExportTraceServiceRequest = serde_json::from_value(json!({
        "resourceSpans": [
            resource_spans(
                vec![
                    make_string_attr("service.name", "frontend"),
                    make_string_attr("service.instance.id", "inst-1"),
                    make_string_attr("k8s.pod.uid", "uid-9"),
                    make_string_attr("k8s.pod.name", "api-1"),
                    make_string_attr("k8s.namespace.name", "default"),
                ],
                vec![span("c05d7a4ec8e1f231f02ed6e8da8655b4", "d24f921c75f68e23", "", 3)],
            ),
            resource_spans(
                vec![make_string_attr("service.name", "cart")],
                vec![span(
                    "c05d7a4ec8e1f231f02ed6e8da8655b4",
                    "9630f2916e2f7909",
                    "d24f921c75f68e23",
                    2,
                )],
            ),
            resource_spans(
                vec![
                    make_string_attr("service.name", "batch"),
                    make_string_attr("service.instance.id", "binst-1"),
                ],
                vec![span("cc9e0991a2e63d274984bd44ee669203", "8f847259b0f6e1ab", "", 1)],
            ),
        ]
    }))
    .unwrap();
    let res = send_trace_v1_req(&client, "graph_traces", req, false).await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"[["service","frontend","service","cart","calls"],["service.instance","batch,binst-1","service","batch","part_of"],["service.instance","frontend,inst-1","service","frontend","part_of"],["service.instance","frontend,inst-1","k8s.pod","uid-9","runs_on"]]"#;
    validate_data(
        "otlp_traces_entity_graph",
        &client,
        "select src_type, src_id, dst_type, dst_id, rel_type \
         from greptime_private.semantic_relationships \
         where observed_at >= '2025-01-01 00:00:00' order by rel_type, src_id;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_otlp_traces_v1(store_type: StorageType) {
    // init
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_traces").await;

    let content = r#"
{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"telemetrygen"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"telemetrygen","version":"","attributes":[],"droppedAttributesCount":0},"spans":[{"traceId":"c05d7a4ec8e1f231f02ed6e8da8655b4","spanId":"9630f2916e2f7909","traceState":"","parentSpanId":"d24f921c75f68e23","flags":256,"name":"okey-dokey-0","kind":2,"startTimeUnixNano":"1736480942444376000","endTimeUnixNano":"1736480942444499000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-client"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"c05d7a4ec8e1f231f02ed6e8da8655b4","spanId":"d24f921c75f68e23","traceState":"","parentSpanId":"","flags":256,"name":"lets-go","kind":3,"startTimeUnixNano":"1736480942444376000","endTimeUnixNano":"1736480942444499000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-server"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"cc9e0991a2e63d274984bd44ee669203","spanId":"8f847259b0f6e1ab","traceState":"","parentSpanId":"eba7be77e3558179","flags":256,"name":"okey-dokey-0","kind":2,"startTimeUnixNano":"1736480942444589000","endTimeUnixNano":"1736480942444712000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-client"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"cc9e0991a2e63d274984bd44ee669203","spanId":"eba7be77e3558179","traceState":"","parentSpanId":"","flags":256,"name":"lets-go","kind":3,"startTimeUnixNano":"1736480942444589000","endTimeUnixNano":"1736480942444712000","attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-server"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.4.0"}]}
"#;

    let trace_table_name = "mytable";
    let req: ExportTraceServiceRequest = serde_json::from_str(content).unwrap();
    let body = req.encode_to_vec();

    // handshake
    let client = TestClient::new(app).await;

    // write traces data
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_name),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"[["telemetrygen"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        &format!(
            "select service_name from {};",
            trace_services_table_name(trace_table_name)
        ),
        expected,
    )
    .await;

    // Validate operations table
    let expected = r#"[["telemetrygen","SPAN_KIND_CLIENT","lets-go"],["telemetrygen","SPAN_KIND_SERVER","okey-dokey-0"]]"#;
    validate_data(
        "otlp_traces_operations_v1",
        &client,
        &format!(
            "select service_name, span_kind, span_name from {} order by span_kind, span_name;",
            trace_operations_table_name(trace_table_name)
        ),
        expected,
    )
    .await;

    // select traces data
    let expected = r#"[[1736480942444376000,1736480942444499000,123000,null,"c05d7a4ec8e1f231f02ed6e8da8655b4","d24f921c75f68e23","SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-server",[],[]],[1736480942444376000,1736480942444499000,123000,"d24f921c75f68e23","c05d7a4ec8e1f231f02ed6e8da8655b4","9630f2916e2f7909","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-client",[],[]],[1736480942444589000,1736480942444712000,123000,null,"cc9e0991a2e63d274984bd44ee669203","eba7be77e3558179","SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-server",[],[]],[1736480942444589000,1736480942444712000,123000,"eba7be77e3558179","cc9e0991a2e63d274984bd44ee669203","8f847259b0f6e1ab","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-client",[],[]]]"#;
    validate_data("otlp_traces", &client, "select * from mytable;", expected).await;

    // `span_events` / `span_links` are JSON columns and the table is partitioned
    // on `trace_id`, so the read back has to reassemble SST files across regions.
    flush_table(&client, "mytable").await;
    validate_data(
        "otlp_traces_after_flush",
        &client,
        "select * from mytable;",
        expected,
    )
    .await;

    // The trace v1 main table carries the trace identity (events/links preserved as
    // JSON columns by the v1 model).
    validate_data(
        "otlp_traces_semantic_identity",
        &client,
        "select count(*) from information_schema.tables where table_name = 'mytable' \
         and create_options like '%greptime.semantic.signal_type=trace%' \
         and create_options like '%greptime.semantic.source=opentelemetry%' \
         and create_options like '%greptime.semantic.pipeline=greptime_trace_v1%';",
        "[[1]]",
    )
    .await;

    let expected_ddl = r#"[["mytable","CREATE TABLE IF NOT EXISTS \"mytable\" (\n  \"timestamp\" TIMESTAMP(9) NOT NULL,\n  \"timestamp_end\" TIMESTAMP(9) NULL,\n  \"duration_nano\" BIGINT NULL,\n  \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_id\" STRING NULL,\n  \"span_kind\" STRING NULL,\n  \"span_name\" STRING NULL,\n  \"span_status_code\" STRING NULL,\n  \"span_status_message\" STRING NULL,\n  \"trace_state\" STRING NULL,\n  \"scope_name\" STRING NULL,\n  \"scope_version\" STRING NULL,\n  \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_attributes.net.peer.ip\" STRING NULL,\n  \"span_attributes.peer.service\" STRING NULL,\n  \"span_events\" JSON NULL,\n  \"span_links\" JSON NULL,\n  TIME INDEX (\"timestamp\"),\n  PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n  trace_id < '1',\n  trace_id >= '1' AND trace_id < '2',\n  trace_id >= '2' AND trace_id < '3',\n  trace_id >= '3' AND trace_id < '4',\n  trace_id >= '4' AND trace_id < '5',\n  trace_id >= '5' AND trace_id < '6',\n  trace_id >= '6' AND trace_id < '7',\n  trace_id >= '7' AND trace_id < '8',\n  trace_id >= '8' AND trace_id < '9',\n  trace_id >= '9' AND trace_id < 'a',\n  trace_id >= 'a' AND trace_id < 'b',\n  trace_id >= 'b' AND trace_id < 'c',\n  trace_id >= 'c' AND trace_id < 'd',\n  trace_id >= 'd' AND trace_id < 'e',\n  trace_id >= 'e' AND trace_id < 'f',\n  trace_id >= 'f'\n)\nENGINE=mito\nWITH(\n  'comment' = 'Created on insertion',\n  append_mode = 'true',\n  'greptime.semantic.entity.service.id' = 'service_name',\n  'greptime.semantic.pipeline' = 'greptime_trace_v1',\n  'greptime.semantic.signal_type' = 'trace',\n  'greptime.semantic.source' = 'opentelemetry',\n  'greptime.semantic.trace.conventions' = 'https://opentelemetry.io/schemas/1.4.0',\n  table_data_model = 'greptime_trace_v1'\n)"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        "show create table mytable;",
        expected_ddl,
    )
    .await;

    let expected_ddl = r#"[["mytable_services","CREATE TABLE IF NOT EXISTS \"mytable_services\" (\n  \"timestamp\" TIMESTAMP(9) NOT NULL,\n  \"service_name\" STRING NULL,\n  TIME INDEX (\"timestamp\"),\n  PRIMARY KEY (\"service_name\")\n)\n\nENGINE=mito\nWITH(\n  'comment' = 'Created on insertion',\n  append_mode = 'false'\n)"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        &format!(
            "show create table {};",
            trace_services_table_name(trace_table_name)
        ),
        expected_ddl,
    )
    .await;

    // drop table
    let res = client.get("/v1/sql?sql=drop table mytable;").send().await;
    assert_eq!(res.status(), StatusCode::OK);

    // test trace table with custom partitions: 1
    let trace_table_part1 = "trace_table_part1";
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_part1),
            ),
            (
                HeaderName::from_static("x-greptime-hints"),
                HeaderValue::from_static("trace_table_partitions=1"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected_ddl = r#"[["trace_table_part1","CREATE TABLE IF NOT EXISTS \"trace_table_part1\" (\n  \"timestamp\" TIMESTAMP(9) NOT NULL,\n  \"timestamp_end\" TIMESTAMP(9) NULL,\n  \"duration_nano\" BIGINT NULL,\n  \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_id\" STRING NULL,\n  \"span_kind\" STRING NULL,\n  \"span_name\" STRING NULL,\n  \"span_status_code\" STRING NULL,\n  \"span_status_message\" STRING NULL,\n  \"trace_state\" STRING NULL,\n  \"scope_name\" STRING NULL,\n  \"scope_version\" STRING NULL,\n  \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_attributes.net.peer.ip\" STRING NULL,\n  \"span_attributes.peer.service\" STRING NULL,\n  \"span_events\" JSON NULL,\n  \"span_links\" JSON NULL,\n  TIME INDEX (\"timestamp\"),\n  PRIMARY KEY (\"service_name\")\n)\n\nENGINE=mito\nWITH(\n  'comment' = 'Created on insertion',\n  append_mode = 'true',\n  'greptime.semantic.entity.service.id' = 'service_name',\n  'greptime.semantic.pipeline' = 'greptime_trace_v1',\n  'greptime.semantic.signal_type' = 'trace',\n  'greptime.semantic.source' = 'opentelemetry',\n  'greptime.semantic.trace.conventions' = 'https://opentelemetry.io/schemas/1.4.0',\n  table_data_model = 'greptime_trace_v1'\n)"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        "show create table trace_table_part1;",
        expected_ddl,
    )
    .await;

    // test trace table with custom partitions: 4
    let trace_table_part4 = "trace_table_part4";
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_part4),
            ),
            (
                HeaderName::from_static("x-greptime-hints"),
                HeaderValue::from_static("trace_table_partitions=4"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected_ddl = r#"[["trace_table_part4","CREATE TABLE IF NOT EXISTS \"trace_table_part4\" (\n  \"timestamp\" TIMESTAMP(9) NOT NULL,\n  \"timestamp_end\" TIMESTAMP(9) NULL,\n  \"duration_nano\" BIGINT NULL,\n  \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_id\" STRING NULL,\n  \"span_kind\" STRING NULL,\n  \"span_name\" STRING NULL,\n  \"span_status_code\" STRING NULL,\n  \"span_status_message\" STRING NULL,\n  \"trace_state\" STRING NULL,\n  \"scope_name\" STRING NULL,\n  \"scope_version\" STRING NULL,\n  \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_attributes.net.peer.ip\" STRING NULL,\n  \"span_attributes.peer.service\" STRING NULL,\n  \"span_events\" JSON NULL,\n  \"span_links\" JSON NULL,\n  TIME INDEX (\"timestamp\"),\n  PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n  trace_id < '4',\n  trace_id >= '4' AND trace_id < '8',\n  trace_id >= '8' AND trace_id < 'c',\n  trace_id >= 'c'\n)\nENGINE=mito\nWITH(\n  'comment' = 'Created on insertion',\n  append_mode = 'true',\n  'greptime.semantic.entity.service.id' = 'service_name',\n  'greptime.semantic.pipeline' = 'greptime_trace_v1',\n  'greptime.semantic.signal_type' = 'trace',\n  'greptime.semantic.source' = 'opentelemetry',\n  'greptime.semantic.trace.conventions' = 'https://opentelemetry.io/schemas/1.4.0',\n  table_data_model = 'greptime_trace_v1'\n)"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        "show create table trace_table_part4;",
        expected_ddl,
    )
    .await;

    // test trace table with custom partitions: 32
    let trace_table_part32 = "trace_table_part32";
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_part32),
            ),
            (
                HeaderName::from_static("x-greptime-hints"),
                HeaderValue::from_static("trace_table_partitions=32"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected_ddl = r#"[["trace_table_part32","CREATE TABLE IF NOT EXISTS \"trace_table_part32\" (\n  \"timestamp\" TIMESTAMP(9) NOT NULL,\n  \"timestamp_end\" TIMESTAMP(9) NULL,\n  \"duration_nano\" BIGINT NULL,\n  \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_id\" STRING NULL,\n  \"span_kind\" STRING NULL,\n  \"span_name\" STRING NULL,\n  \"span_status_code\" STRING NULL,\n  \"span_status_message\" STRING NULL,\n  \"trace_state\" STRING NULL,\n  \"scope_name\" STRING NULL,\n  \"scope_version\" STRING NULL,\n  \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n  \"span_attributes.net.peer.ip\" STRING NULL,\n  \"span_attributes.peer.service\" STRING NULL,\n  \"span_events\" JSON NULL,\n  \"span_links\" JSON NULL,\n  TIME INDEX (\"timestamp\"),\n  PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n  trace_id < '08',\n  trace_id >= '08' AND trace_id < '10',\n  trace_id >= '10' AND trace_id < '18',\n  trace_id >= '18' AND trace_id < '20',\n  trace_id >= '20' AND trace_id < '28',\n  trace_id >= '28' AND trace_id < '30',\n  trace_id >= '30' AND trace_id < '38',\n  trace_id >= '38' AND trace_id < '40',\n  trace_id >= '40' AND trace_id < '48',\n  trace_id >= '48' AND trace_id < '50',\n  trace_id >= '50' AND trace_id < '58',\n  trace_id >= '58' AND trace_id < '60',\n  trace_id >= '60' AND trace_id < '68',\n  trace_id >= '68' AND trace_id < '70',\n  trace_id >= '70' AND trace_id < '78',\n  trace_id >= '78' AND trace_id < '80',\n  trace_id >= '80' AND trace_id < '88',\n  trace_id >= '88' AND trace_id < '90',\n  trace_id >= '90' AND trace_id < '98',\n  trace_id >= '98' AND trace_id < 'a0',\n  trace_id >= 'a0' AND trace_id < 'a8',\n  trace_id >= 'a8' AND trace_id < 'b0',\n  trace_id >= 'b0' AND trace_id < 'b8',\n  trace_id >= 'b8' AND trace_id < 'c0',\n  trace_id >= 'c0' AND trace_id < 'c8',\n  trace_id >= 'c8' AND trace_id < 'd0',\n  trace_id >= 'd0' AND trace_id < 'd8',\n  trace_id >= 'd8' AND trace_id < 'e0',\n  trace_id >= 'e0' AND trace_id < 'e8',\n  trace_id >= 'e8' AND trace_id < 'f0',\n  trace_id >= 'f0' AND trace_id < 'f8',\n  trace_id >= 'f8'\n)\nENGINE=mito\nWITH(\n  'comment' = 'Created on insertion',\n  append_mode = 'true',\n  'greptime.semantic.entity.service.id' = 'service_name',\n  'greptime.semantic.pipeline' = 'greptime_trace_v1',\n  'greptime.semantic.signal_type' = 'trace',\n  'greptime.semantic.source' = 'opentelemetry',\n  'greptime.semantic.trace.conventions' = 'https://opentelemetry.io/schemas/1.4.0',\n  table_data_model = 'greptime_trace_v1'\n)"]]"#;
    validate_data(
        "otlp_traces",
        &client,
        "show create table trace_table_part32;",
        expected_ddl,
    )
    .await;

    // invalid partition count
    let trace_table_part_abc = "trace_table_part_abc";
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_part_abc),
            ),
            (
                HeaderName::from_static("x-greptime-hints"),
                HeaderValue::from_static("trace_table_partitions=abc"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());

    // write traces data with gzip
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_name),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        true,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // select traces data again
    validate_data(
        "otlp_traces_with_gzip",
        &client,
        "select * from mytable;",
        expected,
    )
    .await;

    let coercion_table_name = "trace_type_coercion";
    let coercion_req = make_trace_v1_request(
        "type-coercion",
        vec![make_trace_v1_span(
            "00000000000000000000000000000001",
            "0000000000000001",
            "coercion-seed",
            1_736_480_942_444_376_000,
            1_736_480_942_444_499_000,
            vec![
                make_double_attr("attr_float", 1.5),
                make_int_attr("attr_int", 10),
                make_bool_attr("attr_bool", true),
            ],
        )],
    );
    let res = send_trace_v1_req(&client, coercion_table_name, coercion_req, false).await;
    assert_eq!(StatusCode::OK, res.status());

    let coercion_req = make_trace_v1_request(
        "type-coercion",
        vec![make_trace_v1_span(
            "00000000000000000000000000000002",
            "0000000000000002",
            "coercion-apply",
            1_736_480_942_444_589_000,
            1_736_480_942_444_712_000,
            vec![
                make_int_attr("attr_float", 2),
                make_string_attr("attr_int", "20"),
                make_string_attr("attr_bool", "false"),
            ],
        )],
    );
    let res = send_trace_v1_req(&client, coercion_table_name, coercion_req, false).await;
    assert_eq!(StatusCode::OK, res.status());

    let string_target_table_name = "trace_type_coercion_to_string";
    let string_target_seed_req = make_trace_v1_request(
        "type-coercion-string",
        vec![make_trace_v1_span(
            "00000000000000000000000000000021",
            "0000000000000021",
            "string-target-seed",
            1_736_480_942_444_720_000,
            1_736_480_942_444_820_000,
            vec![
                make_string_attr("attr_int", "seed"),
                make_string_attr("attr_float", "seed"),
                make_string_attr("attr_bool", "seed"),
            ],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        string_target_table_name,
        string_target_seed_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let string_target_req = make_trace_v1_request(
        "type-coercion-string",
        vec![make_trace_v1_span(
            "00000000000000000000000000000022",
            "0000000000000022",
            "string-target-apply",
            1_736_480_942_444_830_000,
            1_736_480_942_444_930_000,
            vec![
                make_int_attr("attr_int", 20),
                make_double_attr("attr_float", 2.5),
                make_bool_attr("attr_bool", false),
            ],
        )],
    );
    let res = send_trace_v1_req(&client, string_target_table_name, string_target_req, false).await;
    assert_eq!(StatusCode::OK, res.status());

    validate_data(
        "otlp_traces_v1_type_coercion_to_string_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_bool\", \"span_attributes.attr_float\", \"span_attributes.attr_int\" from {} order by trace_id;",
            string_target_table_name
        ),
        r#"[["00000000000000000000000000000021","seed","seed","seed"],["00000000000000000000000000000022","false","2.5","20"]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_type_coercion_to_string_schema",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_coercion_to_string' and column_name in ('span_attributes.attr_bool', 'span_attributes.attr_float', 'span_attributes.attr_int') order by column_name;",
        r#"[["span_attributes.attr_bool","string","FIELD"],["span_attributes.attr_float","string","FIELD"],["span_attributes.attr_int","string","FIELD"]]"#,
    )
    .await;

    let intra_batch_prefer_non_string_table_name = "trace_type_prefer_non_string";
    let intra_batch_prefer_non_string_req = make_trace_v1_request(
        "type-prefer-non-string",
        vec![
            make_trace_v1_span(
                "00000000000000000000000000000031",
                "0000000000000031",
                "prefer-non-string-seed",
                1_736_480_942_444_940_000,
                1_736_480_942_445_040_000,
                vec![
                    make_string_attr("attr_int", "10"),
                    make_string_attr("attr_float", "1.5"),
                    make_string_attr("attr_bool", "true"),
                ],
            ),
            make_trace_v1_span(
                "00000000000000000000000000000032",
                "0000000000000032",
                "prefer-non-string-apply",
                1_736_480_942_445_050_000,
                1_736_480_942_445_150_000,
                vec![
                    make_int_attr("attr_int", 20),
                    make_double_attr("attr_float", 2.5),
                    make_bool_attr("attr_bool", false),
                ],
            ),
        ],
    );
    let res = send_trace_v1_req(
        &client,
        intra_batch_prefer_non_string_table_name,
        intra_batch_prefer_non_string_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    validate_data(
        "otlp_traces_v1_prefer_non_string_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_bool\", \"span_attributes.attr_float\", \"span_attributes.attr_int\" from {} order by trace_id;",
            intra_batch_prefer_non_string_table_name
        ),
        r#"[["00000000000000000000000000000031",true,1.5,10],["00000000000000000000000000000032",false,2.5,20]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_prefer_non_string_schema",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_prefer_non_string' and column_name in ('span_attributes.attr_bool', 'span_attributes.attr_float', 'span_attributes.attr_int') order by column_name;",
        r#"[["span_attributes.attr_bool","boolean","FIELD"],["span_attributes.attr_float","double","FIELD"],["span_attributes.attr_int","bigint","FIELD"]]"#,
    )
    .await;

    let existing_float_table_name = "trace_type_existing_float_prefers_schema";
    let existing_float_seed_req = make_trace_v1_request(
        "type-existing-float",
        vec![make_trace_v1_span(
            "00000000000000000000000000000041",
            "0000000000000041",
            "existing-float-seed",
            1_736_480_942_445_160_000,
            1_736_480_942_445_260_000,
            vec![make_double_attr("attr_num", 1.25)],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        existing_float_table_name,
        existing_float_seed_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let existing_float_req = make_trace_v1_request(
        "type-existing-float",
        vec![
            make_trace_v1_span(
                "00000000000000000000000000000042",
                "0000000000000042",
                "existing-float-int-first",
                1_736_480_942_445_270_000,
                1_736_480_942_445_370_000,
                vec![make_int_attr("attr_num", 2)],
            ),
            make_trace_v1_span(
                "00000000000000000000000000000043",
                "0000000000000043",
                "existing-float-float-later",
                1_736_480_942_445_380_000,
                1_736_480_942_445_480_000,
                vec![make_double_attr("attr_num", 3.5)],
            ),
        ],
    );
    let res = send_trace_v1_req(
        &client,
        existing_float_table_name,
        existing_float_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    validate_data(
        "otlp_traces_v1_existing_float_prefers_schema_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_num\" from {} order by trace_id;",
            existing_float_table_name
        ),
        r#"[["00000000000000000000000000000041",1.25],["00000000000000000000000000000042",2.0],["00000000000000000000000000000043",3.5]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_existing_float_prefers_schema_type",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_existing_float_prefers_schema' and column_name = 'span_attributes.attr_num';",
        r#"[["span_attributes.attr_num","double","FIELD"]]"#,
    )
    .await;

    let existing_int_table_name = "trace_type_existing_int_widens_to_float";
    let existing_int_seed_req = make_trace_v1_request(
        "type-existing-int",
        vec![make_trace_v1_span(
            "00000000000000000000000000000051",
            "0000000000000051",
            "existing-int-seed",
            1_736_480_942_445_490_000,
            1_736_480_942_445_590_000,
            vec![make_int_attr("attr_num", 1)],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        existing_int_table_name,
        existing_int_seed_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let existing_int_req = make_trace_v1_request(
        "type-existing-int",
        vec![
            make_trace_v1_span(
                "00000000000000000000000000000052",
                "0000000000000052",
                "existing-int-upcast-int",
                1_736_480_942_445_600_000,
                1_736_480_942_445_700_000,
                vec![make_int_attr("attr_num", 2)],
            ),
            make_trace_v1_span(
                "00000000000000000000000000000053",
                "0000000000000053",
                "existing-int-upcast-float",
                1_736_480_942_445_710_000,
                1_736_480_942_445_810_000,
                vec![make_double_attr("attr_num", 3.5)],
            ),
        ],
    );
    let res = send_trace_v1_req(&client, existing_int_table_name, existing_int_req, false).await;
    assert_eq!(StatusCode::OK, res.status());

    // Trace schema reconciliation widens the existing column through a direct
    // alter-table call, so it must identify itself as an OTLP auto alteration.
    wait_for_event_data(
        &client,
        "SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type = 'alter_table' AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'otlp' AND json_get_string(event_context, 'reason') = 'auto_alter'",
        r#"[["{\"protocol\":\"otlp\",\"reason\":\"auto_alter\"}"]]"#,
    )
    .await;

    validate_data(
        "otlp_traces_v1_existing_int_widens_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_num\" from {} order by trace_id;",
            existing_int_table_name
        ),
        r#"[["00000000000000000000000000000051",1.0],["00000000000000000000000000000052",2.0],["00000000000000000000000000000053",3.5]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_existing_int_widens_type",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_existing_int_widens_to_float' and column_name = 'span_attributes.attr_num';",
        r#"[["span_attributes.attr_num","double","FIELD"]]"#,
    )
    .await;

    let existing_int_atomic_table_name = "trace_type_existing_int_widen_atomic";
    let existing_int_atomic_seed_req = make_trace_v1_request(
        "type-existing-int-atomic",
        vec![make_trace_v1_span(
            "00000000000000000000000000000054",
            "0000000000000054",
            "existing-int-atomic-seed",
            1_736_480_942_445_720_000,
            1_736_480_942_445_820_000,
            vec![
                make_int_attr("attr_num", 1),
                make_int_attr("attr_parse", 10),
            ],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        existing_int_atomic_table_name,
        existing_int_atomic_seed_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let existing_int_atomic_req = make_trace_v1_request(
        "type-existing-int-atomic",
        vec![make_trace_v1_span(
            "00000000000000000000000000000055",
            "0000000000000055",
            "existing-int-atomic-invalid",
            1_736_480_942_445_830_000,
            1_736_480_942_445_930_000,
            vec![
                make_double_attr("attr_num", 3.5),
                make_string_attr("attr_parse", "not_a_number"),
            ],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        existing_int_atomic_table_name,
        existing_int_atomic_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    let body = res.bytes().await;
    let status = GoogleRpcStatus::decode(body.as_ref()).unwrap();
    assert!(
        status
            .message
            .contains("Accepted 0 spans, rejected 1 spans"),
        "unexpected error body: {status:?}"
    );

    validate_data(
        "otlp_traces_v1_existing_int_widen_atomic_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_num\", \"span_attributes.attr_parse\" from {} order by trace_id;",
            existing_int_atomic_table_name
        ),
        r#"[["00000000000000000000000000000054",1,10]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_existing_int_widen_atomic_types",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_existing_int_widen_atomic' and column_name in ('span_attributes.attr_num', 'span_attributes.attr_parse') order by column_name;",
        r#"[["span_attributes.attr_num","bigint","FIELD"],["span_attributes.attr_parse","bigint","FIELD"]]"#,
    )
    .await;

    let existing_int_float_only_table_name = "trace_type_existing_int_float_only";
    let existing_int_float_only_seed_req = make_trace_v1_request(
        "type-existing-int-float-only",
        vec![make_trace_v1_span(
            "00000000000000000000000000000061",
            "0000000000000061",
            "existing-int-float-only-seed",
            1_736_480_942_445_820_000,
            1_736_480_942_445_920_000,
            vec![make_int_attr("attr_num", 1)],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        existing_int_float_only_table_name,
        existing_int_float_only_seed_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let existing_int_float_only_req = make_trace_v1_request(
        "type-existing-int-float-only",
        vec![make_trace_v1_span(
            "00000000000000000000000000000062",
            "0000000000000062",
            "existing-int-float-only-apply",
            1_736_480_942_445_930_000,
            1_736_480_942_446_030_000,
            vec![make_double_attr("attr_num", 2.5)],
        )],
    );
    let res = send_trace_v1_req(
        &client,
        existing_int_float_only_table_name,
        existing_int_float_only_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    validate_data(
        "otlp_traces_v1_existing_int_float_only_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_num\" from {} order by trace_id;",
            existing_int_float_only_table_name
        ),
        r#"[["00000000000000000000000000000061",1.0],["00000000000000000000000000000062",2.5]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_existing_int_float_only_type",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_existing_int_float_only' and column_name = 'span_attributes.attr_num';",
        r#"[["span_attributes.attr_num","double","FIELD"]]"#,
    )
    .await;

    validate_data(
        "otlp_traces_v1_type_coercion_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_int\", \"span_attributes.attr_bool\" from {} order by trace_id;",
            coercion_table_name
        ),
        r#"[["00000000000000000000000000000001",10,true],["00000000000000000000000000000002",20,false]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_type_coercion_float_sum",
        &client,
        &format!(
            "select sum(\"span_attributes.attr_float\") from {};",
            coercion_table_name
        ),
        r#"[[3.5]]"#,
    )
    .await;
    validate_data(
        "otlp_traces_v1_type_coercion_schema",
        &client,
        "select column_name, lower(data_type), semantic_type from information_schema.columns where table_name = 'trace_type_coercion' and column_name in ('span_attributes.attr_bool', 'span_attributes.attr_float', 'span_attributes.attr_int') order by column_name;",
        r#"[["span_attributes.attr_bool","boolean","FIELD"],["span_attributes.attr_float","double","FIELD"],["span_attributes.attr_int","bigint","FIELD"]]"#,
    )
    .await;

    let abort_table_name = "trace_type_abort";
    let abort_seed_req = make_trace_v1_request(
        "type-abort",
        vec![make_trace_v1_span(
            "00000000000000000000000000000011",
            "0000000000000011",
            "abort-seed",
            1_736_480_942_444_800_000,
            1_736_480_942_444_900_000,
            vec![make_int_attr("attr_int", 10)],
        )],
    );
    let res = send_trace_v1_req(&client, abort_table_name, abort_seed_req, false).await;
    assert_eq!(StatusCode::OK, res.status());

    let abort_req = make_trace_v1_request(
        "type-abort",
        vec![
            make_trace_v1_span(
                "00000000000000000000000000000012",
                "0000000000000012",
                "abort-parseable",
                1_736_480_942_445_000_000,
                1_736_480_942_445_100_000,
                vec![make_string_attr("attr_int", "20")],
            ),
            make_trace_v1_span(
                "00000000000000000000000000000013",
                "0000000000000013",
                "abort-unparsable",
                1_736_480_942_445_200_000,
                1_736_480_942_445_300_000,
                vec![make_string_attr("attr_int", "not_a_number")],
            ),
        ],
    );
    let res = send_trace_v1_req(&client, abort_table_name, abort_req, false).await;
    assert_eq!(StatusCode::OK, res.status());
    let body = ExportTraceServiceResponse::decode(res.bytes().await).unwrap();
    let partial_success = body.partial_success.as_ref().unwrap();
    assert_eq!(partial_success.rejected_spans, 1);
    assert!(
        partial_success
            .error_message
            .contains("Accepted 1 spans, rejected 1 spans"),
        "unexpected partial success body: {body:?}"
    );
    assert!(
        partial_success.error_message.contains(
            "Rejected span 00000000000000000000000000000013:0000000000000013 (InvalidArguments)"
        ),
        "unexpected partial success body: {body:?}"
    );
    // The rejection must name the column, the failing value and the target type,
    // otherwise locating the bad attribute needs a collector-side capture.
    assert!(
        partial_success.error_message.contains(
            "failed to coerce trace column 'span_attributes.attr_int' in table \
             'trace_type_abort' from String(\"not_a_number\") to Int64"
        ),
        "unexpected partial success body: {body:?}"
    );

    validate_data(
        "otlp_traces_v1_type_abort_rows",
        &client,
        &format!(
            "select trace_id, \"span_attributes.attr_int\" from {} order by trace_id",
            abort_table_name
        ),
        r#"[["00000000000000000000000000000011",10],["00000000000000000000000000000012",20]]"#,
    )
    .await;

    let chunk_failure_req = make_trace_v1_request(
        "type-discard",
        vec![
            make_trace_v1_span(
                "00000000000000000000000000000021",
                "0000000000000021",
                "discard-one",
                1_736_480_942_445_400_000,
                1_736_480_942_445_500_000,
                vec![make_string_attr("attr_text", "alpha")],
            ),
            make_trace_v1_span(
                "00000000000000000000000000000022",
                "0000000000000022",
                "discard-two",
                1_736_480_942_445_600_000,
                1_736_480_942_445_700_000,
                vec![make_string_attr("attr_text", "beta")],
            ),
        ],
    );
    let res = send_trace_v1_req_with_db(
        &client,
        "nonexistent",
        "trace_chunk_discard",
        chunk_failure_req,
        false,
    )
    .await;
    assert_eq!(StatusCode::BAD_REQUEST, res.status());
    let body = GoogleRpcStatus::decode(res.bytes().await.as_ref()).unwrap();
    assert!(
        body.message.contains("Accepted 0 spans, rejected 2 spans"),
        "unexpected error body: {body:?}"
    );
    assert!(
        body.message
            .contains("Discarded 2 spans after pre-write request failure"),
        "unexpected error body: {body:?}"
    );

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_otlp_logs_skip_wal(distributed: bool) {
    let mut cases = Vec::new();
    let logs = make_log_request(vec![json!({
        "timeUnixNano": "1700000000000000000",
        "body": {"stringValue": "wal test"}
    })]);
    cases.push(HttpWalCase::new(
        "OTLP logs",
        "/v1/otlp/v1/logs",
        "application/x-protobuf",
        logs.encode_to_vec(),
    ));
    check_http_skip_wal("otlp_logs", &cases, distributed).await;
}

pub async fn test_otlp_logs(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_logs").await;

    let client = TestClient::new(app).await;
    let content = r#"
{"resourceLogs":[{"resource":{"attributes":[],"droppedAttributesCount":0},"scopeLogs":[{"scope":{"name":"","version":"","attributes":[{"key":"instance_num","value":{"stringValue":"10"}}],"droppedAttributesCount":0},"logRecords":[{"timeUnixNano":"1736413568497632000","observedTimeUnixNano":"0","severityNumber":9,"severityText":"Info","body":{"stringValue":"the message line one"},"attributes":[{"key":"app","value":{"stringValue":"server1"}}],"droppedAttributesCount":0,"flags":0,"traceId":"f665100a612542b69cc362fe2ae9d3bf","spanId":"e58f01c4c69f4488"}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.4.0"},{"resource":{"attributes":[],"droppedAttributesCount":0},"scopeLogs":[{"scope":{"name":"","version":"","attributes":[],"droppedAttributesCount":0},"logRecords":[{"timeUnixNano":"1736413568538897000","observedTimeUnixNano":"0","severityNumber":9,"severityText":"Info","body":{"stringValue":"the message line two"},"attributes":[{"key":"app","value":{"stringValue":"server2"}}],"droppedAttributesCount":0,"flags":0,"traceId":"f665100a612542b69cc362fe2ae9d3bf","spanId":"e58f01c4c69f4488"}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.4.0"}]}
"#;

    let req: ExportLogsServiceRequest = serde_json::from_str(content).unwrap();
    let body = req.encode_to_vec();

    {
        // write log data
        let res = send_req(
            &client,
            vec![(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            )],
            "/v1/otlp/v1/logs?db=public",
            body.clone(),
            false,
        )
        .await;
        assert_eq!(StatusCode::OK, res.status());
        let expected = "[[1736413568497632000,\"f665100a612542b69cc362fe2ae9d3bf\",\"e58f01c4c69f4488\",\"Info\",9,\"the message line one\",{\"app\":\"server1\"},0,\"\",\"\",{\"instance_num\":\"10\"},\"\",{},\"https://opentelemetry.io/schemas/1.4.0\"],[1736413568538897000,\"f665100a612542b69cc362fe2ae9d3bf\",\"e58f01c4c69f4488\",\"Info\",9,\"the message line two\",{\"app\":\"server2\"},0,\"\",\"\",{},\"\",{},\"https://opentelemetry.io/schemas/1.4.0\"]]";
        validate_data(
            "otlp_logs",
            &client,
            "select * from opentelemetry_logs;",
            expected,
        )
        .await;

        // The attribute columns are JSON, whose SST encoding differs from their
        // memtable representation.
        flush_table(&client, "opentelemetry_logs").await;
        validate_data(
            "otlp_logs_after_flush",
            &client,
            "select * from opentelemetry_logs;",
            expected,
        )
        .await;

        // The auto-created log table carries the log identity.
        validate_data(
            "otlp_logs_semantic_identity",
            &client,
            "select count(*) from information_schema.tables where table_name = 'opentelemetry_logs' \
             and create_options like '%greptime.semantic.signal_type=log%' \
             and create_options like '%greptime.semantic.source=opentelemetry%';",
            "[[1]]",
        )
        .await;

        // Write another batch after the table exists. This verifies existing-schema
        // alignment accepts the built-in JSONB columns in direct OTLP logs.
        let res = send_req(
            &client,
            vec![(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            )],
            "/v1/otlp/v1/logs?db=public",
            body.clone(),
            false,
        )
        .await;
        assert_eq!(StatusCode::OK, res.status());
        validate_data(
            "otlp_logs_multiple_batches",
            &client,
            "select count(*) from opentelemetry_logs;",
            "[[4]]",
        )
        .await;
    }

    {
        // write log data with selector
        let res = send_req(
            &client,
            vec![
                (
                    HeaderName::from_static("content-type"),
                    HeaderValue::from_static("application/x-protobuf"),
                ),
                (
                    HeaderName::from_static("x-greptime-log-table-name"),
                    HeaderValue::from_static("cus_logs"),
                ),
                (
                    HeaderName::from_static("x-greptime-log-extract-keys"),
                    HeaderValue::from_static("resource-attr,instance_num,app,not-exist"),
                ),
            ],
            "/v1/otlp/v1/logs?db=public",
            body.clone(),
            false,
        )
        .await;
        assert_eq!(StatusCode::OK, res.status());

        let expected = "[[1736413568538897000,\"f665100a612542b69cc362fe2ae9d3bf\",\"e58f01c4c69f4488\",\"Info\",9,\"the message line two\",{\"app\":\"server2\"},0,\"\",\"\",{},\"\",{},\"https://opentelemetry.io/schemas/1.4.0\",null,\"server2\"],[1736413568497632000,\"f665100a612542b69cc362fe2ae9d3bf\",\"e58f01c4c69f4488\",\"Info\",9,\"the message line one\",{\"app\":\"server1\"},0,\"\",\"\",{\"instance_num\":\"10\"},\"\",{},\"https://opentelemetry.io/schemas/1.4.0\",\"10\",\"server1\"]]";
        validate_data(
            "otlp_logs_with_selector",
            &client,
            "select * from cus_logs;",
            expected,
        )
        .await;
    }

    {
        // test same selector with multiple value
        let content = r#"
        {"resourceLogs":[{"resource":{"attributes":[{"key":"fromwhere","value":{"stringValue":"resource"}}],"droppedAttributesCount":0},"scopeLogs":[{"scope":{"name":"","version":"","attributes":[{"key":"fromwhere","value":{"stringValue":"scope"}}],"droppedAttributesCount":0},"logRecords":[{"timeUnixNano":"1736413568497632000","observedTimeUnixNano":"0","severityNumber":9,"severityText":"Info","body":{"stringValue":"the message line one"},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"fromwhere","value":{"stringValue":"log_attr"}}],"droppedAttributesCount":0,"flags":0,"traceId":"f665100a612542b69cc362fe2ae9d3bf","spanId":"e58f01c4c69f4488"}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.4.0"}]}
        "#;
        let req: ExportLogsServiceRequest = serde_json::from_str(content).unwrap();
        let body = req.encode_to_vec();
        let res = send_req(
            &client,
            vec![
                (
                    HeaderName::from_static("content-type"),
                    HeaderValue::from_static("application/x-protobuf"),
                ),
                (
                    HeaderName::from_static("x-greptime-log-table-name"),
                    HeaderValue::from_static("logs2"),
                ),
                (
                    HeaderName::from_static("x-greptime-log-extract-keys"),
                    HeaderValue::from_static("fromwhere"),
                ),
            ],
            "/v1/otlp/v1/logs?db=public",
            body.clone(),
            false,
        )
        .await;
        assert_eq!(StatusCode::OK, res.status());

        let expected = "[[1736413568497632000,\"f665100a612542b69cc362fe2ae9d3bf\",\"e58f01c4c69f4488\",\"Info\",9,\"the message line one\",{\"app\":\"server\",\"fromwhere\":\"log_attr\"},0,\"\",\"\",{\"fromwhere\":\"scope\"},\"\",{\"fromwhere\":\"resource\"},\"https://opentelemetry.io/schemas/1.4.0\",\"log_attr\"]]";
        validate_data(
            "otlp_logs_with_selector_overlapping",
            &client,
            "select * from logs2;",
            expected,
        )
        .await;
    }

    {
        let existing_table_name = "otlp_logs_existing_schema";
        let res = execute_sql(
            &client,
            &format!(
                "create table {existing_table_name} (\"timestamp\" timestamp(3) time index, trace_id string, host string, body string, primary key(trace_id, host))"
            ),
        )
        .await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        let req = make_log_request(vec![make_log_record(
            "00000000000000000000000000000001",
            "0000000000000001",
            1_736_413_568_497_632_000,
            "existing schema",
            vec![make_int_attr("host", 42)],
        )]);
        let res = send_log_req(&client, existing_table_name, Some("host"), req, false).await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        validate_data(
            "otlp_logs_existing_schema_rows",
            &client,
            &format!(
                "select trace_id, host, body, \"timestamp\" from {existing_table_name} order by trace_id;"
            ),
            r#"[["00000000000000000000000000000001","42","existing schema",1736413568497]]"#,
        )
        .await;
        validate_data(
            "otlp_logs_existing_schema_semantic_types",
            &client,
            "select column_name, semantic_type from information_schema.columns where table_name = 'otlp_logs_existing_schema' and column_name in ('timestamp', 'trace_id', 'host', 'scope_name') order by column_name;",
            r#"[["host","TAG"],["scope_name","FIELD"],["timestamp","TIMESTAMP"],["trace_id","TAG"]]"#,
        )
        .await;
    }

    {
        let existing_field_table_name = "otlp_logs_existing_string_field";
        let res = execute_sql(
            &client,
            &format!(
                "create table {existing_field_table_name} (\"timestamp\" timestamp(3) time index, host string, body string)"
            ),
        )
        .await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        let req = make_log_request(vec![make_log_record(
            "00000000000000000000000000000004",
            "0000000000000004",
            1_736_413_568_497_700_000,
            "existing string field",
            vec![make_int_attr("host", 42)],
        )]);
        let res = send_log_req(&client, existing_field_table_name, Some("host"), req, false).await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        validate_data(
            "otlp_logs_existing_string_field_rows",
            &client,
            &format!(
                "select host, body, \"timestamp\" from {existing_field_table_name} order by body;"
            ),
            r#"[["42","existing string field",1736413568497]]"#,
        )
        .await;
        validate_data(
            "otlp_logs_existing_string_field_semantic_type",
            &client,
            "select column_name, semantic_type from information_schema.columns where table_name = 'otlp_logs_existing_string_field' and column_name = 'host';",
            r#"[["host","FIELD"]]"#,
        )
        .await;
    }

    {
        let missing_pk_table_name = "otlp_logs_missing_pk";
        let res = execute_sql(
            &client,
            &format!(
                "create table {missing_pk_table_name} (\"timestamp\" timestamp(9) time index, host string, body string, primary key(host))"
            ),
        )
        .await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        let req = make_log_request(vec![make_log_record(
            "00000000000000000000000000000002",
            "0000000000000002",
            1_736_413_568_498_000_000,
            "missing host",
            vec![make_string_attr("not_host", "node-a")],
        )]);
        let res = send_log_req(&client, missing_pk_table_name, None, req, false).await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        validate_data(
            "otlp_logs_missing_pk_rows",
            &client,
            &format!("select host, body from {missing_pk_table_name};"),
            r#"[[null,"missing host"]]"#,
        )
        .await;
    }

    {
        let incompatible_table_name = "otlp_logs_incompatible_schema";
        let res = execute_sql(
            &client,
            &format!(
                "create table {incompatible_table_name} (\"timestamp\" timestamp(9) time index, host bigint, primary key(host))"
            ),
        )
        .await;
        assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

        let req = make_log_request(vec![make_log_record(
            "00000000000000000000000000000003",
            "0000000000000003",
            1_736_413_568_499_000_000,
            "bad host",
            vec![make_string_attr("host", "node-a")],
        )]);
        let res = send_log_req(&client, incompatible_table_name, Some("host"), req, false).await;
        let status = res.status();
        let body = res.text().await;
        assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
        assert!(
            body.contains("failed to align log column 'host'"),
            "unexpected error body: {body}"
        );
        validate_data(
            "otlp_logs_incompatible_schema_rows",
            &client,
            &format!("select count(*) from {incompatible_table_name};"),
            "[[0]]",
        )
        .await;
    }

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_loki_skip_wal(distributed: bool) {
    let mut cases = vec![HttpWalCase::new(
        "Loki JSON",
        "/v1/loki/api/v1/push",
        "application/json",
        r#"{"streams":[{"stream":{"host":"a"},"values":[["1700000000000000000","wal test"]]}]}"#,
    )];
    let loki = loki_proto::logproto::PushRequest {
        streams: vec![loki_proto::logproto::StreamAdapter {
            labels: r#"{host="a"}"#.to_string(),
            entries: vec![loki_proto::logproto::EntryAdapter {
                timestamp: Some(loki_proto::prost_types::Timestamp {
                    seconds: 1700000000,
                    nanos: 0,
                }),
                line: "wal test".to_string(),
                ..Default::default()
            }],
            ..Default::default()
        }],
    };
    cases.push(HttpWalCase::new(
        "Loki protobuf",
        "/v1/loki/api/v1/push",
        "application/x-protobuf",
        prom_store::snappy_compress(&loki.encode_to_vec()).unwrap(),
    ));
    check_http_skip_wal("loki", &cases, distributed).await;
}

pub async fn test_loki_pb_logs(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_loki_pb_logs").await;

    let client = TestClient::new(app).await;

    // init loki request
    let req: PushRequest = PushRequest {
        streams: vec![StreamAdapter {
            labels: r#"{service="test",source="integration",wadaxi="do anything"}"#.to_string(),
            entries: vec![
                EntryAdapter {
                    timestamp: Some(Timestamp::from_str("2024-11-07T10:53:50").unwrap()),
                    line: "this is a log message".to_string(),
                    structured_metadata: vec![
                        LabelPairAdapter {
                            name: "key1".to_string(),
                            value: "value1".to_string(),
                        },
                        LabelPairAdapter {
                            name: "key2".to_string(),
                            value: "value2".to_string(),
                        },
                    ],
                    parsed: vec![],
                },
                EntryAdapter {
                    timestamp: Some(Timestamp::from_str("2024-11-07T10:53:51").unwrap()),
                    line: "this is a log message 2".to_string(),
                    structured_metadata: vec![LabelPairAdapter {
                        name: "key3".to_string(),
                        value: "value3".to_string(),
                    }],
                    parsed: vec![],
                },
                EntryAdapter {
                    timestamp: Some(Timestamp::from_str("2024-11-07T10:53:52").unwrap()),
                    line: "this is a log message 2".to_string(),
                    structured_metadata: vec![],
                    parsed: vec![],
                },
            ],
            hash: rand::random(),
        }],
    };
    let encode = req.encode_to_vec();
    let body = prom_store::snappy_compress(&encode).unwrap();

    // write to loki
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("content-encoding"),
                HeaderValue::from_static("snappy"),
            ),
            (
                HeaderName::from_static("accept-encoding"),
                HeaderValue::from_static("identity"),
            ),
            (
                HeaderName::from_static(GREPTIME_LOG_TABLE_NAME_HEADER_NAME),
                HeaderValue::from_static("loki_table_name"),
            ),
        ],
        "/v1/loki/api/v1/push",
        body,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // test schema
    let expected = "[[\"loki_table_name\",\"CREATE TABLE IF NOT EXISTS \\\"loki_table_name\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(9) NOT NULL,\\n  \\\"line\\\" STRING NULL,\\n  \\\"structured_metadata\\\" JSON NULL,\\n  \\\"service\\\" STRING NULL,\\n  \\\"source\\\" STRING NULL,\\n  \\\"wadaxi\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\"),\\n  PRIMARY KEY (\\\"service\\\", \\\"source\\\", \\\"wadaxi\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true',\\n  'greptime.semantic.signal_type' = 'log',\\n  'greptime.semantic.source' = 'loki'\\n)\"]]";
    validate_data(
        "loki_pb_schema",
        &client,
        "show create table loki_table_name;",
        expected,
    )
    .await;

    // test content
    let expected = "[[1730976830000000000,\"this is a log message\",{\"key1\":\"value1\",\"key2\":\"value2\"},\"test\",\"integration\",\"do anything\"],[1730976831000000000,\"this is a log message 2\",{\"key3\":\"value3\"},\"test\",\"integration\",\"do anything\"],[1730976832000000000,\"this is a log message 2\",{},\"test\",\"integration\",\"do anything\"]]";
    validate_data(
        "loki_pb_content",
        &client,
        "select * from loki_table_name;",
        expected,
    )
    .await;

    // `structured_metadata` is a JSON column, whose SST encoding differs from its
    // memtable representation.
    flush_table(&client, "loki_table_name").await;
    validate_data(
        "loki_pb_content_after_flush",
        &client,
        "select * from loki_table_name;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_loki_pb_logs_with_pipeline(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_loki_pb_logs_with_pipeline").await;

    let client = TestClient::new(app).await;

    let pipeline = r#"
processors:
  - epoch:
      field: greptime_timestamp
      resolution: ms
    "#;

    let res = client
        .post("/v1/pipelines/loki_pipe")
        .header("content-type", "application/x-yaml")
        .body(pipeline)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());

    // init loki request
    let req: PushRequest = PushRequest {
        streams: vec![StreamAdapter {
            labels: r#"{service="test",source="integration",wadaxi="do anything"}"#.to_string(),
            entries: vec![
                EntryAdapter {
                    timestamp: Some(Timestamp::from_str("2024-11-07T10:53:50").unwrap()),
                    line: "this is a log message".to_string(),
                    structured_metadata: vec![
                        LabelPairAdapter {
                            name: "key1".to_string(),
                            value: "value1".to_string(),
                        },
                        LabelPairAdapter {
                            name: "key2".to_string(),
                            value: "value2".to_string(),
                        },
                    ],
                    parsed: vec![],
                },
                EntryAdapter {
                    timestamp: Some(Timestamp::from_str("2024-11-07T10:53:51").unwrap()),
                    line: "this is a log message 2".to_string(),
                    structured_metadata: vec![LabelPairAdapter {
                        name: "key3".to_string(),
                        value: "value3".to_string(),
                    }],
                    parsed: vec![],
                },
                EntryAdapter {
                    timestamp: Some(Timestamp::from_str("2024-11-07T10:53:52").unwrap()),
                    line: "this is a log message 2".to_string(),
                    structured_metadata: vec![],
                    parsed: vec![],
                },
            ],
            hash: rand::random(),
        }],
    };
    let encode = req.encode_to_vec();
    let body = prom_store::snappy_compress(&encode).unwrap();

    // write to loki
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("content-encoding"),
                HeaderValue::from_static("snappy"),
            ),
            (
                HeaderName::from_static("accept-encoding"),
                HeaderValue::from_static("identity"),
            ),
            (
                HeaderName::from_static(GREPTIME_LOG_TABLE_NAME_HEADER_NAME),
                HeaderValue::from_static("loki_table_name"),
            ),
            (
                HeaderName::from_static(GREPTIME_PIPELINE_NAME_HEADER_NAME),
                HeaderValue::from_static("loki_pipe"),
            ),
        ],
        "/v1/loki/api/v1/push",
        body,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // test schema
    // CREATE TABLE IF NOT EXISTS "loki_table_name" (
    //     "greptime_timestamp" TIMESTAMP(3) NOT NULL,
    //     "loki_label_service" STRING NULL,
    //     "loki_label_source" STRING NULL,
    //     "loki_label_wadaxi" STRING NULL,
    //     "loki_line" STRING NULL,
    //     "loki_metadata_key1" STRING NULL,
    //     "loki_metadata_key2" STRING NULL,
    //     "loki_metadata_key3" STRING NULL,
    //     TIME INDEX ("greptime_timestamp")
    //     )
    //   ENGINE=mito
    //   WITH(
    //     'comment' = 'Created on insertion',
    //     append_mode = 'true'
    //   )
    let expected = "[[\"loki_table_name\",\"CREATE TABLE IF NOT EXISTS \\\"loki_table_name\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"loki_label_service\\\" STRING NULL,\\n  \\\"loki_label_source\\\" STRING NULL,\\n  \\\"loki_label_wadaxi\\\" STRING NULL,\\n  \\\"loki_line\\\" STRING NULL,\\n  \\\"loki_metadata_key1\\\" STRING NULL,\\n  \\\"loki_metadata_key2\\\" STRING NULL,\\n  \\\"loki_metadata_key3\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true',\\n  'greptime.semantic.signal_type' = 'log',\\n  'greptime.semantic.source' = 'loki'\\n)\"]]";
    validate_data(
        "loki_pb_schema",
        &client,
        "show create table loki_table_name;",
        expected,
    )
    .await;

    // test content
    let expected = "[[1730976830000,\"test\",\"integration\",\"do anything\",\"this is a log message\",\"value1\",\"value2\",null],[1730976831000,\"test\",\"integration\",\"do anything\",\"this is a log message 2\",null,null,\"value3\"],[1730976832000,\"test\",\"integration\",\"do anything\",\"this is a log message 2\",null,null,null]]";
    validate_data(
        "loki_pb_content",
        &client,
        "select * from loki_table_name;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_loki_json_logs(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_loki_json_logs").await;

    let client = TestClient::new(app).await;

    let body = r#"
{
  "streams": [
    {
      "stream": {
        "source": "test",
        "sender": "integration"
      },
      "values": [
          [ "1735901380059465984", "this is line one", {"key1":"value1","key2":"value2"}],
          [ "1735901398478897920", "this is line two", {"key3":"value3"}],
          [ "1735901398478897921", "this is line two updated"]
      ]
    }
  ]
}
    "#;

    let body = body.as_bytes().to_vec();

    // write plain to loki
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/json"),
            ),
            (
                HeaderName::from_static(GREPTIME_LOG_TABLE_NAME_HEADER_NAME),
                HeaderValue::from_static("loki_table_name"),
            ),
        ],
        "/v1/loki/api/v1/push",
        body,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // test schema
    let expected = "[[\"loki_table_name\",\"CREATE TABLE IF NOT EXISTS \\\"loki_table_name\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(9) NOT NULL,\\n  \\\"line\\\" STRING NULL,\\n  \\\"structured_metadata\\\" JSON NULL,\\n  \\\"sender\\\" STRING NULL,\\n  \\\"source\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\"),\\n  PRIMARY KEY (\\\"sender\\\", \\\"source\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true',\\n  'greptime.semantic.signal_type' = 'log',\\n  'greptime.semantic.source' = 'loki'\\n)\"]]";
    validate_data(
        "loki_json_schema",
        &client,
        "show create table loki_table_name;",
        expected,
    )
    .await;

    // test content
    let expected = "[[1735901380059465984,\"this is line one\",{\"key1\":\"value1\",\"key2\":\"value2\"},\"integration\",\"test\"],[1735901398478897920,\"this is line two\",{\"key3\":\"value3\"},\"integration\",\"test\"],[1735901398478897921,\"this is line two updated\",{},\"integration\",\"test\"]]";
    validate_data(
        "loki_json_content",
        &client,
        "select * from loki_table_name;",
        expected,
    )
    .await;

    // `structured_metadata` is a JSON column, whose SST encoding differs from its
    // memtable representation.
    flush_table(&client, "loki_table_name").await;
    validate_data(
        "loki_json_content_after_flush",
        &client,
        "select * from loki_table_name;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_loki_json_logs_with_pipeline(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_loki_json_logs_with_pipeline").await;

    let client = TestClient::new(app).await;

    let pipeline = r#"
processors:
  - epoch:
      field: greptime_timestamp
      resolution: ms
    "#;

    let res = client
        .post("/v1/pipelines/loki_pipe")
        .header("content-type", "application/x-yaml")
        .body(pipeline)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());

    let body = r#"
{
  "streams": [
    {
      "stream": {
        "source": "test",
        "sender": "integration"
      },
      "values": [
          [ "1735901380059465984", "this is line one", {"key1":"value1","key2":"value2"}],
          [ "1735901398478897920", "this is line two", {"key3":"value3"}],
          [ "1735901398478897921", "this is line two updated"]
      ]
    }
  ]
}
    "#;

    let body = body.as_bytes().to_vec();

    // write plain to loki
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/json"),
            ),
            (
                HeaderName::from_static(GREPTIME_LOG_TABLE_NAME_HEADER_NAME),
                HeaderValue::from_static("loki_table_name"),
            ),
            (
                HeaderName::from_static(GREPTIME_PIPELINE_NAME_HEADER_NAME),
                HeaderValue::from_static("loki_pipe"),
            ),
        ],
        "/v1/loki/api/v1/push",
        body,
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // test schema
    // CREATE TABLE IF NOT EXISTS "loki_table_name" (
    //     "greptime_timestamp" TIMESTAMP(3) NOT NULL,
    //     "loki_label_sender" STRING NULL,
    //     "loki_label_source" STRING NULL,
    //     "loki_line" STRING NULL,
    //     "loki_metadata_key1" STRING NULL,
    //     "loki_metadata_key2" STRING NULL,
    //     "loki_metadata_key3" STRING NULL,
    //     TIME INDEX ("greptime_timestamp")
    //     )
    //   ENGINE=mito
    //   WITH(
    //     'comment' = 'Created on insertion',
    //     append_mode = 'true'
    //   )
    let expected = "[[\"loki_table_name\",\"CREATE TABLE IF NOT EXISTS \\\"loki_table_name\\\" (\\n  \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n  \\\"loki_label_sender\\\" STRING NULL,\\n  \\\"loki_label_source\\\" STRING NULL,\\n  \\\"loki_line\\\" STRING NULL,\\n  \\\"loki_metadata_key1\\\" STRING NULL,\\n  \\\"loki_metadata_key2\\\" STRING NULL,\\n  \\\"loki_metadata_key3\\\" STRING NULL,\\n  TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true',\\n  'greptime.semantic.signal_type' = 'log',\\n  'greptime.semantic.source' = 'loki'\\n)\"]]";
    validate_data(
        "loki_json_schema",
        &client,
        "show create table loki_table_name;",
        expected,
    )
    .await;

    // test content
    let expected = "[[1735901380059,\"integration\",\"test\",\"this is line one\",\"value1\",\"value2\",null],[1735901398478,\"integration\",\"test\",\"this is line two updated\",null,null,null],[1735901398478,\"integration\",\"test\",\"this is line two\",null,null,\"value3\"]]";
    validate_data(
        "loki_json_content",
        &client,
        "select * from loki_table_name;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_elasticsearch_skip_wal(distributed: bool) {
    let cases = vec![HttpWalCase::new(
        "Elasticsearch",
        "/v1/elasticsearch/_bulk",
        "application/json",
        "{\"create\":{\"_index\":\"wal_elastic\"}}\n{\"message\":\"wal test\"}\n",
    )];
    check_http_skip_wal("elasticsearch", &cases, distributed).await;
}

pub async fn test_elasticsearch_logs(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_elasticsearch_logs").await;

    let client = TestClient::new(app).await;

    let body = r#"
        {"create":{"_index":"test","_id":"1"}}
        {"foo":"foo_value1", "bar":"value1"}
        {"create":{"_index":"test","_id":"2"}}
        {"foo":"foo_value2","bar":"value2"}
    "#;

    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/json"),
        )],
        "/v1/elasticsearch/_bulk",
        body.as_bytes().to_vec(),
        false,
    )
    .await;

    assert_eq!(StatusCode::OK, res.status());

    let expected = "[[\"foo_value1\",\"value1\"],[\"foo_value2\",\"value2\"]]";

    validate_data(
        "test_elasticsearch_logs",
        &client,
        "select foo, bar from test;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_elasticsearch_logs_with_index(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_elasticsearch_logs_with_index").await;

    let client = TestClient::new(app).await;

    // It will write to test_index1 and test_index2(specified in the path).
    let body = r#"
        {"create":{"_index":"test_index1","_id":"1"}}
        {"foo":"foo_value1", "bar":"value1"}
        {"create":{"_id":"2"}}
        {"foo":"foo_value2","bar":"value2"}
    "#;

    let res = send_req(
        &client,
        vec![(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/json"),
        )],
        "/v1/elasticsearch/test_index2/_bulk",
        body.as_bytes().to_vec(),
        false,
    )
    .await;

    assert_eq!(StatusCode::OK, res.status());

    // test content of test_index1
    let expected = "[[\"foo_value1\",\"value1\"]]";
    validate_data(
        "test_elasticsearch_logs_with_index",
        &client,
        "select foo, bar from test_index1;",
        expected,
    )
    .await;

    // test content of test_index2
    let expected = "[[\"foo_value2\",\"value2\"]]";
    validate_data(
        "test_elasticsearch_logs_with_index_2",
        &client,
        "select foo, bar from test_index2;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

pub async fn test_log_query(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_log_query").await;

    let client = TestClient::new(app).await;

    // prepare data with SQL API
    let res = client
        .get("/v1/sql?sql=create table logs (`ts` timestamp time index, `message` string fulltext index with(backend='bloom'));")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    let res = client
        .post(
            "/v1/sql?sql=insert into logs values ('2024-11-06 23:59:59', 'before-date-end'), ('2024-11-07 10:53:50', 'before-explicit-end'), ('2024-11-07 10:53:51', 'at-explicit-end'), ('2024-11-07 10:53:52', 'after-explicit-end');",
        )
        .header("Content-Type", "application/x-www-form-urlencoded")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

    // test log query
    let mut log_query = LogQuery {
        table: TableName {
            catalog_name: "greptime".to_string(),
            schema_name: "public".to_string(),
            table_name: "logs".to_string(),
        },
        time_filter: TimeFilter {
            start: Some("2024-11-07".to_string()),
            end: Some("2024-11-07T10:53:51Z".to_string()),
            span: None,
        },
        limit: Limit {
            skip: None,
            fetch: Some(3),
        },
        columns: vec!["ts".to_string(), "message".to_string()],
        filters: Default::default(),
        context: Context::None,
        exprs: vec![],
    };
    let res = client
        .post("/v1/logs")
        .header("Content-Type", "application/json")
        .body(serde_json::to_string(&log_query).unwrap())
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    let resp = res.text().await;
    let v = get_rows_from_output(&resp);
    assert_eq!(v, "[[1730976830000,\"before-explicit-end\"]]");

    log_query.time_filter.start = Some("2024-11-06".to_string());
    log_query.time_filter.end = Some("2024-11-07".to_string());
    log_query.limit.fetch = Some(4);
    let res = client
        .post("/v1/logs")
        .header("Content-Type", "application/json")
        .body(serde_json::to_string(&log_query).unwrap())
        .send()
        .await;

    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    let resp = res.text().await;
    let output = serde_json::from_str::<Value>(&resp).unwrap();
    let rows = output["output"][0]["records"]["rows"].as_array().unwrap();
    let mut messages = rows
        .iter()
        .map(|row| row[1].as_str().unwrap())
        .collect::<Vec<_>>();
    messages.sort_unstable();
    assert_eq!(
        messages,
        vec![
            "after-explicit-end",
            "at-explicit-end",
            "before-date-end",
            "before-explicit-end",
        ]
    );

    // The fulltext index file is only written on flush. Replay the log query to
    // cover the SST read path, and add a `matches_term` query, which is the
    // predicate the index applier actually recognises.
    flush_table(&client, "logs").await;
    let res = client
        .post("/v1/logs")
        .header("Content-Type", "application/json")
        .body(serde_json::to_string(&log_query).unwrap())
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    let resp = res.text().await;
    let output = serde_json::from_str::<Value>(&resp).unwrap();
    let rows = output["output"][0]["records"]["rows"].as_array().unwrap();
    let mut flushed_messages = rows
        .iter()
        .map(|row| row[1].as_str().unwrap())
        .collect::<Vec<_>>();
    flushed_messages.sort_unstable();
    assert_eq!(flushed_messages, messages);

    validate_data(
        "log_query_fulltext_index",
        &client,
        "select message from logs where matches_term(message, 'at-explicit-end')",
        r#"[["at-explicit-end"]]"#,
    )
    .await;

    let res = client
        .get("/v1/sql?sql=create table logs_limit (`ts` timestamp time index, `message` string);")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    let res = client
        .get("/v1/sql?sql=insert into logs_limit select number, 'hello' from numbers limit 1001;")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);

    let aggregate_query = LogQuery {
        table: TableName::new("greptime", "public", "logs_limit"),
        time_filter: TimeFilter {
            start: Some("1970-01-01T00:00:00Z".to_string()),
            end: Some("1970-01-01T00:00:02Z".to_string()),
            span: None,
        },
        limit: Limit {
            skip: None,
            fetch: None,
        },
        columns: vec![],
        filters: Default::default(),
        context: Context::None,
        exprs: vec![LogExpr::AggrFunc {
            expr: vec![AggFunc::new(
                "count".to_string(),
                vec![LogExpr::NamedIdent("message".to_string())],
                Some("count_result".to_string()),
            )],
            by: vec![],
        }],
    };
    let res = client
        .post("/v1/logs")
        .header("Content-Type", "application/json")
        .body(serde_json::to_string(&aggregate_query).unwrap())
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    assert_eq!(get_rows_from_output(&res.text().await), "[[1001]]");

    let mut output_query = LogQuery {
        table: TableName::new("greptime", "public", "logs_limit"),
        time_filter: TimeFilter {
            start: Some("1970-01-01T00:00:00Z".to_string()),
            end: Some("1970-01-01T00:00:02Z".to_string()),
            span: None,
        },
        limit: Limit {
            skip: None,
            fetch: None,
        },
        columns: vec!["ts".to_string(), "message".to_string()],
        filters: Default::default(),
        context: Context::None,
        exprs: vec![],
    };
    let res = client
        .post("/v1/logs")
        .header("Content-Type", "application/json")
        .body(serde_json::to_string(&output_query).unwrap())
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    assert_eq!(get_output_row_count(&res.text().await), 1000);

    output_query.limit.fetch = Some(17);
    let res = client
        .post("/v1/logs")
        .header("Content-Type", "application/json")
        .body(serde_json::to_string(&output_query).unwrap())
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await);
    assert_eq!(get_output_row_count(&res.text().await), 17);

    guard.remove_all().await;
}

pub async fn test_jaeger_query_api(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_jaeger_query_api").await;

    let client = TestClient::new(app).await;

    // Test empty response for `/api/services` API before writing any traces.
    let res = client.get("/v1/jaeger/api/services").send().await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": null,
        "total": 0,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    let content = r#"
    {
        "resourceSpans": [
            {
                "resource": {
                    "attributes": [
                        {
                            "key": "service.name",
                            "value": {
                                "stringValue": "test-jaeger-query-api"
                            }
                        }
                    ]
                },
                "scopeSpans": [
                    {
                        "scope": {
                        "name": "test-jaeger-query-api",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08ea",
                                "spanId": "008421dbbd33a3e9",
                                "name": "access-mysql",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754492421000",
                                "endTimeUnixNano": "1738726754592421000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-mysql"
                                        }
                                    },
                                    {
                                        "key": "net.peer.ip",
                                        "value": {
                                        "stringValue": "1.2.3.4"
                                        }
                                    },
                                    {
                                        "key": "peer.service",
                                        "value": {
                                        "stringValue": "test-jaeger-query-api"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    },
                    {
                        "scope": {
                        "name": "test-jaeger-query-api",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08ea",
                                "spanId": "ffa03416a7b9ea48",
                                "name": "access-redis",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754492422000",
                                "endTimeUnixNano": "1738726754592422000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-redis"
                                        }
                                    },
                                    {
                                        "key": "net.peer.ip",
                                        "value": {
                                        "stringValue": "1.2.3.4"
                                        }
                                    },
                                    {
                                        "key": "peer.service",
                                        "value": {
                                        "stringValue": "test-jaeger-query-api"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    }
                ],
                "schemaUrl": "https://opentelemetry.io/schemas/1.4.0"
            }
        ]
    }
    "#;

    let req: ExportTraceServiceRequest = serde_json::from_str(content).unwrap();
    let body = req.encode_to_vec();

    // write traces data.
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static("greptime_trace_v0"),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    // Test `/api/services` API.
    let res = client.get("/v1/jaeger/api/services").send().await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": [
            "test-jaeger-query-api"
        ],
        "total": 1,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/operations` API.
    let res = client
        .get("/v1/jaeger/api/operations?service=test-jaeger-query-api&start=1738726754492421&end=1738726754642422")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": [
            {
                "name": "access-mysql",
                "spanKind": "server"
            },
            {
                "name": "access-redis",
                "spanKind": "server"
            }
        ],
        "total": 2,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/services/{service_name}/operations` API.
    let res = client
        .get("/v1/jaeger/api/services/test-jaeger-query-api/operations?start=1738726754492421&end=1738726754642422")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": [
            "access-mysql",
            "access-redis"
        ],
        "total": 2,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces/{trace_id}` API.
    let res = client
        .get("/v1/jaeger/api/traces/5611dce1bc9ebed65352d99a027b08ea")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
{
  "data": [
    {
      "traceID": "5611dce1bc9ebed65352d99a027b08ea",
      "spans": [
        {
          "traceID": "5611dce1bc9ebed65352d99a027b08ea",
          "spanID": "ffa03416a7b9ea48",
          "operationName": "access-redis",
          "references": [],
          "startTime": 1738726754492422,
          "duration": 100000,
          "tags": [
            {
              "key": "net.peer.ip",
              "type": "string",
              "value": "1.2.3.4"
            },
            {
              "key": "operation.type",
              "type": "string",
              "value": "access-redis"
            },
            {
              "key": "otel.scope.name",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "otel.scope.version",
              "type": "string",
              "value": "1.0.0"
            },
            {
              "key": "otel.status_description",
              "type": "string",
              "value": "success"
            },
            {
              "key": "peer.service",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "span.kind",
              "type": "string",
              "value": "server"
            }
          ],
          "logs": [],
          "processID": "p1"
        },
        {
          "traceID": "5611dce1bc9ebed65352d99a027b08ea",
          "spanID": "008421dbbd33a3e9",
          "operationName": "access-mysql",
          "references": [],
          "startTime": 1738726754492421,
          "duration": 100000,
          "tags": [
            {
              "key": "net.peer.ip",
              "type": "string",
              "value": "1.2.3.4"
            },
            {
              "key": "operation.type",
              "type": "string",
              "value": "access-mysql"
            },
            {
              "key": "otel.scope.name",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "otel.scope.version",
              "type": "string",
              "value": "1.0.0"
            },
            {
              "key": "otel.status_description",
              "type": "string",
              "value": "success"
            },
            {
              "key": "peer.service",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "span.kind",
              "type": "string",
              "value": "server"
            }
          ],
          "logs": [],
          "processID": "p1"
        }
      ],
      "processes": {
        "p1": {
          "serviceName": "test-jaeger-query-api",
          "tags": []
        }
      }
    }
  ],
  "total": 0,
  "limit": 0,
  "offset": 0,
  "errors": []
}
    "#;

    let resp_txt = &res.text().await;
    let resp: Value = serde_json::from_str(resp_txt).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces` API.
    let res = client
        .get("/v1/jaeger/api/traces?service=test-jaeger-query-api&operation=access-mysql&start=1738726754492421&end=1738726754642422&tags=%7B%22operation.type%22%3A%22access-mysql%22%7D")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
{
    "data":
    [
        {
            "traceID": "5611dce1bc9ebed65352d99a027b08ea",
            "spans":
            [
                {
                    "traceID": "5611dce1bc9ebed65352d99a027b08ea",
                    "spanID": "ffa03416a7b9ea48",
                    "operationName": "access-redis",
                    "references":
                    [],
                    "startTime": 1738726754492422,
                    "duration": 100000,
                    "tags":
                    [
                        {
                            "key": "net.peer.ip",
                            "type": "string",
                            "value": "1.2.3.4"
                        },
                        {
                            "key": "operation.type",
                            "type": "string",
                            "value": "access-redis"
                        },
                        {
                            "key": "otel.scope.name",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "otel.scope.version",
                            "type": "string",
                            "value": "1.0.0"
                        },
                        {
                            "key": "otel.status_description",
                            "type": "string",
                            "value": "success"
                        },
                        {
                            "key": "peer.service",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "span.kind",
                            "type": "string",
                            "value": "server"
                        }
                    ],
                    "logs":
                    [],
                    "processID": "p1"
                },
                {
                    "traceID": "5611dce1bc9ebed65352d99a027b08ea",
                    "spanID": "008421dbbd33a3e9",
                    "operationName": "access-mysql",
                    "references":
                    [],
                    "startTime": 1738726754492421,
                    "duration": 100000,
                    "tags":
                    [
                        {
                            "key": "net.peer.ip",
                            "type": "string",
                            "value": "1.2.3.4"
                        },
                        {
                            "key": "operation.type",
                            "type": "string",
                            "value": "access-mysql"
                        },
                        {
                            "key": "otel.scope.name",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "otel.scope.version",
                            "type": "string",
                            "value": "1.0.0"
                        },
                        {
                            "key": "otel.status_description",
                            "type": "string",
                            "value": "success"
                        },
                        {
                            "key": "peer.service",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "span.kind",
                            "type": "string",
                            "value": "server"
                        }
                    ],
                    "logs":
                    [],
                    "processID": "p1"
                }
            ],
            "processes":
            {
                "p1":
                {
                    "serviceName": "test-jaeger-query-api",
                    "tags":
                    []
                }
            }
        }
    ],
    "total": 0,
    "limit": 0,
    "offset": 0,
    "errors":
    []
}
    "#;

    let resp_txt = &res.text().await;
    let resp: Value = serde_json::from_str(resp_txt).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    guard.remove_all().await;
}

pub async fn test_jaeger_query_api_for_trace_v2(
    store_type: StorageType,
) -> Result<(), Box<dyn std::error::Error>> {
    common_telemetry::init_default_ut_logging();

    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_jaeger_query_api_v2").await;
    let client = TestClient::new(app).await;
    let request: ExportTraceServiceRequest = serde_json::from_value(json!({
        "resourceSpans": [{
            "resource": {"attributes": [
                make_string_attr("service.name", "jaeger-v2"),
                make_string_attr("region", "west"),
                make_string_attr("shared", "resource"),
                make_string_attr("nullable", "resource"),
                make_int_attr("count", 7),
                make_int_attr("complex", 7)
            ]},
            "scopeSpans": [{
                "scope": {"name": "v2-scope", "version": "1.0"},
                "spans": [{
                    "traceId": "c05d7a4ec8e1f231f02ed6e8da8655b4",
                    "spanId": "9630f2916e2f7909", "parentSpanId": "8f847259b0f6e1ab",
                    "name": "op", "kind": 2,
                    "startTimeUnixNano": "1736480942444376000",
                    "endTimeUnixNano": "1736480942444499000",
                    "attributes": [
                        make_string_attr("shared", "span"),
                        {"key": "nullable"},
                        make_string_attr("a\"b\\c.d", "escaped"),
                        make_int_attr("http.status_code", 500),
                        make_bool_attr("enabled", true),
                        {"key": "ratio", "value": {"doubleValue": 1.5}},
                        make_string_attr("count", "invalid"),
                        {"key": "complex", "value": {"arrayValue": {"values": [{"intValue":"7"}]}}}
                    ],
                    "events": [{"name": "cache.hit", "timeUnixNano": "1736480942444400000",
                        "attributes": [make_int_attr("event.code", 7)]}],
                    "links": [{"traceId": "cc9e0991a2e63d274984bd44ee669203",
                        "spanId": "8f847259b0f6e1ab"}],
                    "status": {"code": 2, "message": "failure"}
                }, {
                    "traceId": "cc9e0991a2e63d274984bd44ee669203",
                    "spanId": "8f847259b0f6e1ab", "name": "op", "kind": 2,
                    "startTimeUnixNano": "1736480942444376000",
                    "endTimeUnixNano": "1736480942444499000"
                }]
            }]
        }]
    }))?;
    let response = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V2_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static("jaeger_v2"),
            ),
        ],
        "/v1/otlp/v1/traces",
        request.encode_to_vec(),
        false,
    )
    .await;
    assert_eq!(response.status(), StatusCode::OK);

    for (path, expected) in [
        ("/v1/jaeger/api/services", json!(["jaeger-v2"])),
        (
            "/v1/jaeger/api/operations?service=jaeger-v2",
            json!([{"name":"op", "spanKind":"server"}]),
        ),
        (
            "/v1/jaeger/api/services/jaeger-v2/operations",
            json!(["op"]),
        ),
    ] {
        let response = client
            .get(path)
            .header("x-greptime-trace-table-name", "jaeger_v2")
            .send()
            .await;
        assert_eq!(response.status(), StatusCode::OK);
        let body: Value = serde_json::from_str(&response.text().await)?;
        assert_eq!(body["data"], expected, "{path}: {body}");
    }

    for flushed in [false, true] {
        if flushed {
            let response = client
                .post("/v1/sql")
                .form(&[("sql", "ADMIN FLUSH_TABLE('jaeger_v2')")])
                .send()
                .await;
            assert_eq!(response.status(), StatusCode::OK);
        }
        for (tags, expected) in [
            (json!({"shared":"span"}), 1),
            (json!({"shared":"resource"}), 1),
            (json!({"region":"west"}), 2),
            (json!({"nullable":"resource"}), 2),
            (json!({"count":7}), 2),
            (json!({"complex":7}), 2),
            (json!({"a\"b\\c.d":"escaped"}), 1),
            (
                json!({"http.status_code":500, "enabled":true, "ratio":1.5}),
                1,
            ),
            (json!({"error":true}), 1),
            (json!({"missing":"absent"}), 0),
            (json!({"missing":null}), 2),
            (json!({"count":u64::MAX}), 0),
        ] {
            let url = format!(
                "/v1/jaeger/api/traces?service=jaeger-v2&start=1736480942444376&end=1736480942444500&tags={}",
                encode(&tags.to_string())
            );
            let response = client
                .get(&url)
                .header("user-agent", if flushed { "Grafana" } else { "Jaeger" })
                .header("x-greptime-trace-table-name", "jaeger_v2")
                .send()
                .await;
            let status = response.status();
            let body: Value = serde_json::from_str(&response.text().await)?;
            assert_eq!(status, StatusCode::OK, "{tags}: {body}");
            assert_eq!(
                body["data"].as_array().map_or(0, Vec::len),
                expected,
                "{tags}: {body}"
            );
        }
        let response = client
            .get("/v1/jaeger/api/traces/c05d7a4ec8e1f231f02ed6e8da8655b4")
            .header("x-greptime-trace-table-name", "jaeger_v2")
            .send()
            .await;
        assert_eq!(response.status(), StatusCode::OK);
        let body: Value = serde_json::from_str(&response.text().await)?;
        let process = &body["data"][0]["processes"]["p1"];
        assert_eq!(process["serviceName"], "jaeger-v2");
        assert!(
            process["tags"]
                .as_array()
                .ok_or("Expected process tags")?
                .iter()
                .any(|tag| tag["key"] == "region" && tag["value"] == "west")
        );
        let span = &body["data"][0]["spans"][0];
        assert_eq!(span["startTime"], 1736480942444376u64);
        assert_eq!(span["duration"], 123);
        assert_eq!(
            span["references"],
            json!([
                {"refType":"CHILD_OF", "traceID":"c05d7a4ec8e1f231f02ed6e8da8655b4", "spanID":"8f847259b0f6e1ab"},
                {"refType":"FOLLOWS_FROM", "traceID":"cc9e0991a2e63d274984bd44ee669203", "spanID":"8f847259b0f6e1ab"},
            ])
        );
        assert_eq!(span["logs"][0]["timestamp"], 1736480942444400u64);
        assert!(
            span["logs"][0]["fields"]
                .as_array()
                .ok_or("Expected event fields")?
                .iter()
                .any(|field| field["key"] == "event.code" && field["value"] == 7)
        );
        assert!(
            span["tags"]
                .as_array()
                .ok_or("Expected span tags")?
                .iter()
                .any(|tag| tag["key"] == "http.status_code"
                    && tag["type"] == "int64"
                    && tag["value"] == 500)
        );
    }
    guard.remove_all().await;
    Ok(())
}

pub async fn test_jaeger_query_api_for_trace_v1(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_jaeger_query_api_v1").await;

    let client = TestClient::new(app).await;

    // Test empty response for `/api/services` API before writing any traces.
    let res = client.get("/v1/jaeger/api/services").send().await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": null,
        "total": 0,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value = serde_json::from_str(expected).unwrap();
    assert_eq!(resp, expected);

    let content = r#"
    {
        "resourceSpans": [
            {
                "resource": {
                    "attributes": [
                        {
                            "key": "service.name",
                            "value": {
                                "stringValue": "test-jaeger-query-api"
                            }
                        }
                    ]
                },
                "scopeSpans": [
                    {
                        "scope": {
                        "name": "test-jaeger-query-api",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08ea",
                                "spanId": "008421dbbd33a3e9",
                                "name": "access-mysql",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754492421000",
                                "endTimeUnixNano": "1738726754592421000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-mysql"
                                        }
                                    },
                                    {
                                        "key": "net.peer.ip",
                                        "value": {
                                        "stringValue": "1.2.3.4"
                                        }
                                    },
                                    {
                                        "key": "peer.service",
                                        "value": {
                                        "stringValue": "test-jaeger-query-api"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    },
                    {
                        "scope": {
                        "name": "test-jaeger-query-api",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08ea",
                                "spanId": "ffa03416a7b9ea48",
                                "name": "access-redis",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754492422000",
                                "endTimeUnixNano": "1738726754592422000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-redis"
                                        }
                                    },
                                    {
                                        "key": "net.peer.ip",
                                        "value": {
                                        "stringValue": "1.2.3.4"
                                        }
                                    },
                                    {
                                        "key": "peer.service",
                                        "value": {
                                        "stringValue": "test-jaeger-query-api"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    },
                    {
                        "scope": {
                        "name": "test-jaeger-get-operations",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08ff",
                                "spanId": "ffa03416a7b9ea48",
                                "name": "access-pg",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754492422000",
                                "endTimeUnixNano": "1738726754592422000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-pg"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    },
                    {
                        "scope": {
                        "name": "test-jaeger-find-traces",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08fa",
                                "spanId": "ffa03416a7b9ea49",
                                "name": "access-pg",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754492423000",
                                "endTimeUnixNano": "1738726754592423000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-pg"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    },
                    {
                        "scope": {
                        "name": "test-jaeger-grafana-ua",
                        "version": "1.0.0"
                        },
                        "spans": [
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08fb",
                                "spanId": "ffa03416a7b9ea50",
                                "name": "access-span-1",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754600000000",
                                "endTimeUnixNano": "1738726754700000000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-span-1"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            },
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08fb",
                                "spanId": "ffa03416a7b9ea51",
                                "name": "access-span-2",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754600001000",
                                "endTimeUnixNano": "1738726754700001000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-span-2"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            },
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08fb",
                                "spanId": "ffa03416a7b9ea52",
                                "name": "access-span-3",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754600002000",
                                "endTimeUnixNano": "1738726754700002000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-span-3"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            },
                            {
                                "traceId": "5611dce1bc9ebed65352d99a027b08fb",
                                "spanId": "ffa03416a7b9ea53",
                                "name": "access-span-4",
                                "kind": 2,
                                "startTimeUnixNano": "1738726754600003000",
                                "endTimeUnixNano": "1738726754700003000",
                                "attributes": [
                                    {
                                        "key": "operation.type",
                                        "value": {
                                        "stringValue": "access-span-4"
                                        }
                                    }
                                ],
                                "status": {
                                    "message": "success",
                                    "code": 0
                                }
                            }
                        ]
                    }
                ],
                "schemaUrl": "https://opentelemetry.io/schemas/1.4.0"
            }
        ]
    }
    "#;

    let mut req: ExportTraceServiceRequest = serde_json::from_str(content).unwrap();
    // The fixture is written against a fixed base instant. Move it next to now:
    // this table is created with `ttl=7d`, and stale spans would be dropped as
    // expired the moment they are flushed into SST files.
    let now = Utc::now().timestamp_nanos_opt().unwrap() as u64;
    let delta_micros = (now / 1_000) as i64 - 60_000_000 - JAEGER_V1_FIXTURE_BASE_MICROS as i64;
    let delta_nanos = delta_micros * 1_000;
    for span in req.resource_spans.iter_mut() {
        for scope_span in span.scope_spans.iter_mut() {
            for span in scope_span.spans.iter_mut() {
                span.start_time_unix_nano = (span.start_time_unix_nano as i64 + delta_nanos) as u64;
                span.end_time_unix_nano = (span.end_time_unix_nano as i64 + delta_nanos) as u64;
            }
            // `/api/operations` is queried without a time range, so these spans
            // stay pinned to the last few seconds.
            if scope_span.scope.as_ref().unwrap().name == "test-jaeger-get-operations" {
                for span in scope_span.spans.iter_mut() {
                    span.start_time_unix_nano = now - 5_000_000_000; // 5 seconds ago
                    span.end_time_unix_nano = now;
                }
            }
        }
    }
    let body = req.encode_to_vec();

    let trace_table_name = "mytable";
    // write traces data.
    let res = send_req(
        &client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-log-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-hints"),
                HeaderValue::from_static("ttl=7d"),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_static(trace_table_name),
            ),
        ],
        "/v1/otlp/v1/traces",
        body.clone(),
        false,
    )
    .await;
    assert_eq!(StatusCode::OK, res.status());

    let trace_table_sql = "[[\"mytable\",\"CREATE TABLE IF NOT EXISTS \\\"mytable\\\" (\\n  \\\"timestamp\\\" TIMESTAMP(9) NOT NULL,\\n  \\\"timestamp_end\\\" TIMESTAMP(9) NULL,\\n  \\\"duration_nano\\\" BIGINT NULL,\\n  \\\"parent_span_id\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n  \\\"trace_id\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n  \\\"span_id\\\" STRING NULL,\\n  \\\"span_kind\\\" STRING NULL,\\n  \\\"span_name\\\" STRING NULL,\\n  \\\"span_status_code\\\" STRING NULL,\\n  \\\"span_status_message\\\" STRING NULL,\\n  \\\"trace_state\\\" STRING NULL,\\n  \\\"scope_name\\\" STRING NULL,\\n  \\\"scope_version\\\" STRING NULL,\\n  \\\"service_name\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n  \\\"span_attributes.operation.type\\\" STRING NULL,\\n  \\\"span_attributes.net.peer.ip\\\" STRING NULL,\\n  \\\"span_attributes.peer.service\\\" STRING NULL,\\n  \\\"span_events\\\" JSON NULL,\\n  \\\"span_links\\\" JSON NULL,\\n  TIME INDEX (\\\"timestamp\\\"),\\n  PRIMARY KEY (\\\"service_name\\\")\\n)\\nPARTITION ON COLUMNS (\\\"trace_id\\\") (\\n  trace_id < '1',\\n  trace_id >= '1' AND trace_id < '2',\\n  trace_id >= '2' AND trace_id < '3',\\n  trace_id >= '3' AND trace_id < '4',\\n  trace_id >= '4' AND trace_id < '5',\\n  trace_id >= '5' AND trace_id < '6',\\n  trace_id >= '6' AND trace_id < '7',\\n  trace_id >= '7' AND trace_id < '8',\\n  trace_id >= '8' AND trace_id < '9',\\n  trace_id >= '9' AND trace_id < 'a',\\n  trace_id >= 'a' AND trace_id < 'b',\\n  trace_id >= 'b' AND trace_id < 'c',\\n  trace_id >= 'c' AND trace_id < 'd',\\n  trace_id >= 'd' AND trace_id < 'e',\\n  trace_id >= 'e' AND trace_id < 'f',\\n  trace_id >= 'f'\\n)\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'true',\\n  'greptime.semantic.entity.service.id' = 'service_name',\\n  'greptime.semantic.pipeline' = 'greptime_trace_v1',\\n  'greptime.semantic.signal_type' = 'trace',\\n  'greptime.semantic.source' = 'opentelemetry',\\n  'greptime.semantic.trace.conventions' = 'https://opentelemetry.io/schemas/1.4.0',\\n  table_data_model = 'greptime_trace_v1',\\n  ttl = '7days'\\n)\"]]";
    validate_data(
        "trace_v1_create_table",
        &client,
        "show create table mytable;",
        trace_table_sql,
    )
    .await;

    let trace_meta_table_sql = "[[\"mytable_services\",\"CREATE TABLE IF NOT EXISTS \\\"mytable_services\\\" (\\n  \\\"timestamp\\\" TIMESTAMP(9) NOT NULL,\\n  \\\"service_name\\\" STRING NULL,\\n  TIME INDEX (\\\"timestamp\\\"),\\n  PRIMARY KEY (\\\"service_name\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n  'comment' = 'Created on insertion',\\n  append_mode = 'false'\\n)\"]]";
    validate_data(
        "trace_v1_create_meta_table",
        &client,
        "show create table mytable_services;",
        trace_meta_table_sql,
    )
    .await;

    // Test `/api/services` API.
    let res = client
        .get("/v1/jaeger/api/services")
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": [
            "test-jaeger-query-api"
        ],
        "total": 1,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/operations` API.
    let res = client
        .get("/v1/jaeger/api/operations?service=test-jaeger-query-api")
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": [
            {
                "name": "access-mysql",
                "spanKind": "server"
            },
            {
                "name": "access-pg",
                "spanKind": "server"
            },
            {
                "name": "access-redis",
                "spanKind": "server"
            },
            {
                "name": "access-span-1",
                "spanKind": "server"
            },
            {
                "name": "access-span-2",
                "spanKind": "server"
            },
            {
                "name": "access-span-3",
                "spanKind": "server"
            },
            {
                "name": "access-span-4",
                "spanKind": "server"
            }
        ],
        "total": 7,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/services/{service_name}/operations` API.
    let res = client
        .get(&shift_fixture_micros("/v1/jaeger/api/services/test-jaeger-query-api/operations?start=1738726754492421&end=1738726754642422", delta_micros))
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
    {
        "data": [
            "access-mysql",
            "access-pg",
            "access-redis",
            "access-span-1",
            "access-span-2",
            "access-span-3",
            "access-span-4"
        ],
        "total": 7,
        "limit": 0,
        "offset": 0,
        "errors": []
    }
    "#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces/{trace_id}` API without start and end.
    let res = client
        .get("/v1/jaeger/api/traces/5611dce1bc9ebed65352d99a027b08ea")
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"{
  "data": [
    {
      "traceID": "5611dce1bc9ebed65352d99a027b08ea",
      "spans": [
        {
          "traceID": "5611dce1bc9ebed65352d99a027b08ea",
          "spanID": "ffa03416a7b9ea48",
          "operationName": "access-redis",
          "references": [],
          "startTime": 1738726754492422,
          "duration": 100000,
          "tags": [
            {
              "key": "net.peer.ip",
              "type": "string",
              "value": "1.2.3.4"
            },
            {
              "key": "operation.type",
              "type": "string",
              "value": "access-redis"
            },
            {
              "key": "otel.scope.name",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "otel.scope.version",
              "type": "string",
              "value": "1.0.0"
            },
            {
              "key": "otel.status_description",
              "type": "string",
              "value": "success"
            },
            {
              "key": "peer.service",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "span.kind",
              "type": "string",
              "value": "server"
            }
          ],
          "logs": [],
          "processID": "p1"
        },
        {
          "traceID": "5611dce1bc9ebed65352d99a027b08ea",
          "spanID": "008421dbbd33a3e9",
          "operationName": "access-mysql",
          "references": [],
          "startTime": 1738726754492421,
          "duration": 100000,
          "tags": [
            {
              "key": "net.peer.ip",
              "type": "string",
              "value": "1.2.3.4"
            },
            {
              "key": "operation.type",
              "type": "string",
              "value": "access-mysql"
            },
            {
              "key": "otel.scope.name",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "otel.scope.version",
              "type": "string",
              "value": "1.0.0"
            },
            {
              "key": "otel.status_description",
              "type": "string",
              "value": "success"
            },
            {
              "key": "peer.service",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "span.kind",
              "type": "string",
              "value": "server"
            }
          ],
          "logs": [],
          "processID": "p1"
        }
      ],
      "processes": {
        "p1": {
          "serviceName": "test-jaeger-query-api",
          "tags": []
        }
      }
    }
  ],
  "total": 0,
  "limit": 0,
  "offset": 0,
  "errors": []
}
"#;

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Replay the same lookup against SST files: `trace_id` and `service_name`
    // carry BLOOM skipping indexes whose puffin files only exist after a flush,
    // and `span_events` / `span_links` are JSON columns.
    flush_table(&client, trace_table_name).await;
    let res = client
        .get("/v1/jaeger/api/traces/5611dce1bc9ebed65352d99a027b08ea")
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces/{trace_id}` API with start and end in microseconds.
    let res = client
        .get(&shift_fixture_micros("/v1/jaeger/api/traces/5611dce1bc9ebed65352d99a027b08ea?start=1738726754492421&end=1738726754642422", delta_micros))
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"{
  "data": [
    {
      "traceID": "5611dce1bc9ebed65352d99a027b08ea",
      "spans": [
        {
          "traceID": "5611dce1bc9ebed65352d99a027b08ea",
          "spanID": "ffa03416a7b9ea48",
          "operationName": "access-redis",
          "references": [],
          "startTime": 1738726754492422,
          "duration": 100000,
          "tags": [
            {
              "key": "net.peer.ip",
              "type": "string",
              "value": "1.2.3.4"
            },
            {
              "key": "operation.type",
              "type": "string",
              "value": "access-redis"
            },
            {
              "key": "otel.scope.name",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "otel.scope.version",
              "type": "string",
              "value": "1.0.0"
            },
            {
              "key": "otel.status_description",
              "type": "string",
              "value": "success"
            },
            {
              "key": "peer.service",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "span.kind",
              "type": "string",
              "value": "server"
            }
          ],
          "logs": [],
          "processID": "p1"
        },
        {
          "traceID": "5611dce1bc9ebed65352d99a027b08ea",
          "spanID": "008421dbbd33a3e9",
          "operationName": "access-mysql",
          "references": [],
          "startTime": 1738726754492421,
          "duration": 100000,
          "tags": [
            {
              "key": "net.peer.ip",
              "type": "string",
              "value": "1.2.3.4"
            },
            {
              "key": "operation.type",
              "type": "string",
              "value": "access-mysql"
            },
            {
              "key": "otel.scope.name",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "otel.scope.version",
              "type": "string",
              "value": "1.0.0"
            },
            {
              "key": "otel.status_description",
              "type": "string",
              "value": "success"
            },
            {
              "key": "peer.service",
              "type": "string",
              "value": "test-jaeger-query-api"
            },
            {
              "key": "span.kind",
              "type": "string",
              "value": "server"
            }
          ],
          "logs": [],
          "processID": "p1"
        }
      ],
      "processes": {
        "p1": {
          "serviceName": "test-jaeger-query-api",
          "tags": []
        }
      }
    }
  ],
  "total": 0,
  "limit": 0,
  "offset": 0,
  "errors": []
}
"#;

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces/{trace_id}` API for non-existent trace.
    let res = client
        .get(&shift_fixture_micros(
            "/v1/jaeger/api/traces/0000000000000000000000000000dead",
            delta_micros,
        ))
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::NOT_FOUND, res.status());
    let expected = r#"{
  "data": null,
  "total": 0,
  "limit": 0,
  "offset": 0,
  "errors": [
    {
      "code": 404,
      "msg": "trace not found"
    }
  ]
}
"#;
    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces` API.
    let res = client
        .get(&shift_fixture_micros("/v1/jaeger/api/traces?service=test-jaeger-query-api&operation=access-mysql&start=1738726754492421&end=1738726754642422&tags=%7B%22operation.type%22%3A%22access-mysql%22%7D", delta_micros))
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());
    let expected = r#"
 {
    "data": [
        {
            "traceID": "5611dce1bc9ebed65352d99a027b08ea",
            "spans": [
                {
                    "traceID": "5611dce1bc9ebed65352d99a027b08ea",
                    "spanID": "ffa03416a7b9ea48",
                    "operationName": "access-redis",
                    "references": [],
                    "startTime": 1738726754492422,
                    "duration": 100000,
                    "tags": [
                        {
                            "key": "net.peer.ip",
                            "type": "string",
                            "value": "1.2.3.4"
                        },
                        {
                            "key": "operation.type",
                            "type": "string",
                            "value": "access-redis"
                        },
                        {
                            "key": "otel.scope.name",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "otel.scope.version",
                            "type": "string",
                            "value": "1.0.0"
                        },
                        {
                            "key": "otel.status_description",
                            "type": "string",
                            "value": "success"
                        },
                        {
                            "key": "peer.service",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "span.kind",
                            "type": "string",
                            "value": "server"
                        }
                    ],
                    "logs": [],
                    "processID": "p1"
                },
                {
                    "traceID": "5611dce1bc9ebed65352d99a027b08ea",
                    "spanID": "008421dbbd33a3e9",
                    "operationName": "access-mysql",
                    "references": [],
                    "startTime": 1738726754492421,
                    "duration": 100000,
                    "tags": [
                        {
                            "key": "net.peer.ip",
                            "type": "string",
                            "value": "1.2.3.4"
                        },
                        {
                            "key": "operation.type",
                            "type": "string",
                            "value": "access-mysql"
                        },
                        {
                            "key": "otel.scope.name",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "otel.scope.version",
                            "type": "string",
                            "value": "1.0.0"
                        },
                        {
                            "key": "otel.status_description",
                            "type": "string",
                            "value": "success"
                        },
                        {
                            "key": "peer.service",
                            "type": "string",
                            "value": "test-jaeger-query-api"
                        },
                        {
                            "key": "span.kind",
                            "type": "string",
                            "value": "server"
                        }
                    ],
                    "logs": [],
                    "processID": "p1"
                }
            ],
            "processes": {
                "p1": {
                    "serviceName": "test-jaeger-query-api",
                    "tags": []
                }
            }
        }
    ],
    "total": 0,
    "limit": 0,
    "offset": 0,
    "errors": []
}
    "#;

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces` API with tags.
    // 1. first query without tags, get 2 results
    let res = client
            .get(&shift_fixture_micros("/v1/jaeger/api/traces?service=test-jaeger-query-api&start=1738726754492422&end=1738726754592422", delta_micros))
            .header("x-greptime-trace-table-name", trace_table_name)
            .send()
            .await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"
{"data":[{"processes":{"p1":{"serviceName":"test-jaeger-query-api","tags":[]}},"spans":[{"duration":100000,"logs":[],"operationName":"access-redis","processID":"p1","references":[],"spanID":"ffa03416a7b9ea48","startTime":1738726754492422,"tags":[{"key":"net.peer.ip","type":"string","value":"1.2.3.4"},{"key":"operation.type","type":"string","value":"access-redis"},{"key":"otel.scope.name","type":"string","value":"test-jaeger-query-api"},{"key":"otel.scope.version","type":"string","value":"1.0.0"},{"key":"otel.status_description","type":"string","value":"success"},{"key":"peer.service","type":"string","value":"test-jaeger-query-api"},{"key":"span.kind","type":"string","value":"server"}],"traceID":"5611dce1bc9ebed65352d99a027b08ea"}],"traceID":"5611dce1bc9ebed65352d99a027b08ea"},{"processes":{"p1":{"serviceName":"test-jaeger-query-api","tags":[]}},"spans":[{"duration":100000,"logs":[],"operationName":"access-pg","processID":"p1","references":[],"spanID":"ffa03416a7b9ea49","startTime":1738726754492423,"tags":[{"key":"operation.type","type":"string","value":"access-pg"},{"key":"otel.scope.name","type":"string","value":"test-jaeger-find-traces"},{"key":"otel.scope.version","type":"string","value":"1.0.0"},{"key":"otel.status_description","type":"string","value":"success"},{"key":"span.kind","type":"string","value":"server"}],"traceID":"5611dce1bc9ebed65352d99a027b08fa"}],"traceID":"5611dce1bc9ebed65352d99a027b08fa"}],"errors":[],"limit":0,"offset":0,"total":0}
    "#;

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // 2. second query with tags, get 1 result
    let res = client
.get(&shift_fixture_micros("/v1/jaeger/api/traces?service=test-jaeger-query-api&start=1738726754492422&end=1738726754592422&tags=%7B%22operation.type%22%3A%22access-pg%22%7D", delta_micros))
.header("x-greptime-trace-table-name", trace_table_name)
.send()
.await;
    assert_eq!(StatusCode::OK, res.status());

    let expected = r#"
{"data":[{"processes":{"p1":{"serviceName":"test-jaeger-query-api","tags":[]}},"spans":[{"duration":100000,"logs":[],"operationName":"access-pg","processID":"p1","references":[],"spanID":"ffa03416a7b9ea49","startTime":1738726754492423,"tags":[{"key":"operation.type","type":"string","value":"access-pg"},{"key":"otel.scope.name","type":"string","value":"test-jaeger-find-traces"},{"key":"otel.scope.version","type":"string","value":"1.0.0"},{"key":"otel.status_description","type":"string","value":"success"},{"key":"span.kind","type":"string","value":"server"}],"traceID":"5611dce1bc9ebed65352d99a027b08fa"}],"traceID":"5611dce1bc9ebed65352d99a027b08fa"}],"errors":[],"limit":0,"offset":0,"total":0}
"#;

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let expected: Value =
        serde_json::from_str(&shift_fixture_micros(expected, delta_micros)).unwrap();
    assert_eq!(resp, expected);

    // Test `/api/traces` API with Grafana User-Agent.
    // When user agent is Grafana, only return at most 3 spans per trace (earliest by timestamp).
    // Trace `5611dce1bc9ebed65352d99a027b08fb` has 4 spans, so only 3 should be returned.
    let res = client
        .get(&shift_fixture_micros("/v1/jaeger/api/traces?service=test-jaeger-query-api&start=1738726754600000&end=1738726754700004", delta_micros))
        .header("x-greptime-trace-table-name", trace_table_name)
        .header("User-Agent", "Grafana/8.0.0")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    // Verify that the trace has exactly 3 spans (limited by Grafana user agent)
    let data = resp.get("data").unwrap().as_array().unwrap();
    assert_eq!(data.len(), 1, "Expected 1 trace");
    let trace = &data[0];
    assert_eq!(
        trace.get("traceID").unwrap().as_str().unwrap(),
        "5611dce1bc9ebed65352d99a027b08fb"
    );
    let spans = trace.get("spans").unwrap().as_array().unwrap();
    assert_eq!(
        spans.len(),
        3,
        "Expected 3 spans (limited by Grafana user agent), got {}",
        spans.len()
    );
    // Verify that the 3 earliest spans are returned (by timestamp ascending)
    let span_names: Vec<&str> = spans
        .iter()
        .map(|s| s.get("operationName").unwrap().as_str().unwrap())
        .collect();
    assert!(
        span_names.contains(&"access-span-1"),
        "Expected access-span-1 in spans"
    );
    assert!(
        span_names.contains(&"access-span-2"),
        "Expected access-span-2 in spans"
    );
    assert!(
        span_names.contains(&"access-span-3"),
        "Expected access-span-3 in spans"
    );
    assert!(
        !span_names.contains(&"access-span-4"),
        "access-span-4 should NOT be in spans (4th span)"
    );

    // Test `/api/traces` API without User-Agent (default behavior).
    // All 4 spans should be returned for the trace.
    let res = client
        .get(&shift_fixture_micros("/v1/jaeger/api/traces?service=test-jaeger-query-api&start=1738726754600000&end=1738726754700004", delta_micros))
        .header("x-greptime-trace-table-name", trace_table_name)
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let data = resp.get("data").unwrap().as_array().unwrap();
    assert_eq!(data.len(), 1, "Expected 1 trace");
    let trace = &data[0];
    let spans = trace.get("spans").unwrap().as_array().unwrap();
    assert_eq!(
        spans.len(),
        4,
        "Expected 4 spans (no user agent limit), got {}",
        spans.len()
    );

    // Test `/api/traces` API with Jaeger User-Agent (should return all spans like default).
    let res = client
        .get(&shift_fixture_micros("/v1/jaeger/api/traces?service=test-jaeger-query-api&start=1738726754600000&end=1738726754700004", delta_micros))
        .header("x-greptime-trace-table-name", trace_table_name)
        .header("User-Agent", "Jaeger-Query/1.0.0")
        .send()
        .await;
    assert_eq!(StatusCode::OK, res.status());

    let resp: Value = serde_json::from_str(&res.text().await).unwrap();
    let data = resp.get("data").unwrap().as_array().unwrap();
    assert_eq!(data.len(), 1, "Expected 1 trace");
    let trace = &data[0];
    let spans = trace.get("spans").unwrap().as_array().unwrap();
    assert_eq!(
        spans.len(),
        4,
        "Expected 4 spans (Jaeger user agent, no limit), got {}",
        spans.len()
    );

    guard.remove_all().await;
}

#[apply(both_deployment_cases)]
async fn test_opentsdb_skip_wal(distributed: bool) {
    let cases = vec![HttpWalCase::new(
        "OpenTSDB",
        "/v1/opentsdb/api/put",
        "application/json",
        r#"{"metric":"wal_tsdb","timestamp":1700000000,"value":1,"tags":{"host":"a"}}"#,
    )];
    check_http_skip_wal("opentsdb", &cases, distributed).await;
}

pub async fn test_influxdb_write(store_type: StorageType) {
    common_telemetry::init_default_ut_logging();
    let (app, mut guard) =
        setup_test_http_app_with_frontend(store_type, "test_influxdb_write").await;

    let client = TestClient::new(app).await;

    // Only write field cpu.
    let result = client
        .post("/v1/influxdb/write?db=public&p=greptime&u=greptime")
        .body("test_alter,host=host1 cpu=1.2 1664370459457010101")
        .send()
        .await;
    assert_eq!(result.status(), 204);
    assert!(result.text().await.is_empty());

    // Only write field mem.
    let result = client
        .post("/v1/influxdb/write?db=public&p=greptime&u=greptime")
        .body("test_alter,host=host1 mem=10240.0 1664370469457010101")
        .send()
        .await;
    assert_eq!(result.status(), 204);
    assert!(result.text().await.is_empty());

    // Write field cpu & mem.
    let result = client
        .post("/v1/influxdb/write?db=public&p=greptime&u=greptime")
        .body("test_alter,host=host1 cpu=3.2,mem=20480.0 1664370479457010101")
        .send()
        .await;
    assert_eq!(result.status(), 204);
    assert!(result.text().await.is_empty());

    let expected = r#"[["host1",1.2,1664370459457010101,null],["host1",null,1664370469457010101,10240.0],["host1",3.2,1664370479457010101,20480.0]]"#;
    validate_data(
        "test_influxdb_write",
        &client,
        "select * from test_alter order by greptime_timestamp;",
        expected,
    )
    .await;

    guard.remove_all().await;
}

/// Flushes `table` so the rows written so far move from the memtable into SST
/// files. Replaying a query afterwards covers the encode/decode round trip and,
/// for indexed columns, the puffin files that only exist after a flush.
/// Base microsecond instant the `test_jaeger_query_api_for_trace_v1` fixture and
/// its expected responses are written against.
const JAEGER_V1_FIXTURE_BASE_MICROS: u64 = 1_738_726_754_492_421;

/// Shifts every fixture timestamp in `text` by `delta` microseconds, so expected
/// responses and query bounds follow the payload after it is moved next to now.
///
/// Only integer literals within a day of [`JAEGER_V1_FIXTURE_BASE_MICROS`] are
/// rewritten; durations, counts and ids are left alone.
fn shift_fixture_micros(text: &str, delta: i64) -> String {
    let mut out = String::with_capacity(text.len());
    let mut rest = text;
    while let Some(start) = rest.find(|c: char| c.is_ascii_digit()) {
        out.push_str(&rest[..start]);
        let digits_end = rest[start..]
            .find(|c: char| !c.is_ascii_digit())
            .map_or(rest.len(), |offset| start + offset);
        let digits = &rest[start..digits_end];
        match digits.parse::<i64>() {
            Ok(value) if (value - JAEGER_V1_FIXTURE_BASE_MICROS as i64).abs() < 86_400_000_000 => {
                out.push_str(&(value + delta).to_string())
            }
            _ => out.push_str(digits),
        }
        rest = &rest[digits_end..];
    }
    out.push_str(rest);
    out
}

async fn flush_table(client: &TestClient, table: &str) {
    // The argument is parsed as a SQL table name, so an unquoted identifier is
    // lower-cased and will not resolve a mixed-case table.
    let quoted = table
        .split('.')
        .map(|part| format!("\"{part}\""))
        .collect::<Vec<_>>()
        .join(".");
    let res = client
        .post("/v1/sql")
        .form(&[("sql", format!("ADMIN FLUSH_TABLE('{quoted}')"))])
        .send()
        .await;
    let status = res.status();
    let body = res.text().await;
    assert_eq!(status, StatusCode::OK, "flush {table} fail: {body}");
    // A failing statement still comes back as 200 with an `error` envelope.
    assert!(!body.contains("\"error\""), "flush {table} fail: {body}");
}

async fn validate_data(test_name: &str, client: &TestClient, sql: &str, expected: &str) {
    let res = client
        .get(format!("/v1/sql?sql={sql}").as_str())
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "validate {test_name} fail");
    let resp = res.text().await;
    let v = get_rows_from_output(&resp);

    assert_eq!(
        expected, v,
        "validate {test_name} fail, expected: {expected}, actual: {v}"
    );
}

async fn wait_for_data(client: &TestClient, sql: &str, expected: &str) {
    wait_for_data_with_timeout(client, sql, expected, Duration::from_secs(10)).await;
}

async fn wait_for_event_data(client: &TestClient, sql: &str, expected: &str) {
    wait_for_data_with_timeout(client, sql, expected, Duration::from_secs(30)).await;
}

async fn wait_for_data_with_timeout(
    client: &TestClient,
    sql: &str,
    expected: &str,
    timeout: Duration,
) {
    tokio::time::timeout(timeout, async {
        let encoded_sql = encode(sql);
        loop {
            let res = client
                .get(format!("/v1/sql?sql={encoded_sql}").as_str())
                .send()
                .await;
            if res.status() != StatusCode::OK {
                tokio::time::sleep(Duration::from_millis(50)).await;
                continue;
            }
            let resp = res.text().await;
            let v = get_rows_from_output(&resp);

            if expected == v {
                break;
            }

            tokio::time::sleep(Duration::from_millis(50)).await;
        }
    })
    .await
    .unwrap();
}

async fn execute_sql(client: &TestClient, sql: &str) -> TestResponse {
    let encoded_sql = encode(sql);
    client
        .get(format!("/v1/sql?sql={encoded_sql}").as_str())
        .send()
        .await
}

struct HttpWalCase {
    name: &'static str,
    get: bool,
    create_table: Option<&'static str>,
    expected_written_nodes: Option<usize>,
    path: &'static str,
    content_type: &'static str,
    body: Vec<u8>,
    headers: Vec<(&'static str, &'static str)>,
}

impl HttpWalCase {
    fn new(
        name: &'static str,
        path: &'static str,
        content_type: &'static str,
        body: impl Into<Vec<u8>>,
    ) -> Self {
        Self {
            name,
            get: false,
            create_table: None,
            expected_written_nodes: None,
            path,
            content_type,
            body: body.into(),
            headers: Vec::new(),
        }
    }
}

async fn check_http_skip_wal(name: &str, cases: &[HttpWalCase], distributed: bool) {
    common_telemetry::init_default_ut_logging();
    let mut instance = MockInstanceImpl::new(&format!("http_skip_wal_{name}"), distributed).await;
    let fe = instance.frontend();
    let server = HttpServerBuilder::new(HttpOptions::default())
        .with_sql_handler(fe.clone())
        .with_influxdb_handler(fe.clone())
        .with_opentsdb_handler(fe.clone())
        .with_log_ingest_handler(fe.clone(), None, None)
        .with_otlp_handler(fe.clone(), true)
        // The pending batcher uses BulkInsert, deliberately outside this PR.
        .with_prom_handler(fe.clone(), Some(fe), true, PromValidationMode::Strict, None)
        .build();
    let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
    for case in cases {
        if let Some(create_table) = case.create_table {
            let response = execute_sql(&client, create_table).await;
            assert!(response.status().is_success(), "{}", response.text().await);
        }
        // Warm up schema-on-write, then change only the request hint. Reusing
        // the same payload also verifies policy is not persisted on the table.
        for (round, hint) in [None, Some("true"), Some("false"), None]
            .into_iter()
            .enumerate()
        {
            common_telemetry::info!("Protocol WAL case: {}, hint: {:?}", case.name, hint);
            let before = instance.flush_and_snapshot_wal().await;
            let mut headers = vec![(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static(case.content_type),
            )];
            headers.extend(case.headers.iter().map(|(key, value)| {
                (
                    HeaderName::from_static(key),
                    HeaderValue::from_static(value),
                )
            }));
            if let Some(value) = hint {
                headers.push((
                    HeaderName::from_static("x-greptime-insert-skip-wal"),
                    HeaderValue::from_static(value),
                ));
            }
            let response = if case.get {
                let mut request = client.get(case.path).body(case.body.clone());
                for (key, value) in headers {
                    request = request.header(key, value);
                }
                request.send().await
            } else {
                send_req(&client, headers, case.path, case.body.clone(), false).await
            };
            let status = response.status();
            let body = response.text().await;
            assert!(status.is_success(), "{}: {status}: {body}", case.name);
            let after = instance.flush_and_snapshot_wal().await;
            if before.keys().eq(after.keys()) {
                assert_wal_delta(&before, &after, hint == Some("true"));
                if let Some(expected_written_nodes) = case.expected_written_nodes {
                    let written_nodes = after
                        .iter()
                        .filter(|(key, (flushed_sequence, _))| *flushed_sequence > before[*key].0)
                        .map(|((node_id, _), _)| *node_id)
                        .collect::<std::collections::BTreeSet<_>>();
                    assert_eq!(
                        written_nodes.len(),
                        expected_written_nodes,
                        "{}: unexpected number of written datanodes",
                        case.name
                    );
                }
            } else {
                assert!(round == 0, "only warmup may create regions: {}", case.name);
            }
        }
    }
    instance.shutdown().await;
}

async fn send_req(
    client: &TestClient,
    headers: Vec<(HeaderName, HeaderValue)>,
    path: &str,
    body: Vec<u8>,
    with_gzip: bool,
) -> TestResponse {
    let mut req = client.post(path);

    for (k, v) in headers {
        req = req.header(k, v);
    }

    let mut len = body.len();

    if with_gzip {
        let encoded = compress_vec_with_gzip(body);
        len = encoded.len();
        req = req.header("content-encoding", "gzip").body(encoded);
    } else {
        req = req.body(body);
    }

    req.header("content-length", len).send().await
}

async fn send_log_req(
    client: &TestClient,
    table_name: &str,
    extract_keys: Option<&str>,
    req: ExportLogsServiceRequest,
    with_gzip: bool,
) -> TestResponse {
    let mut headers = vec![
        (
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("application/x-protobuf"),
        ),
        (
            HeaderName::from_static(GREPTIME_LOG_TABLE_NAME_HEADER_NAME),
            HeaderValue::from_str(table_name).unwrap(),
        ),
    ];
    if let Some(extract_keys) = extract_keys {
        headers.push((
            HeaderName::from_static(GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME),
            HeaderValue::from_str(extract_keys).unwrap(),
        ));
    }

    send_req(
        client,
        headers,
        "/v1/otlp/v1/logs?db=public",
        req.encode_to_vec(),
        with_gzip,
    )
    .await
}

async fn send_trace_v1_req(
    client: &TestClient,
    table_name: &str,
    req: ExportTraceServiceRequest,
    with_gzip: bool,
) -> TestResponse {
    send_trace_v1_req_with_db(client, "public", table_name, req, with_gzip).await
}

async fn send_trace_v1_req_with_db(
    client: &TestClient,
    db_name: &str,
    table_name: &str,
    req: ExportTraceServiceRequest,
    with_gzip: bool,
) -> TestResponse {
    send_req(
        client,
        vec![
            (
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("application/x-protobuf"),
            ),
            (
                HeaderName::from_static("x-greptime-pipeline-name"),
                HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME),
            ),
            (
                HeaderName::from_static("x-greptime-trace-table-name"),
                HeaderValue::from_str(table_name).unwrap(),
            ),
            (
                GREPTIME_DB_HEADER_NAME.clone(),
                HeaderValue::from_str(db_name).unwrap(),
            ),
        ],
        "/v1/otlp/v1/traces",
        req.encode_to_vec(),
        with_gzip,
    )
    .await
}

fn make_log_request(records: Vec<Value>) -> ExportLogsServiceRequest {
    serde_json::from_value(json!({
        "resourceLogs": [{
            "resource": {
                "attributes": []
            },
            "scopeLogs": [{
                "scope": {
                    "name": "",
                    "version": "",
                    "attributes": []
                },
                "logRecords": records,
                "schemaUrl": ""
            }],
            "schemaUrl": "https://opentelemetry.io/schemas/1.4.0"
        }]
    }))
    .unwrap()
}

fn make_log_record(
    trace_id: &str,
    span_id: &str,
    time_unix_nano: i64,
    body: &str,
    attributes: Vec<Value>,
) -> Value {
    json!({
        "timeUnixNano": time_unix_nano.to_string(),
        "observedTimeUnixNano": "0",
        "severityNumber": 9,
        "severityText": "Info",
        "body": {
            "stringValue": body
        },
        "attributes": attributes,
        "droppedAttributesCount": 0,
        "flags": 0,
        "traceId": trace_id,
        "spanId": span_id
    })
}

fn make_trace_v1_request(service_name: &str, spans: Vec<Value>) -> ExportTraceServiceRequest {
    serde_json::from_value(json!({
        "resourceSpans": [{
            "resource": {
                "attributes": [{
                    "key": "service.name",
                    "value": { "stringValue": service_name }
                }]
            },
            "scopeSpans": [{
                "scope": {
                    "name": "trace-v1-type-tests"
                },
                "spans": spans
            }],
            "schemaUrl": "https://opentelemetry.io/schemas/1.4.0"
        }]
    }))
    .unwrap()
}

fn make_trace_v1_span(
    trace_id: &str,
    span_id: &str,
    name: &str,
    start_time_unix_nano: i64,
    end_time_unix_nano: i64,
    attributes: Vec<Value>,
) -> Value {
    json!({
        "traceId": trace_id,
        "spanId": span_id,
        "name": name,
        "kind": 2,
        "startTimeUnixNano": start_time_unix_nano.to_string(),
        "endTimeUnixNano": end_time_unix_nano.to_string(),
        "attributes": attributes,
        "status": {
            "message": "",
            "code": 0
        }
    })
}

fn make_string_attr(key: &str, value: &str) -> Value {
    json!({
        "key": key,
        "value": {
            "stringValue": value
        }
    })
}

fn make_int_attr(key: &str, value: i64) -> Value {
    json!({
        "key": key,
        "value": {
            "intValue": value.to_string()
        }
    })
}

fn make_double_attr(key: &str, value: f64) -> Value {
    json!({
        "key": key,
        "value": {
            "doubleValue": value
        }
    })
}

fn make_bool_attr(key: &str, value: bool) -> Value {
    json!({
        "key": key,
        "value": {
            "boolValue": value
        }
    })
}

fn get_rows_from_output(output: &str) -> String {
    let resp: Value = serde_json::from_str(output).unwrap();
    resp.get("output")
        .and_then(Value::as_array)
        .and_then(|v| v.first())
        .and_then(|v| v.get("records"))
        .and_then(|v| v.get("rows"))
        .unwrap()
        .to_string()
}

fn get_output_row_count(output: &str) -> usize {
    let resp: Value = serde_json::from_str(output).unwrap();
    resp.get("output")
        .and_then(Value::as_array)
        .and_then(|v| v.first())
        .and_then(|v| v.get("records"))
        .and_then(|v| v.get("rows"))
        .and_then(Value::as_array)
        .unwrap()
        .len()
}

fn compress_vec_with_gzip(data: Vec<u8>) -> Vec<u8> {
    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
    encoder.write_all(&data).unwrap();
    encoder.finish().unwrap()
}

pub async fn test_http_memory_limit(store_type: StorageType) {
    use tests_integration::test_util::setup_test_http_app_with_frontend_and_custom_options;

    common_telemetry::init_default_ut_logging();

    let http_opts = servers::http::HttpOptions {
        addr: format!("127.0.0.1:{}", common_test_util::ports::get_port()),
        ..Default::default()
    };

    // Create memory limiter with 2KB limit and fail-fast policy.
    // Note: MemoryManager uses 1KB granularity (PermitGranularity::Kilobyte),
    // so 2KB = 2 permits. Small/medium requests should fit, large should fail.
    let memory_limiter = ServerMemoryLimiter::new(2048, OnExhaustedPolicy::Fail);

    let (app, mut guard) = setup_test_http_app_with_frontend_and_custom_options(
        store_type,
        "test_http_memory_limit",
        None,
        Some(http_opts),
        Some(memory_limiter),
    )
    .await;

    let client = TestClient::new(app).await;

    // Create table first
    let res = client
        .get("/v1/sql?sql=CREATE TABLE test_mem_limit(host STRING, cpu DOUBLE, ts TIMESTAMP TIME INDEX)")
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK);

    // Test 1: Small POST request should succeed (uses actual memory limiting path)
    // Note: GET requests have no body, so they bypass body memory limiting entirely.
    let small_insert = "INSERT INTO test_mem_limit VALUES ('host1', 1.0, 0)";
    let res = client
        .post("/v1/sql")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .body(format!("sql={}", small_insert))
        .send()
        .await;
    assert_eq!(res.status(), StatusCode::OK, "Small POST should succeed");

    // Test 1B: Medium request in the 500-1024 byte range should also succeed
    // (due to 1KB granularity alignment)
    let medium_values: Vec<String> = (0..8)
        .map(|i| format!("('host{}', {}.5, {})", i, i, i * 1000))
        .collect();
    let medium_insert = format!(
        "INSERT INTO test_mem_limit VALUES {}",
        medium_values.join(", ")
    );

    let res = client
        .post("/v1/sql")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .body(format!("sql={}", medium_insert))
        .send()
        .await;
    assert_eq!(
        res.status(),
        StatusCode::OK,
        "Medium request (~700 bytes) should succeed within aligned 1KB limit"
    );

    // Test 2: Large write request should be rejected (exceeds 2KB limit)
    // Generate a large INSERT with many rows and long strings to definitely exceed 2KB
    let large_values: Vec<String> = (0..100)
        .map(|i| {
            format!(
                "('this_is_a_very_long_hostname_string_to_increase_body_size_row_{}', {}.5, {})",
                i,
                i,
                i * 1000
            )
        })
        .collect();
    let large_insert = format!(
        "INSERT INTO test_mem_limit VALUES {}",
        large_values.join(", ")
    );

    let res = client
        .post("/v1/sql")
        .header("Content-Type", "application/x-www-form-urlencoded")
        .body(format!("sql={}", large_insert))
        .send()
        .await;

    assert_eq!(
        res.status(),
        StatusCode::TOO_MANY_REQUESTS,
        "Large write should be rejected with 429"
    );

    let error_body = res.text().await;
    assert!(
        error_body.contains("Request body memory limit exceeded"),
        "Error message should be 'Request body memory limit exceeded', got: {}",
        error_body
    );

    // Test 3A: Small InfluxDB write should succeed
    let small_influx = "test_influx,host=host1 cpu=1.5 1000000000";
    let res = client
        .post("/v1/influxdb/write?db=public")
        .body(small_influx)
        .send()
        .await;
    assert_eq!(
        res.status(),
        StatusCode::NO_CONTENT,
        "Small InfluxDB write should succeed"
    );

    // Test 3B: Large InfluxDB write should be rejected
    let large_influx_body = (0..100)
        .map(|i| {
            format!(
                "test_influx,host=host{} cpu={}.5 {}",
                i,
                i,
                (i as i64) * 1000000000
            )
        })
        .collect::<Vec<_>>()
        .join("\n");

    let res = client
        .post("/v1/influxdb/write?db=public")
        .body(large_influx_body)
        .send()
        .await;

    assert_eq!(
        res.status(),
        StatusCode::TOO_MANY_REQUESTS,
        "Large InfluxDB write should be rejected"
    );

    guard.remove_all().await;
}