Skip to main content

tests_integration/
otlp.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#[cfg(test)]
16mod test {
17    use std::sync::{Arc, Mutex};
18    use std::time::Duration;
19
20    use TimeUnit as ArrowTimeUnit;
21    use api::region::RegionResponse;
22    use api::v1::region::{RegionRequest, region_request};
23    use client::{DEFAULT_CATALOG_NAME, OutputData};
24    use common_meta::cache::LayeredCacheRegistryBuilder;
25    use common_meta::node_manager::{
26        Datanode, DatanodeManager, DatanodeRef, FlownodeManager, FlownodeRef, NodeManagerRef,
27    };
28    use common_meta::peer::Peer;
29    use common_query::request::QueryRequest;
30    use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
31    use datatypes::arrow::array::AsArray;
32    use datatypes::arrow::datatypes::{
33        DataType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
34        TimestampNanosecondType, TimestampSecondType,
35    };
36    use frontend::instance::Instance;
37    use frontend::instance::builder::FrontendBuilder;
38    use frontend::service_config::BatcherOptions;
39    use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
40    use otel_arrow_rust::proto::opentelemetry::collector::metrics::v1::ExportMetricsServiceRequest;
41    use otel_arrow_rust::proto::opentelemetry::common::v1::any_value::Value as Val;
42    use otel_arrow_rust::proto::opentelemetry::common::v1::{
43        AnyValue, InstrumentationScope, KeyValue,
44    };
45    use otel_arrow_rust::proto::opentelemetry::metrics::v1::number_data_point::Value;
46    use otel_arrow_rust::proto::opentelemetry::metrics::v1::{
47        AggregationTemporality, DataPointFlags, Gauge, Histogram, HistogramDataPoint, Metric,
48        NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric,
49    };
50    use otel_arrow_rust::proto::opentelemetry::resource::v1::Resource;
51    use pipeline::{GreptimePipelineParams, PipelineWay};
52    use serde_json::json;
53    use servers::batcher::BatchingProtocol;
54    use servers::otlp::trace::{TraceAuxData, span};
55    use servers::query_handler::OpenTelemetryProtocolHandler;
56    use servers::query_handler::sql::SqlQueryHandler;
57    use session::context::QueryContext;
58    use session::protocol_ctx::{OtlpMetricCtx, ProtocolCtx};
59    use store_api::region_engine::RegionEngine;
60    use store_api::storage::RegionId;
61    use tokio::sync::oneshot;
62
63    use crate::standalone::{GreptimeDbStandalone, GreptimeDbStandaloneBuilder};
64    use crate::tests;
65
66    #[tokio::test(flavor = "multi_thread")]
67    async fn test_trace_aux_cache() {
68        check_trace_aux_cache(false).await;
69    }
70
71    #[tokio::test(flavor = "multi_thread")]
72    async fn test_trace_aux_cache_with_batching_v0_v1() {
73        check_trace_aux_cache(true).await;
74    }
75
76    #[test]
77    fn test_trace_aux_async_completion() {
78        temp_env::with_var("PENDING_ROWS_BATCH_SYNC", Some("false"), || {
79            tokio::runtime::Builder::new_multi_thread()
80                .enable_all()
81                .build()
82                .unwrap()
83                .block_on(check_trace_aux_async_completion());
84        });
85    }
86
87    #[test]
88    fn test_trace_aux_admission() {
89        temp_env::with_var("PENDING_ROWS_BATCH_SYNC", Some("false"), || {
90            tokio::runtime::Builder::new_multi_thread()
91                .enable_all()
92                .build()
93                .unwrap()
94                .block_on(async {
95                    tokio::time::timeout(Duration::from_secs(60), check_trace_aux_admission())
96                        .await
97                        .expect("auxiliary admission must make progress");
98                });
99        });
100    }
101
102    struct TraceAuxWriteGate {
103        region_id: RegionId,
104        started: oneshot::Sender<()>,
105        release: oneshot::Receiver<bool>,
106    }
107
108    struct GatedTraceNodeManager {
109        inner: NodeManagerRef,
110        gate: Arc<Mutex<Option<TraceAuxWriteGate>>>,
111    }
112
113    #[async_trait::async_trait]
114    impl DatanodeManager for GatedTraceNodeManager {
115        async fn datanode(&self, peer: &Peer) -> DatanodeRef {
116            Arc::new(GatedTraceDatanode {
117                inner: self.inner.datanode(peer).await,
118                gate: self.gate.clone(),
119            })
120        }
121    }
122
123    #[async_trait::async_trait]
124    impl FlownodeManager for GatedTraceNodeManager {
125        async fn flownode(&self, peer: &Peer) -> FlownodeRef {
126            self.inner.flownode(peer).await
127        }
128    }
129
130    struct GatedTraceDatanode {
131        inner: DatanodeRef,
132        gate: Arc<Mutex<Option<TraceAuxWriteGate>>>,
133    }
134
135    #[async_trait::async_trait]
136    impl Datanode for GatedTraceDatanode {
137        async fn handle_request(
138            &self,
139            request: Arc<RegionRequest>,
140        ) -> common_meta::error::Result<RegionResponse> {
141            let gate = {
142                let mut gate = self.gate.lock().unwrap();
143                if gate.as_ref().is_some_and(|gate| {
144                    matches!(&request.body, Some(region_request::Body::Inserts(inserts))
145                        if inserts.requests.iter().any(|insert| insert.region_id == gate.region_id.as_u64()))
146                }) {
147                    gate.take()
148                } else {
149                    None
150                }
151            };
152            if let Some(gate) = gate {
153                gate.started.send(()).unwrap();
154                if !gate.release.await.unwrap() {
155                    return common_meta::error::UnexpectedSnafu {
156                        err_msg: "injected auxiliary write failure",
157                    }
158                    .fail();
159                }
160            }
161            self.inner.handle(request).await
162        }
163
164        async fn handle_query(
165            &self,
166            request: QueryRequest,
167        ) -> common_meta::error::Result<SendableRecordBatchStream> {
168            self.inner.handle_query(request).await
169        }
170    }
171
172    async fn check_trace_aux_admission() {
173        let standalone = GreptimeDbStandaloneBuilder::new("trace_aux_admission")
174            .with_table_batcher(BatcherOptions {
175                protocols: vec![BatchingProtocol::Otlp],
176                pending_rows_flush_interval: Duration::from_secs(3600),
177                max_batch_rows: 1,
178                max_inflight_requests: 1,
179                ..Default::default()
180            })
181            .build()
182            .await;
183        let source = standalone.fe_instance();
184        let gate = Arc::new(Mutex::new(None));
185        let cache_registry = Arc::new(
186            cache::with_default_composite_cache_registry(
187                LayeredCacheRegistryBuilder::default().add_cache_registry(
188                    cache::build_fundamental_cache_registry(standalone.kv_backend.clone()),
189                ),
190            )
191            .unwrap()
192            .build(),
193        );
194        let mut options = standalone.opts.frontend_options();
195        options.otlp.trace_ingest_chunk_size = 1;
196        let instance = Arc::new(
197            FrontendBuilder::new(
198                options,
199                standalone.kv_backend.clone(),
200                cache_registry,
201                source.catalog_manager().clone(),
202                Arc::new(GatedTraceNodeManager {
203                    inner: source.node_manager().clone(),
204                    gate: gate.clone(),
205                }),
206                source.procedure_executor().clone(),
207                Arc::new(catalog::process_manager::ProcessManager::new(
208                    "trace_aux_admission".into(),
209                    None,
210                )),
211            )
212            .try_build()
213            .await
214            .unwrap(),
215        );
216
217        for version in 0..=1 {
218            let table_name = format!("trace_aux_admission_v{version}");
219            let mut context = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
220            context.set_extension(
221                common_catalog::consts::TRACE_TABLE_NAME_SESSION_KEY,
222                &table_name,
223            );
224            context.set_extension(table::requests::TRACE_TABLE_PARTITIONS_HINT_KEY, "1");
225            context.set_batching_enabled(true);
226            let ctx = Arc::new(context);
227            let write = async |request| {
228                instance
229                    .traces(
230                        instance.clone(),
231                        request,
232                        if version == 0 {
233                            PipelineWay::OtlpTraceDirectV0
234                        } else {
235                            PipelineWay::OtlpTraceDirectV1
236                        },
237                        GreptimePipelineParams::default(),
238                        table_name.clone(),
239                        ctx.clone(),
240                    )
241                    .await
242                    .unwrap()
243            };
244            let cached = |request: ExportTraceServiceRequest| {
245                let mut data = TraceAuxData::default();
246                for span in span::parse(request).iter().flat_map(|group| &group.spans) {
247                    data.observe_span(span);
248                }
249                instance.trace_aux_data_cached_for_test(&table_name, data, &ctx)
250            };
251            let warm = trace_aux_request();
252            assert_eq!(1, write(warm.clone()).await.accepted_spans);
253            while !cached(warm.clone()) {
254                tokio::task::yield_now().await;
255            }
256            let table = instance
257                .catalog_manager()
258                .table(
259                    DEFAULT_CATALOG_NAME,
260                    "public",
261                    &format!("{table_name}_operations"),
262                    None,
263                )
264                .await
265                .unwrap()
266                .unwrap();
267            let route = instance
268                .partition_manager()
269                .find_physical_table_route(table.table_info().table_id())
270                .await
271                .unwrap();
272            let region_id = route.region_routes[0].region.id;
273
274            for success in [true, false] {
275                let mut cold = warm.clone();
276                let spans = &mut cold.resource_spans[0].scope_spans[0].spans;
277                spans[0].name = format!("cold_{success}");
278                spans.push(spans[0].clone());
279                let (started_tx, started_rx) = oneshot::channel();
280                let (release_tx, release_rx) = oneshot::channel();
281                *gate.lock().unwrap() = Some(TraceAuxWriteGate {
282                    region_id,
283                    started: started_tx,
284                    release: release_rx,
285                });
286
287                // Two chunks must progress with just one main and one auxiliary slot.
288                assert_eq!(2, write(cold.clone()).await.accepted_spans);
289                started_rx.await.unwrap();
290                assert!(
291                    !cached(cold.clone()),
292                    "pending writes must not warm the cache"
293                );
294                let before = trace_aux_sequences(&standalone, &table_name).await;
295                let mut waiting = Box::pin(write(cold.clone()));
296                assert!(futures::poll!(waiting.as_mut()).is_pending());
297
298                // Main admission is free while auxiliary storage is blocked. Warm
299                // requests must bypass auxiliary admission and keep making progress.
300                assert_eq!(1, write(warm.clone()).await.accepted_spans);
301                assert!(futures::poll!(waiting.as_mut()).is_pending());
302                release_tx.send(success).unwrap();
303                assert_eq!(2, waiting.await.accepted_spans);
304                if !success {
305                    // The failed write cannot satisfy the waiting request's miss.
306                    while !cached(cold.clone()) {
307                        tokio::task::yield_now().await;
308                    }
309                }
310                let after = trace_aux_sequences(&standalone, &table_name).await;
311                assert_eq!(before[1], after[1], "cached service must not be rewritten");
312                assert_eq!(
313                    before[2] + 1,
314                    after[2],
315                    "exactly one successful operation write"
316                );
317            }
318        }
319    }
320
321    fn trace_aux_request() -> ExportTraceServiceRequest {
322        serde_json::from_value(json!({
323            "resourceSpans": [{
324                "resource": {"attributes": [
325                    {"key": "service.name", "value": {"stringValue": "svc"}}
326                ]},
327                "scopeSpans": [{"spans": [{
328                    "traceId": "c05d7a4ec8e1f231f02ed6e8da8655b4",
329                    "spanId": "9630f2916e2f7909",
330                    "name": "operation",
331                    "kind": 2,
332                    "startTimeUnixNano": "1736480942444376000",
333                    "endTimeUnixNano": "1736480942444499000"
334                }]}]
335            }]
336        }))
337        .unwrap()
338    }
339
340    async fn trace_aux_sequences(standalone: &GreptimeDbStandalone, table_name: &str) -> Vec<u64> {
341        let instance = standalone.fe_instance();
342        let mut sequences = Vec::new();
343        for suffix in ["", "_services", "_operations"] {
344            let table = instance
345                .catalog_manager()
346                .table(
347                    DEFAULT_CATALOG_NAME,
348                    "public",
349                    &format!("{table_name}{suffix}"),
350                    None,
351                )
352                .await
353                .unwrap()
354                .unwrap();
355            let route = instance
356                .partition_manager()
357                .find_physical_table_route(table.table_info().table_id())
358                .await
359                .unwrap();
360            assert_eq!(route.region_routes.len(), 1);
361            sequences.push(
362                standalone
363                    .mito_engine
364                    .get_committed_sequence(route.region_routes[0].region.id)
365                    .await
366                    .unwrap(),
367            );
368        }
369        sequences
370    }
371
372    async fn check_trace_aux_async_completion() {
373        let standalone = GreptimeDbStandaloneBuilder::new("trace_aux_async_completion")
374            .with_table_batcher(BatcherOptions {
375                protocols: vec![BatchingProtocol::Otlp],
376                pending_rows_flush_interval: Duration::from_secs(3600),
377                max_batch_rows: 2,
378                ..Default::default()
379            })
380            .build()
381            .await;
382        let instance = standalone.fe_instance();
383        // V2 bulk ingestion has a separate Variant/Struct conversion failure.
384        for version in 0..=1 {
385            let table_name = format!("trace_aux_async_v{version}");
386            let mut context = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
387            context.set_extension(
388                common_catalog::consts::TRACE_TABLE_NAME_SESSION_KEY,
389                &table_name,
390            );
391            context.set_extension(table::requests::TRACE_TABLE_PARTITIONS_HINT_KEY, "1");
392            context.set_batching_enabled(true);
393            let ctx = Arc::new(context);
394            let request = trace_aux_request();
395            let write = async |request, ctx| {
396                let outcome = tokio::time::timeout(
397                    Duration::from_secs(30),
398                    instance.traces(
399                        instance.clone(),
400                        request,
401                        if version == 0 {
402                            PipelineWay::OtlpTraceDirectV0
403                        } else {
404                            PipelineWay::OtlpTraceDirectV1
405                        },
406                        GreptimePipelineParams::default(),
407                        table_name.clone(),
408                        ctx,
409                    ),
410                )
411                .await
412                .expect("async trace ingestion must return before the batch flush")
413                .unwrap();
414                assert_eq!((outcome.accepted_spans, outcome.rejected_spans), (1, 0));
415                assert!(outcome.error_message.is_none());
416            };
417            write(request.clone(), ctx.clone()).await;
418            assert_trace_v2_query(
419                instance,
420                &format!("SELECT COUNT(*) = 0 FROM {table_name}"),
421                ctx.clone(),
422            )
423            .await
424            .unwrap();
425            for suffix in ["_services", "_operations"] {
426                assert!(
427                    instance
428                        .catalog_manager()
429                        .table(
430                            DEFAULT_CATALOG_NAME,
431                            "public",
432                            &format!("{table_name}{suffix}"),
433                            None
434                        )
435                        .await
436                        .unwrap()
437                        .is_none()
438                );
439            }
440
441            // The second submission deterministically triggers the main flush.
442            // Omitting the service avoids a second concurrent auxiliary write.
443            let mut flush_request = request.clone();
444            flush_request.resource_spans[0]
445                .resource
446                .as_mut()
447                .unwrap()
448                .attributes
449                .clear();
450            write(flush_request, ctx.clone()).await;
451            tokio::time::timeout(Duration::from_secs(30), async {
452                loop {
453                    let mut aux_data = TraceAuxData::default();
454                    for span in span::parse(request.clone())
455                        .into_iter()
456                        .flat_map(|group| group.spans)
457                    {
458                        aux_data.observe_span(&span);
459                    }
460                    if instance.trace_aux_data_cached_for_test(&table_name, aux_data, &ctx) {
461                        break;
462                    }
463                    tokio::task::yield_now().await;
464                }
465            })
466            .await
467            .expect("deferred auxiliary writes must populate the cache");
468            for suffix in ["_services", "_operations"] {
469                assert_trace_v2_query(
470                    instance,
471                    &format!("SELECT COUNT(*) = 1 FROM {table_name}{suffix}"),
472                    ctx.clone(),
473                )
474                .await
475                .unwrap();
476            }
477            assert_trace_v2_query(
478                instance,
479                &format!("SELECT COUNT(*) > 0 FROM {table_name}"),
480                ctx.clone(),
481            )
482            .await
483            .unwrap();
484
485            let before = trace_aux_sequences(&standalone, &table_name).await;
486            // A direct warm request finishes before we compare storage sequences.
487            let mut warm_ctx = ctx.fork();
488            warm_ctx.set_batching_enabled(false);
489            write(request, Arc::new(warm_ctx)).await;
490            let after = trace_aux_sequences(&standalone, &table_name).await;
491            assert!(after[0] > before[0]);
492            assert_eq!(after[1..], before[1..], "v{version}");
493        }
494    }
495
496    async fn check_trace_aux_cache(batching: bool) {
497        let standalone = GreptimeDbStandaloneBuilder::new("trace_aux_cache")
498            .with_table_batcher(if batching {
499                BatcherOptions {
500                    protocols: vec![BatchingProtocol::Otlp],
501                    // Two main spans trigger a flush, while each auxiliary table
502                    // has only one row and would wait an hour if batched.
503                    pending_rows_flush_interval: Duration::from_secs(3600),
504                    max_batch_rows: 2,
505                    ..Default::default()
506                }
507            } else {
508                BatcherOptions::default()
509            })
510            .build()
511            .await;
512        let instance = standalone.fe_instance();
513        let cloned_instance = Arc::new(instance.as_ref().clone());
514
515        // V2 bulk writes currently fail on Variant/Struct schema conversion
516        // before reaching auxiliary ingestion; cover V2 with direct main writes.
517        let max_version = if batching { 1 } else { 2 };
518        for version in 0..=max_version {
519            let table_name = format!("trace_aux_v{version}");
520            let mut context = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
521            context.set_extension(
522                common_catalog::consts::TRACE_TABLE_NAME_SESSION_KEY,
523                &table_name,
524            );
525            context.set_extension(table::requests::TRACE_TABLE_PARTITIONS_HINT_KEY, "1");
526            context.set_batching_enabled(batching);
527            let ctx = Arc::new(context);
528            let mut request = trace_aux_request();
529            if batching {
530                let spans = &mut request.resource_spans[0].scope_spans[0].spans;
531                spans.push(spans[0].clone());
532            }
533            let accepted_spans = if batching { 2 } else { 1 };
534            let write = async |frontend: &Arc<Instance>, request| {
535                tokio::time::timeout(
536                    Duration::from_secs(30),
537                    frontend.traces(
538                        frontend.clone(),
539                        request,
540                        match version {
541                            0 => PipelineWay::OtlpTraceDirectV0,
542                            1 => PipelineWay::OtlpTraceDirectV1,
543                            _ => PipelineWay::OtlpTraceDirectV2,
544                        },
545                        GreptimePipelineParams::default(),
546                        table_name.clone(),
547                        ctx.clone(),
548                    ),
549                )
550                .await
551                .expect("auxiliary writes must not wait for the batch flush interval")
552                .unwrap()
553            };
554
555            // An incompatible auxiliary column fails after the main span is
556            // accepted. No candidate key may be cached from the failed batch.
557            instance
558                .do_query(
559                    &format!(
560                        "CREATE TABLE {table_name}_operations (\
561                         \"timestamp\" TIMESTAMP(9) TIME INDEX, service_name STRING, \
562                         span_name STRING, span_kind BIGINT)"
563                    ),
564                    ctx.clone(),
565                )
566                .await
567                .remove(0)
568                .unwrap();
569            let failed_aux = write(instance, request.clone()).await;
570            assert_eq!(
571                (failed_aux.accepted_spans, failed_aux.rejected_spans),
572                (accepted_spans, 0)
573            );
574            assert!(
575                failed_aux
576                    .error_message
577                    .unwrap()
578                    .contains("Auxiliary trace tables")
579            );
580
581            instance
582                .do_query(&format!("DROP TABLE {table_name}_operations"), ctx.clone())
583                .await
584                .remove(0)
585                .unwrap();
586            let retry = write(instance, request.clone()).await;
587            assert_eq!(
588                (retry.accepted_spans, retry.rejected_spans),
589                (accepted_spans, 0)
590            );
591            assert!(retry.error_message.is_none(), "{retry:?}");
592            let before = trace_aux_sequences(&standalone, &table_name).await;
593            assert!(before.iter().all(|sequence| *sequence > 0));
594
595            // Storage sequences, unlike row counts in an upsert table, prove
596            // the writes were skipped. Cloned Instances must share the cache.
597            let warm = write(&cloned_instance, request.clone()).await;
598            assert_eq!(
599                (warm.accepted_spans, warm.rejected_spans),
600                (accepted_spans, 0)
601            );
602            assert!(warm.error_message.is_none(), "{warm:?}");
603            let after = trace_aux_sequences(&standalone, &table_name).await;
604            assert!(after[0] > before[0]);
605            assert_eq!(after[1..], before[1..], "v{version}");
606
607            request.resource_spans[0].scope_spans[0].spans[0].name = "new_operation".into();
608            let new_operation = write(instance, request).await;
609            assert_eq!(
610                (new_operation.accepted_spans, new_operation.rejected_spans),
611                (accepted_spans, 0)
612            );
613            assert!(new_operation.error_message.is_none(), "{new_operation:?}");
614            let new_sequences = trace_aux_sequences(&standalone, &table_name).await;
615            assert!(new_sequences[0] > after[0]);
616            assert_eq!(new_sequences[1], after[1]);
617            assert!(new_sequences[2] > after[2]);
618        }
619    }
620
621    #[tokio::test(flavor = "multi_thread")]
622    async fn test_trace_v2_on_standalone() -> Result<(), Box<dyn std::error::Error>> {
623        let standalone = GreptimeDbStandaloneBuilder::new("trace_v2_standalone")
624            .build()
625            .await;
626        test_trace_v2(standalone.fe_instance()).await?;
627        Ok(())
628    }
629
630    #[tokio::test(flavor = "multi_thread")]
631    async fn test_trace_v2_on_distributed() -> Result<(), Box<dyn std::error::Error>> {
632        let distributed = tests::create_distributed_instance("trace_v2_distributed").await;
633        test_trace_v2(&distributed.frontend()).await?;
634        Ok(())
635    }
636
637    async fn test_trace_v2(instance: &Arc<Instance>) -> Result<(), Box<dyn std::error::Error>> {
638        let mut context = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
639        context.set_extension(
640            common_catalog::consts::TRACE_TABLE_NAME_SESSION_KEY,
641            "trace_v2",
642        );
643        let ctx = Arc::new(context);
644        let request: ExportTraceServiceRequest = serde_json::from_value(json!({
645            "resourceSpans": [{
646                "resource": {"attributes": [
647                    {"key": "deployment.environment", "value": {"stringValue": "prod"}},
648                    {"key": "cloud.region", "value": {"stringValue": "us-west"}},
649                    {"key": "capacity", "value": {"intValue": "8"}}
650                ]},
651                "scopeSpans": [{
652                    "scope": {"attributes": [
653                        {"key": "sample.rate", "value": {"doubleValue": 0.5}},
654                        {"key": "enabled", "value": {"boolValue": true}},
655                        {"key": "config", "value": {"kvlistValue": {"values": [
656                            {"key": "batch_size", "value": {"intValue": "4"}}
657                        ]}}}
658                    ]},
659                    "spans": [{
660                        "traceId": "c05d7a4ec8e1f231f02ed6e8da8655b4",
661                        "spanId": "9630f2916e2f7909", "name": "op", "kind": 2,
662                        "startTimeUnixNano": "1736480942444376000",
663                        "endTimeUnixNano": "1736480942444499000",
664                        "events": [{
665                            "timeUnixNano": "1736480942444400000",
666                            "name": "cache.hit",
667                            "attributes": [{"key": "event.code", "value": {"intValue": "7"}}]
668                        }],
669                        "links": [{
670                            "traceId": "cc9e0991a2e63d274984bd44ee669203",
671                            "spanId": "8f847259b0f6e1ab",
672                            "traceState": "vendor=value",
673                            "attributes": [{"key": "link.type", "value": {"stringValue": "follows_from"}}]
674                        }],
675                        "attributes": [
676                            {"key": "http.status_code", "value": {"intValue": "200"}},
677                            {"key": "latency", "value": {"doubleValue": 12.5}},
678                            {"key": "empty"},
679                            {"key": "bytes", "value": {"bytesValue": "AQID"}},
680                            {"key": "nested", "value": {"kvlistValue": {"values": [
681                                {"key": "a.b", "value": {"arrayValue": {"values": [{"boolValue": true}, {"stringValue": "ok"}]}}}
682                            ]}}}
683                        ]
684                    }]
685                }]
686            }]
687        }))?;
688        let result = instance
689            .traces(
690                instance.clone(),
691                request.clone(),
692                PipelineWay::OtlpTraceDirectV2,
693                GreptimePipelineParams::default(),
694                "trace_v2".to_string(),
695                ctx.clone(),
696            )
697            .await?;
698        assert_eq!((result.accepted_spans, result.rejected_spans), (1, 0));
699
700        // Check the complete schema created by the first export, before later writes.
701        let output = instance
702            .do_query("SHOW CREATE TABLE trace_v2", ctx.clone())
703            .await;
704        let batches = match output
705            .into_iter()
706            .next()
707            .ok_or("Expected non-empty SQL output")??
708            .data
709        {
710            OutputData::Stream(stream) => RecordBatches::try_collect(stream).await?,
711            OutputData::RecordBatches(batches) => batches,
712            OutputData::AffectedRows(_) => return Err("Expected SHOW CREATE TABLE rows".into()),
713        };
714        let expected = r#"CREATE TABLE IF NOT EXISTS "trace_v2" (
715  "timestamp" TIMESTAMP(9) NOT NULL,
716  "timestamp_end" TIMESTAMP(9) NULL,
717  "duration_nano" BIGINT NULL,
718  "parent_span_id" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),
719  "trace_id" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),
720  "span_id" STRING NULL,
721  "span_kind" STRING NULL,
722  "span_name" STRING NULL,
723  "span_status_code" STRING NULL,
724  "span_status_message" STRING NULL,
725  "trace_state" STRING NULL,
726  "scope_name" STRING NULL,
727  "scope_version" STRING NULL,
728  "service_name" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),
729  "span_attributes" JSON2(
730    max_auto_expanded_paths = 100
731  ) NULL,
732  "scope_attributes" JSON2(
733    max_auto_expanded_paths = 100
734  ) NULL,
735  "resource_attributes" JSON2(
736    max_auto_expanded_paths = 100
737  ) NULL,
738  "span_events" JSON NULL,
739  "span_links" JSON NULL,
740  TIME INDEX ("timestamp"),
741  PRIMARY KEY ("service_name")
742)
743PARTITION ON COLUMNS ("trace_id") (
744  trace_id < '1',
745  trace_id >= '1' AND trace_id < '2',
746  trace_id >= '2' AND trace_id < '3',
747  trace_id >= '3' AND trace_id < '4',
748  trace_id >= '4' AND trace_id < '5',
749  trace_id >= '5' AND trace_id < '6',
750  trace_id >= '6' AND trace_id < '7',
751  trace_id >= '7' AND trace_id < '8',
752  trace_id >= '8' AND trace_id < '9',
753  trace_id >= '9' AND trace_id < 'a',
754  trace_id >= 'a' AND trace_id < 'b',
755  trace_id >= 'b' AND trace_id < 'c',
756  trace_id >= 'c' AND trace_id < 'd',
757  trace_id >= 'd' AND trace_id < 'e',
758  trace_id >= 'e' AND trace_id < 'f',
759  trace_id >= 'f'
760)
761ENGINE=mito
762WITH(
763  'comment' = 'Created on insertion',
764  append_mode = 'true',
765  'greptime.semantic.entity.service.id' = 'service_name',
766  'greptime.semantic.pipeline' = 'greptime_trace_v2',
767  'greptime.semantic.signal_type' = 'trace',
768  'greptime.semantic.source' = 'opentelemetry',
769  'greptime.semantic.trace.conventions' = 'unknown',
770  table_data_model = 'greptime_trace_v2'
771)"#;
772        let batches = batches.take();
773        assert_eq!(batches.len(), 1);
774        assert_eq!(batches[0].num_rows(), 1);
775        assert_eq!(batches[0].num_columns(), 2);
776        assert_eq!(
777            batches[0].iter_column_as_string(1).collect::<Vec<_>>(),
778            vec![Some(expected.to_string())],
779        );
780
781        // First write has no service.name: it must still create the tag, but no auxiliary rows.
782        assert_trace_v2_query(
783            instance,
784            "SELECT COUNT(*) = 1
785             FROM trace_v2
786             WHERE service_name IS NULL",
787            ctx.clone(),
788        )
789        .await?;
790        assert_trace_v2_query(
791            instance,
792            "SELECT COUNT(*) = 19
793             FROM information_schema.columns
794             WHERE table_name = 'trace_v2'",
795            ctx.clone(),
796        )
797        .await?;
798        assert_trace_v2_query(
799            instance,
800            "SELECT COUNT(*) = 0
801             FROM information_schema.tables
802             WHERE table_name IN ('trace_v2_services', 'trace_v2_operations')",
803            ctx.clone(),
804        )
805        .await?;
806        assert_trace_v2_query(
807            instance,
808            r#"SELECT COUNT(*) = 1
809             FROM trace_v2
810             WHERE span_attributes."http.status_code"::BIGINT = 200
811               AND span_attributes.empty::STRING IS NULL
812               AND span_attributes.bytes[1]::BIGINT = 2
813               AND span_attributes.nested."a.b"[0]::BOOLEAN = true"#,
814            ctx.clone(),
815        )
816        .await?;
817
818        // Exercise each JSON2 column and expressions combining columns, before and after flush.
819        let event_time =
820            common_time::Timestamp::new_nanosecond(1736480942444400000).to_iso8601_string();
821        let event_query = format!(
822            r#"SELECT COUNT(*) > 0 AND COUNT(*) = COUNT(CASE
823                 WHEN json_get_string(span_events, '$[0].name') = 'cache.hit'
824                   AND json_get_string(span_events, '$[0].time') = '{event_time}'
825                   AND json_get_int(span_events, '$[0].attributes."event.code"') = 7
826                   AND json_get_string(span_links, '$[0].trace_id') = 'cc9e0991a2e63d274984bd44ee669203'
827                   AND json_get_string(span_links, '$[0].span_id') = '8f847259b0f6e1ab'
828                   AND json_get_string(span_links, '$[0].trace_state') = 'vendor=value'
829                   AND json_get_string(span_links, '$[0].attributes."link.type"') = 'follows_from'
830                 THEN 1 END)
831             FROM trace_v2"#
832        );
833        let json_queries = [
834            event_query.as_str(),
835            r#"SELECT COUNT(*) > 0 AND COUNT(*) = COUNT(CASE
836                 WHEN span_attributes.latency::DOUBLE * 2 = 25
837                   AND upper(span_attributes.nested."a.b"[1]::STRING) = 'OK'
838                   AND span_attributes.missing::STRING IS NULL
839                 THEN 1 END)
840             FROM trace_v2"#,
841            r#"SELECT COUNT(*) > 0 AND COUNT(*) = COUNT(CASE
842                 WHEN scope_attributes."sample.rate"::DOUBLE + 0.25 = 0.75
843                   AND scope_attributes.enabled::BOOLEAN
844                   AND scope_attributes.config.batch_size::BIGINT * 2 = 8
845                 THEN 1 END)
846             FROM trace_v2"#,
847            r#"SELECT COUNT(*) > 0 AND COUNT(*) = COUNT(CASE
848                 WHEN upper(resource_attributes."cloud.region"::STRING) = 'US-WEST'
849                   AND resource_attributes.capacity::BIGINT - 2 = 6
850                 THEN 1 END)
851             FROM trace_v2"#,
852            r#"SELECT AVG(span_attributes.latency::DOUBLE
853                        * scope_attributes."sample.rate"::DOUBLE) = 6.25
854                   AND SUM(resource_attributes.capacity::BIGINT)
855                       = SUM(scope_attributes.config.batch_size::BIGINT) * 2
856             FROM trace_v2"#,
857        ];
858        for sql in json_queries {
859            assert_trace_v2_query(instance, sql, ctx.clone()).await?;
860        }
861
862        let mut mixed = request.clone();
863        mixed.resource_spans[0]
864            .resource
865            .as_mut()
866            .ok_or("Expected non-empty resource in trace fixture")?
867            .attributes[0]
868            .key = "service.name".to_string();
869        let spans = &mut mixed.resource_spans[0].scope_spans[0].spans;
870        // Reject one span in the first chunk, merge its healthy rows, then write the next chunk.
871        spans.resize(514, spans[0].clone());
872        for (index, span) in spans.iter_mut().enumerate() {
873            span.span_id = (index as u64).to_be_bytes().to_vec();
874        }
875        spans[0].attributes[0]
876            .value
877            .as_mut()
878            .ok_or("Expected non-empty attribute value in trace fixture")?
879            .value = Some(
880            opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue("ok".to_string()),
881        );
882        spans[511].end_time_unix_nano = u64::MAX;
883        let result = instance
884            .traces(
885                instance.clone(),
886                mixed,
887                PipelineWay::OtlpTraceDirectV2,
888                GreptimePipelineParams::default(),
889                "trace_v2".to_string(),
890                ctx.clone(),
891            )
892            .await?;
893        assert_eq!((result.accepted_spans, result.rejected_spans), (513, 1));
894        assert!(
895            result
896                .error_message
897                .as_deref()
898                .is_some_and(|s| s.contains("Timestamp overflow"))
899        );
900        assert_trace_v2_query(
901            instance,
902            "SELECT COUNT(*) = 514
903             FROM trace_v2",
904            ctx.clone(),
905        )
906        .await?;
907        assert_trace_v2_query(
908            instance,
909            r#"SELECT COUNT(*) = 1
910             FROM trace_v2
911             WHERE span_attributes."http.status_code"::STRING = 'ok'"#,
912            ctx.clone(),
913        )
914        .await?;
915        assert_trace_v2_query(
916            instance,
917            "SELECT COUNT(*) = 1
918             FROM trace_v2_services
919             WHERE service_name = 'prod'",
920            ctx.clone(),
921        )
922        .await?;
923        assert_trace_v2_query(
924            instance,
925            "SELECT COUNT(*) = 1
926             FROM trace_v2_operations
927             WHERE service_name = 'prod'",
928            ctx.clone(),
929        )
930        .await?;
931
932        let mut invalid = request.clone();
933        invalid.resource_spans[0].scope_spans[0].spans[0].start_time_unix_nano = u64::MAX;
934        let result = instance
935            .traces(
936                instance.clone(),
937                invalid,
938                PipelineWay::OtlpTraceDirectV2,
939                GreptimePipelineParams::default(),
940                "trace_v2".to_string(),
941                ctx.clone(),
942            )
943            .await?;
944        assert_eq!((result.accepted_spans, result.rejected_spans), (0, 1));
945
946        // A v1 request must fail before auto-ALTER can flatten attributes into this table.
947        let error = instance
948            .traces(
949                instance.clone(),
950                request.clone(),
951                PipelineWay::OtlpTraceDirectV1,
952                GreptimePipelineParams::default(),
953                "trace_v2".to_string(),
954                ctx.clone(),
955            )
956            .await
957            .err()
958            .ok_or("Expected mixed trace models to be rejected")?;
959        let error = format!("{error:?}");
960        assert!(
961            error.contains("Trace table `trace_v2` uses greptime_trace_v2, but the request uses greptime_trace_v1"),
962            "{error}",
963        );
964        assert_trace_v2_query(
965            instance,
966            "SELECT COUNT(*) = 19
967             FROM information_schema.columns
968             WHERE table_name = 'trace_v2'",
969            ctx.clone(),
970        )
971        .await?;
972
973        let mut context = (*ctx).clone();
974        context.set_extension(
975            common_catalog::consts::TRACE_TABLE_NAME_SESSION_KEY,
976            "trace_v1",
977        );
978        let v1_ctx = Arc::new(context);
979        // Create the v1 table with an explicit service identity for the isolation check.
980        let mut v1_request = request.clone();
981        v1_request.resource_spans[0]
982            .resource
983            .as_mut()
984            .ok_or("Expected non-empty resource in trace fixture")?
985            .attributes[0]
986            .key = "service.name".to_string();
987        let result = instance
988            .traces(
989                instance.clone(),
990                v1_request,
991                PipelineWay::OtlpTraceDirectV1,
992                GreptimePipelineParams::default(),
993                "trace_v1".to_string(),
994                v1_ctx.clone(),
995            )
996            .await?;
997        assert_eq!(result.accepted_spans, 1);
998        let error = instance
999            .traces(
1000                instance.clone(),
1001                request.clone(),
1002                PipelineWay::OtlpTraceDirectV2,
1003                GreptimePipelineParams::default(),
1004                "trace_v1".to_string(),
1005                v1_ctx,
1006            )
1007            .await
1008            .err()
1009            .ok_or("Expected mixed trace models to be rejected")?;
1010        let error = format!("{error:?}");
1011        assert!(
1012            error.contains("Trace table `trace_v1` uses greptime_trace_v1, but the request uses greptime_trace_v2"),
1013            "{error}",
1014        );
1015
1016        let output = instance
1017            .do_query("ADMIN FLUSH_TABLE('trace_v2')", ctx.clone())
1018            .await;
1019        assert!(output.into_iter().all(|r| r.is_ok()));
1020        assert_trace_v2_query(
1021            instance,
1022            r#"SELECT COUNT(*) = 513
1023             FROM trace_v2
1024             WHERE span_attributes."http.status_code"::BIGINT = 200"#,
1025            ctx.clone(),
1026        )
1027        .await?;
1028        for sql in json_queries {
1029            assert_trace_v2_query(instance, sql, ctx.clone()).await?;
1030        }
1031        // Empty events and links are stored as JSON arrays in the same 19-column table.
1032        let mut empty_request = request;
1033        let empty_span = &mut empty_request.resource_spans[0].scope_spans[0].spans[0];
1034        empty_span.events.clear();
1035        empty_span.links.clear();
1036        let result = instance
1037            .traces(
1038                instance.clone(),
1039                empty_request,
1040                PipelineWay::OtlpTraceDirectV2,
1041                GreptimePipelineParams::default(),
1042                "trace_v2".to_string(),
1043                ctx.clone(),
1044            )
1045            .await?;
1046        assert_eq!((result.accepted_spans, result.rejected_spans), (1, 0));
1047        assert_trace_v2_query(
1048            instance,
1049            "SELECT COUNT(*) = 1 FROM trace_v2
1050             WHERE json_to_string(span_events) = '[]' AND json_to_string(span_links) = '[]'",
1051            ctx.clone(),
1052        )
1053        .await?;
1054        Ok(())
1055    }
1056
1057    async fn assert_trace_v2_query(
1058        instance: &Arc<Instance>,
1059        sql: &str,
1060        ctx: Arc<QueryContext>,
1061    ) -> Result<(), Box<dyn std::error::Error>> {
1062        let output = instance.do_query(sql, ctx).await;
1063        let OutputData::Stream(stream) = output
1064            .into_iter()
1065            .next()
1066            .ok_or("Expected non-empty SQL output")??
1067            .data
1068        else {
1069            return Err(format!("Expected a query stream: {sql}").into());
1070        };
1071        let batches = RecordBatches::try_collect(stream).await?;
1072        let batches = batches.take();
1073        assert_eq!(batches.len(), 1, "{sql}");
1074        assert_eq!(batches[0].num_rows(), 1, "{sql}");
1075        assert_eq!(
1076            batches[0].column(0).as_boolean().iter().collect::<Vec<_>>(),
1077            vec![Some(true)],
1078            "{sql}"
1079        );
1080        Ok(())
1081    }
1082
1083    #[tokio::test(flavor = "multi_thread")]
1084    async fn test_otlp_logical_batcher_non_millisecond_physical_table() {
1085        // Microsecond and nanosecond physical tables must both use the
1086        // logical batcher: requests are converted to the physical table's
1087        // unit during batch alignment
1088        // (<https://github.com/GreptimeTeam/greptimedb/issues/9342>).
1089        run_otlp_logical_batcher_non_millisecond_physical_table(
1090            "us",
1091            "TIMESTAMP(6)",
1092            ArrowTimeUnit::Microsecond,
1093            [60_000_000, 120_000_000],
1094        )
1095        .await;
1096        run_otlp_logical_batcher_non_millisecond_physical_table(
1097            "ns",
1098            "TIMESTAMP(9)",
1099            ArrowTimeUnit::Nanosecond,
1100            [60_000_000_000, 120_000_000_000],
1101        )
1102        .await;
1103    }
1104
1105    async fn run_otlp_logical_batcher_non_millisecond_physical_table(
1106        suite: &str,
1107        sql_ts_type: &str,
1108        unit: ArrowTimeUnit,
1109        expected: [i64; 2],
1110    ) {
1111        use std::time::Duration;
1112
1113        use common_base::Plugins;
1114        use datatypes::arrow::array::AsArray;
1115        use datatypes::arrow::datatypes::{TimestampMicrosecondType, TimestampNanosecondType};
1116        use frontend::server::Services;
1117        use frontend::service_config::pending_rows_batcher::BatcherOptions;
1118        use prost::Message;
1119        use servers::batcher::BatchingProtocol;
1120        use servers::http::test_helpers::TestClient;
1121        use session::protocol_ctx::{OtlpMetricCtx, ProtocolCtx};
1122
1123        let standalone =
1124            GreptimeDbStandaloneBuilder::new(&format!("otlp_logical_{suite}_physical"))
1125                .with_logical_batcher(BatcherOptions {
1126                    protocols: vec![BatchingProtocol::Otlp],
1127                    pending_rows_flush_interval: Duration::from_millis(5),
1128                    ..Default::default()
1129                })
1130                .build()
1131                .await;
1132        let instance = standalone.fe_instance();
1133        let options = standalone.opts.clone();
1134        let services = Services::new(options.clone(), instance.clone(), Plugins::default());
1135        let server = services
1136            .http_server_builder(
1137                &options.frontend_options(),
1138                services.server_memory_limiter.clone(),
1139            )
1140            .build();
1141        let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
1142
1143        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
1144        ctx.set_logical_batching_enabled(true);
1145        ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx {
1146            with_metric_engine: true,
1147            ..Default::default()
1148        }));
1149        let ctx = Arc::new(ctx);
1150
1151        // Pre-create the physical metric table with a non-millisecond time
1152        // index BEFORE any ingestion, so the batcher's bulk path must
1153        // handle it.
1154        let mut output = instance
1155            .do_query(
1156                &format!(
1157                    "CREATE TABLE greptime_physical_table (\
1158                 greptime_timestamp {sql_ts_type} NOT NULL, \
1159                 greptime_value DOUBLE NULL, \
1160                 TIME INDEX (greptime_timestamp)) \
1161                 ENGINE = metric WITH ('physical_metric_table' = 'true')"
1162                ),
1163                ctx.clone(),
1164            )
1165            .await;
1166        assert!(output.remove(0).is_ok());
1167
1168        // submit_build_and_align counts every batcher submission in both
1169        // acknowledgement modes, so the assertion detects a regression that
1170        // disables non-millisecond batching.
1171        let submissions = servers::metrics::PENDING_ROWS_BATCH_INGEST_STAGE_ELAPSED
1172            .with_label_values(&["submit_build_and_align"]);
1173        let before = submissions.get_sample_count();
1174        for (ts, value) in [(60, 10), (120, 20)] {
1175            let request = build_sum_request(
1176                "non.ms.alignment",
1177                AggregationTemporality::Cumulative,
1178                &[(ts, value)],
1179            );
1180            let response = client
1181                .post("/v1/otlp/v1/metrics")
1182                .header("content-type", "application/x-protobuf")
1183                .body(request.encode_to_vec())
1184                .send()
1185                .await;
1186            assert_eq!(response.status().as_u16(), 200);
1187        }
1188        assert_eq!(
1189            submissions.get_sample_count() - before,
1190            2,
1191            "non-millisecond physical tables must use the logical batcher"
1192        );
1193
1194        // The rows land on the non-millisecond physical table, converted
1195        // from the nanosecond encoding.
1196        let sql = "SELECT greptime_timestamp, greptime_value FROM non_ms_alignment_total ORDER BY greptime_timestamp";
1197        let batches = tokio::time::timeout(Duration::from_secs(5), async {
1198            loop {
1199                let output = instance.do_query(sql, ctx.clone()).await.remove(0).unwrap();
1200                let OutputData::Stream(stream) = output.data else {
1201                    panic!("expected stream")
1202                };
1203                let batches = RecordBatches::try_collect(stream).await.unwrap();
1204                if batches.iter().map(|batch| batch.num_rows()).sum::<usize>() == 2 {
1205                    break batches;
1206                }
1207                tokio::time::sleep(Duration::from_millis(10)).await;
1208            }
1209        })
1210        .await
1211        .unwrap();
1212        let batch = &batches.take()[0];
1213        let stored = |row: usize| match unit {
1214            ArrowTimeUnit::Microsecond => batch
1215                .column(0)
1216                .as_primitive::<TimestampMicrosecondType>()
1217                .value(row),
1218            ArrowTimeUnit::Nanosecond => batch
1219                .column(0)
1220                .as_primitive::<TimestampNanosecondType>()
1221                .value(row),
1222            _ => unreachable!("covered units only"),
1223        };
1224        assert_eq!(stored(0), expected[0]);
1225        assert_eq!(stored(1), expected[1]);
1226        let values = batch
1227            .column(1)
1228            .as_any()
1229            .downcast_ref::<datatypes::arrow::array::Float64Array>()
1230            .unwrap_or_else(|| panic!("expected f64 values"));
1231        assert_eq!(values.value(0), 10.0);
1232        assert_eq!(values.value(1), 20.0);
1233    }
1234
1235    #[tokio::test(flavor = "multi_thread")]
1236    async fn test_otlp_logical_batcher_fallback_for_cross_physical_destination() {
1237        use std::time::Duration;
1238
1239        use common_base::Plugins;
1240        use datatypes::arrow::array::AsArray;
1241        use datatypes::arrow::datatypes::TimestampMicrosecondType;
1242        use frontend::server::Services;
1243        use frontend::service_config::pending_rows_batcher::BatcherOptions;
1244        use prost::Message;
1245        use servers::batcher::BatchingProtocol;
1246        use servers::http::test_helpers::TestClient;
1247        use session::protocol_ctx::{OtlpMetricCtx, ProtocolCtx};
1248
1249        // With the logical batcher enabled, an OTLP request targeting an
1250        // existing logical table bound to ANOTHER physical table must fall
1251        // back to the ordinary insert path (the bulk eligibility check
1252        // rejects the destination binding), which converts the request to
1253        // the destination's unit. This covers the case previously guarded
1254        // by the removed OTLP pre-gate alignment.
1255        let standalone = GreptimeDbStandaloneBuilder::new("otlp_logical_cross_physical")
1256            .with_logical_batcher(BatcherOptions {
1257                protocols: vec![BatchingProtocol::Otlp],
1258                pending_rows_flush_interval: Duration::from_millis(5),
1259                ..Default::default()
1260            })
1261            .build()
1262            .await;
1263        let instance = standalone.fe_instance();
1264        let options = standalone.opts.clone();
1265        let services = Services::new(options.clone(), instance.clone(), Plugins::default());
1266        let server = services
1267            .http_server_builder(
1268                &options.frontend_options(),
1269                services.server_memory_limiter.clone(),
1270            )
1271            .build();
1272        let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
1273
1274        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
1275        ctx.set_logical_batching_enabled(true);
1276        ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx {
1277            with_metric_engine: true,
1278            ..Default::default()
1279        }));
1280        let ctx = Arc::new(ctx);
1281
1282        // A physical table with a microsecond time index and a logical table
1283        // bound to it; OTLP requests always select the default
1284        // (millisecond) physical table, so the destination binding differs.
1285        for sql in [
1286            "CREATE TABLE phy_us (\
1287             greptime_timestamp TIMESTAMP(6) NOT NULL, greptime_value DOUBLE NULL, \
1288             TIME INDEX (greptime_timestamp)) \
1289             ENGINE = metric WITH ('physical_metric_table' = 'true')",
1290            "CREATE TABLE cross_alignment_total (\
1291             greptime_timestamp TIMESTAMP(6) NOT NULL, greptime_value DOUBLE NULL, \
1292             \"stream\" STRING NULL, TIME INDEX (greptime_timestamp), PRIMARY KEY (\"stream\")) \
1293             ENGINE = metric WITH ('on_physical_table' = 'phy_us')",
1294        ] {
1295            let mut output = instance.do_query(sql, ctx.clone()).await;
1296            let result = output.remove(0);
1297            assert!(result.is_ok(), "setup ddl failed: {result:?} — {sql}");
1298        }
1299
1300        let submissions = servers::metrics::PENDING_ROWS_BATCH_INGEST_STAGE_ELAPSED
1301            .with_label_values(&["submit_build_and_align"]);
1302        let before = submissions.get_sample_count();
1303        let request = build_sum_request(
1304            "cross.alignment",
1305            AggregationTemporality::Cumulative,
1306            &[(60, 10)],
1307        );
1308        let response = client
1309            .post("/v1/otlp/v1/metrics")
1310            .header("content-type", "application/x-protobuf")
1311            .body(request.encode_to_vec())
1312            .send()
1313            .await;
1314        assert_eq!(response.status().as_u16(), 200);
1315        // The request must NOT enter the logical batcher.
1316        assert_eq!(
1317            submissions.get_sample_count() - before,
1318            0,
1319            "cross-physical destinations must fall back to the ordinary insert path"
1320        );
1321
1322        // The row lands on the microsecond logical table, converted from the
1323        // nanosecond encoding: 60s -> 60_000_000us.
1324        let sql = "SELECT greptime_timestamp, greptime_value FROM cross_alignment_total";
1325        let batches = tokio::time::timeout(Duration::from_secs(5), async {
1326            loop {
1327                let output = instance.do_query(sql, ctx.clone()).await.remove(0).unwrap();
1328                let OutputData::Stream(stream) = output.data else {
1329                    panic!("expected stream")
1330                };
1331                let batches = RecordBatches::try_collect(stream).await.unwrap();
1332                if batches.iter().map(|batch| batch.num_rows()).sum::<usize>() == 1 {
1333                    break batches;
1334                }
1335                tokio::time::sleep(Duration::from_millis(10)).await;
1336            }
1337        })
1338        .await
1339        .unwrap();
1340        let batch = &batches.take()[0];
1341        let timestamps = batch.column(0).as_primitive::<TimestampMicrosecondType>();
1342        assert_eq!(timestamps.value(0), 60_000_000);
1343    }
1344
1345    #[tokio::test(flavor = "multi_thread")]
1346    async fn test_otlp_logical_batcher_alignment() {
1347        use std::time::Duration;
1348
1349        use common_base::Plugins;
1350        use frontend::server::Services;
1351        use frontend::service_config::pending_rows_batcher::BatcherOptions;
1352        use otel_arrow_rust::proto::opentelemetry::metrics::v1::{
1353            ExponentialHistogram, ExponentialHistogramDataPoint, exponential_histogram_data_point,
1354        };
1355        use prost::Message;
1356        use servers::batcher::{BatchingProtocol, pending_rows_batch_sync_enabled};
1357        use servers::http::test_helpers::TestClient;
1358        use session::protocol_ctx::{OtlpMetricCtx, ProtocolCtx};
1359
1360        // Fixed workload, only the batcher opt-in changes. Compare stored rows,
1361        // schema evolution and visibility under the configured acknowledgement policy.
1362        let mut results = Vec::new();
1363        for enabled in [false, true] {
1364            let standalone = GreptimeDbStandaloneBuilder::new(&format!("otlp_logical_{enabled}"))
1365                .with_logical_batcher(BatcherOptions {
1366                    protocols: if enabled {
1367                        vec![BatchingProtocol::Otlp]
1368                    } else {
1369                        vec![]
1370                    },
1371                    pending_rows_flush_interval: Duration::from_millis(5),
1372                    ..Default::default()
1373                })
1374                .build()
1375                .await;
1376            let instance = standalone.fe_instance();
1377            let options = standalone.opts.clone();
1378            let services = Services::new(options.clone(), instance.clone(), Plugins::default());
1379            let server = services
1380                .http_server_builder(
1381                    &options.frontend_options(),
1382                    services.server_memory_limiter.clone(),
1383                )
1384                .build();
1385            let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
1386            let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
1387            ctx.set_logical_batching_enabled(true);
1388            ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx {
1389                with_metric_engine: true,
1390                ..Default::default()
1391            }));
1392            let ctx = Arc::new(ctx);
1393            let submissions = servers::metrics::PENDING_ROWS_BATCH_INGEST_STAGE_ELAPSED
1394                .with_label_values(&["submit_wait_flush_result"]);
1395            let before = submissions.get_sample_count();
1396            for (ts, value) in [(60, 10), (120, 20)] {
1397                let mut request = build_sum_request(
1398                    "batch.alignment",
1399                    AggregationTemporality::Cumulative,
1400                    &[(ts, value)],
1401                );
1402                if ts == 120
1403                    && let Some(metric::Data::Sum(sum)) =
1404                        &mut request.resource_metrics[0].scope_metrics[0].metrics[0].data
1405                {
1406                    sum.data_points[0].attributes.push(keyvalue("extra", "new"));
1407                }
1408                let response = client
1409                    .post("/v1/otlp/v1/metrics")
1410                    .header("content-type", "application/x-protobuf")
1411                    .body(request.encode_to_vec())
1412                    .send()
1413                    .await;
1414                assert_eq!(response.status().as_u16(), 200);
1415            }
1416            if enabled {
1417                assert_eq!(
1418                    submissions.get_sample_count() - before,
1419                    if pending_rows_batch_sync_enabled() {
1420                        2
1421                    } else {
1422                        0
1423                    },
1424                    "logical submissions must follow the global acknowledgement policy"
1425                );
1426            }
1427            let sql = "SELECT greptime_timestamp, greptime_value, stream, extra FROM batch_alignment_total ORDER BY greptime_timestamp";
1428            let batches = tokio::time::timeout(Duration::from_secs(5), async {
1429                loop {
1430                    let output = instance.do_query(sql, ctx.clone()).await.remove(0).unwrap();
1431                    let OutputData::Stream(stream) = output.data else {
1432                        panic!("expected stream")
1433                    };
1434                    let batches = RecordBatches::try_collect(stream).await.unwrap();
1435                    if !enabled
1436                        || pending_rows_batch_sync_enabled()
1437                        || batches.iter().map(|batch| batch.num_rows()).sum::<usize>() == 2
1438                    {
1439                        break batches;
1440                    }
1441                    tokio::time::sleep(Duration::from_millis(10)).await;
1442                }
1443            })
1444            .await
1445            .unwrap();
1446            assert_eq!(
1447                batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
1448                2
1449            );
1450            results.push(batches.pretty_print().unwrap());
1451
1452            let before_histograms = submissions.get_sample_count();
1453            // First create a histogram table, then reuse it in a mixed export.
1454            // Neither request may enter the scalar-only logical batcher.
1455            for mixed in [false, true] {
1456                let mut request = build_sum_request(
1457                    "batch.mixed",
1458                    AggregationTemporality::Cumulative,
1459                    &[(180, 30)],
1460                );
1461                let metrics = &mut request.resource_metrics[0].scope_metrics[0].metrics;
1462                let mut histogram = metrics[0].clone();
1463                histogram.name = "batch.histogram".to_string();
1464                histogram.data = Some(metric::Data::ExponentialHistogram(ExponentialHistogram {
1465                    aggregation_temporality: AggregationTemporality::Cumulative as i32,
1466                    data_points: vec![ExponentialHistogramDataPoint {
1467                        start_time_unix_nano: 1_000_000_000,
1468                        time_unix_nano: if mixed { 4_000_000_000 } else { 3_000_000_000 },
1469                        count: 4,
1470                        sum: Some(8.0),
1471                        zero_count: 1,
1472                        positive: Some(exponential_histogram_data_point::Buckets {
1473                            offset: -1,
1474                            bucket_counts: vec![1, 2],
1475                        }),
1476                        ..Default::default()
1477                    }],
1478                }));
1479                if !mixed {
1480                    metrics.clear();
1481                }
1482                metrics.push(histogram);
1483                let response = client
1484                    .post("/v1/otlp/v1/metrics")
1485                    .header("content-type", "application/x-protobuf")
1486                    .body(request.encode_to_vec())
1487                    .send()
1488                    .await;
1489                assert_eq!(
1490                    response.status().as_u16(),
1491                    200,
1492                    "enabled={enabled}, mixed={mixed}"
1493                );
1494            }
1495            assert_eq!(submissions.get_sample_count(), before_histograms);
1496            for (sql, expected_rows) in [
1497                (
1498                    "SELECT greptime_timestamp, greptime_native_histogram FROM batch_histogram ORDER BY greptime_timestamp",
1499                    2,
1500                ),
1501                (
1502                    "SELECT greptime_timestamp, greptime_value FROM batch_mixed_total ORDER BY greptime_timestamp",
1503                    1,
1504                ),
1505            ] {
1506                let output = instance.do_query(sql, ctx.clone()).await.remove(0).unwrap();
1507                let OutputData::Stream(stream) = output.data else {
1508                    panic!("expected stream")
1509                };
1510                let batches = RecordBatches::try_collect(stream).await.unwrap();
1511                assert_eq!(
1512                    batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
1513                    expected_rows
1514                );
1515                results.push(batches.pretty_print().unwrap());
1516            }
1517
1518            // An existing metric attached to another physical table must use
1519            // the original routing, not the request's default physical table.
1520            for sql in [
1521                "CREATE TABLE custom_physical (greptime_timestamp TIMESTAMP TIME INDEX, greptime_value DOUBLE) ENGINE=metric WITH ('physical_metric_table'='')",
1522                "CREATE TABLE custom_total (greptime_timestamp TIMESTAMP(3) TIME INDEX, greptime_value DOUBLE, \"stream\" STRING PRIMARY KEY) ENGINE=metric WITH ('on_physical_table'='custom_physical')",
1523            ] {
1524                instance.do_query(sql, ctx.clone()).await.remove(0).unwrap();
1525            }
1526            instance
1527                .metrics(
1528                    build_sum_request("custom", AggregationTemporality::Cumulative, &[(60, 7)]),
1529                    ctx.clone(),
1530                )
1531                .await
1532                .unwrap();
1533            let output = instance
1534                .do_query("SELECT greptime_value FROM custom_total", ctx.clone())
1535                .await
1536                .remove(0)
1537                .unwrap();
1538            let OutputData::Stream(stream) = output.data else {
1539                panic!("expected stream")
1540            };
1541            assert!(
1542                RecordBatches::try_collect(stream)
1543                    .await
1544                    .unwrap()
1545                    .pretty_print()
1546                    .unwrap()
1547                    .contains("7.0")
1548            );
1549
1550            // Request-level schema policy cannot be bypassed by batching.
1551            let mut fixed = ctx.fork();
1552            fixed.set_extension("auto_create_table", "false");
1553            assert!(
1554                instance
1555                    .metrics(
1556                        build_sum_request(
1557                            "missing",
1558                            AggregationTemporality::Cumulative,
1559                            &[(60, 1)]
1560                        ),
1561                        Arc::new(fixed)
1562                    )
1563                    .await
1564                    .is_err()
1565            );
1566        }
1567        assert_eq!(results[..3], results[3..]);
1568    }
1569
1570    #[tokio::test(flavor = "multi_thread")]
1571    pub async fn test_otlp_on_standalone() {
1572        let standalone = GreptimeDbStandaloneBuilder::new("test_standalone_otlp")
1573            .build()
1574            .await;
1575        let instance = standalone.fe_instance();
1576
1577        test_otlp(instance).await;
1578    }
1579
1580    #[tokio::test(flavor = "multi_thread")]
1581    pub async fn test_otlp_on_distributed() {
1582        let instance = tests::create_distributed_instance("test_standalone_otlp").await;
1583
1584        test_otlp(&instance.frontend()).await;
1585    }
1586
1587    #[tokio::test(flavor = "multi_thread")]
1588    pub async fn test_otlp_fixed_schema_rejects_missing_temporality_tag() {
1589        let standalone = GreptimeDbStandaloneBuilder::new("test_otlp_fixed_schema")
1590            .with_auto_create_table(false)
1591            .build()
1592            .await;
1593        let instance = standalone.fe_instance();
1594        let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, "public"));
1595        let mut output = instance
1596            .do_query(
1597                "CREATE TABLE fixed_delta_total (\
1598                 \"stream\" STRING, greptime_timestamp TIMESTAMP(3) NOT NULL, \
1599                 greptime_value DOUBLE, TIME INDEX (greptime_timestamp), \
1600                 PRIMARY KEY (\"stream\")) ENGINE=mito",
1601                ctx.clone(),
1602            )
1603            .await;
1604        let result = output.remove(0);
1605        assert!(result.is_ok(), "{result:?}");
1606
1607        let error = instance
1608            .metrics(
1609                build_sum_request("fixed.delta", AggregationTemporality::Delta, &[(60, 10)]),
1610                ctx.clone(),
1611            )
1612            .await
1613            .unwrap_err();
1614        assert!(format!("{error:?}").contains("otlp_aggregation_temporality"));
1615
1616        let mut output = instance
1617            .do_query("SELECT COUNT(*) FROM fixed_delta_total", ctx)
1618            .await;
1619        let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1620            unreachable!()
1621        };
1622        assert!(
1623            RecordBatches::try_collect(stream)
1624                .await
1625                .unwrap()
1626                .pretty_print()
1627                .unwrap()
1628                .contains("| 0        |")
1629        );
1630    }
1631
1632    async fn test_otlp(instance: &Arc<Instance>) {
1633        let req = build_request();
1634        let db = "otlp";
1635        let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
1636
1637        assert!(
1638            SqlQueryHandler::do_query(
1639                instance.as_ref(),
1640                &format!("CREATE DATABASE IF NOT EXISTS {db}"),
1641                ctx.clone(),
1642            )
1643            .await
1644            .first()
1645            .unwrap()
1646            .is_ok()
1647        );
1648
1649        let resp = instance.metrics(req, ctx.clone()).await;
1650        assert!(resp.is_ok());
1651
1652        let mut output = instance
1653            .do_query(
1654                "CREATE TABLE raw_delta_mito_total (\
1655                 \"stream\" STRING, greptime_timestamp TIMESTAMP(3) NOT NULL, \
1656                 greptime_value DOUBLE, TIME INDEX (greptime_timestamp), \
1657                 PRIMARY KEY (\"stream\")) ENGINE=mito",
1658                ctx.clone(),
1659            )
1660            .await;
1661        let result = output.remove(0);
1662        assert!(result.is_ok(), "{result:?}");
1663
1664        for (metric, table) in [
1665            ("raw.delta", "raw_delta_total"),
1666            ("raw.delta.mito", "raw_delta_mito_total"),
1667        ] {
1668            for request in [
1669                build_sum_request(metric, AggregationTemporality::Cumulative, &[(60, 10)]),
1670                build_sum_request(
1671                    metric,
1672                    AggregationTemporality::Delta,
1673                    &[(60, 10), (120, 20), (180, 15)],
1674                ),
1675                build_sum_request(metric, AggregationTemporality::Cumulative, &[(180, 30)]),
1676            ] {
1677                let result = instance.metrics(request, ctx.clone()).await;
1678                assert!(result.is_ok(), "{metric}: {result:?}");
1679            }
1680
1681            let mut output = instance
1682                .do_query(
1683                    &format!(
1684                        "SELECT COALESCE(otlp_aggregation_temporality, '') AS temporality, \
1685                         COUNT(*) AS samples, SUM(greptime_value) AS total \
1686                         FROM {table} GROUP BY otlp_aggregation_temporality ORDER BY temporality"
1687                    ),
1688                    ctx.clone(),
1689                )
1690                .await;
1691            let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1692                unreachable!()
1693            };
1694            assert_eq!(
1695                RecordBatches::try_collect(stream)
1696                    .await
1697                    .unwrap()
1698                    .pretty_print()
1699                    .unwrap(),
1700                "\
1701+-------------+---------+-------+
1702| temporality | samples | total |
1703+-------------+---------+-------+
1704|             | 2       | 40.0  |
1705| delta       | 3       | 45.0  |
1706+-------------+---------+-------+"
1707            );
1708
1709            for (function, expected) in [("increase", "45.0"), ("rate", "0.25")] {
1710                let mut output = instance
1711                    .do_query(
1712                        &format!("TQL EVAL (180, 180, '1m') {function}({table}[3m])"),
1713                        ctx.clone(),
1714                    )
1715                    .await;
1716                let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1717                    unreachable!()
1718                };
1719                let rendered = RecordBatches::try_collect(stream)
1720                    .await
1721                    .unwrap()
1722                    .pretty_print()
1723                    .unwrap();
1724                assert!(rendered.contains("delta"), "{rendered}");
1725                assert!(rendered.contains(expected), "{rendered}");
1726            }
1727
1728            let mut stale = build_sum_request(metric, AggregationTemporality::Delta, &[(240, 99)]);
1729            let Some(metric::Data::Sum(sum)) = stale.resource_metrics[0].scope_metrics[0].metrics
1730                [0]
1731            .data
1732            .as_mut() else {
1733                unreachable!()
1734            };
1735            sum.data_points[0].flags = DataPointFlags::NoRecordedValueMask as u32;
1736            assert!(instance.metrics(stale, ctx.clone()).await.is_ok());
1737
1738            for (matcher, expected_rows) in [
1739                ("otlp_aggregation_temporality=\"delta\"", 0),
1740                ("otlp_aggregation_temporality!=\"delta\"", 1),
1741            ] {
1742                let mut output = instance
1743                    .do_query(
1744                        &format!("TQL EVAL (240, 240, '1m') {table}{{{matcher}}}"),
1745                        ctx.clone(),
1746                    )
1747                    .await;
1748                let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1749                    unreachable!()
1750                };
1751                let batches = RecordBatches::try_collect(stream).await.unwrap();
1752                assert_eq!(
1753                    expected_rows,
1754                    batches.iter().map(|batch| batch.num_rows()).sum::<usize>()
1755                );
1756                if expected_rows == 1 {
1757                    assert!(batches.pretty_print().unwrap().contains("30.0"));
1758                }
1759            }
1760        }
1761
1762        let malformed = ExportMetricsServiceRequest {
1763            resource_metrics: vec![ResourceMetrics {
1764                scope_metrics: vec![ScopeMetrics {
1765                    metrics: vec![Metric {
1766                        name: "rejected.delta.histogram".to_string(),
1767                        data: Some(metric::Data::Histogram(Histogram {
1768                            data_points: vec![HistogramDataPoint {
1769                                count: 1,
1770                                bucket_counts: vec![1],
1771                                explicit_bounds: vec![1.0, 2.0],
1772                                ..Default::default()
1773                            }],
1774                            aggregation_temporality: AggregationTemporality::Delta as i32,
1775                        })),
1776                        ..Default::default()
1777                    }],
1778                    ..Default::default()
1779                }],
1780                ..Default::default()
1781            }],
1782        };
1783        let outcome = instance.metrics(malformed, ctx.clone()).await.unwrap();
1784        assert_eq!(0, outcome.accepted_data_points);
1785        assert_eq!(1, outcome.rejected_data_points);
1786        assert!(
1787            outcome
1788                .error_message
1789                .as_deref()
1790                .unwrap()
1791                .contains("bucket_counts length")
1792        );
1793        let mut output = instance
1794            .do_query(
1795                "SELECT COUNT(*) FROM information_schema.tables WHERE table_name IN \
1796                 ('rejected_delta_histogram_bucket', 'rejected_delta_histogram_sum', \
1797                  'rejected_delta_histogram_count')",
1798                ctx.clone(),
1799            )
1800            .await;
1801        let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1802            unreachable!()
1803        };
1804        assert!(
1805            RecordBatches::try_collect(stream)
1806                .await
1807                .unwrap()
1808                .pretty_print()
1809                .unwrap()
1810                .contains("| 0        |")
1811        );
1812
1813        let point = |stream: &str, seconds: u64, bounds: Vec<f64>, sum| HistogramDataPoint {
1814            attributes: vec![keyvalue("stream", stream)],
1815            time_unix_nano: seconds * 1_000_000_000,
1816            count: u64::try_from(bounds.len() + 1).unwrap(),
1817            sum,
1818            bucket_counts: vec![1; bounds.len() + 1],
1819            explicit_bounds: bounds,
1820            ..Default::default()
1821        };
1822        let tombstone = |stream: &str, seconds: u64, bounds: Vec<f64>, sum| HistogramDataPoint {
1823            flags: DataPointFlags::NoRecordedValueMask as u32,
1824            ..point(stream, seconds, bounds, sum)
1825        };
1826        for points in [
1827            vec![
1828                point("same", 300, vec![1.0, 2.0], Some(4.0)),
1829                tombstone("same", 360, vec![1.0, 2.0], None),
1830            ],
1831            vec![
1832                point("changed", 420, vec![3.0, 5.0], Some(8.0)),
1833                tombstone("changed", 480, vec![1.0, 2.0], Some(99.0)),
1834            ],
1835            vec![
1836                point("boundless", 420, vec![3.0, 5.0], Some(8.0)),
1837                tombstone("boundless", 480, vec![], None),
1838            ],
1839            vec![tombstone("new", 480, vec![], None)],
1840        ] {
1841            let outcome = instance
1842                .metrics(
1843                    build_histogram_request("raw.delta.histogram", points),
1844                    ctx.clone(),
1845                )
1846                .await
1847                .unwrap();
1848            assert_eq!(0, outcome.rejected_data_points);
1849        }
1850
1851        let mut output = instance
1852            .do_query(
1853                "TQL EVAL (300, 300, '1m') \
1854                 histogram_quantile(0.5, rate(raw_delta_histogram_bucket[2m]))",
1855                ctx.clone(),
1856            )
1857            .await;
1858        let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1859            unreachable!()
1860        };
1861        let rendered = RecordBatches::try_collect(stream)
1862            .await
1863            .unwrap()
1864            .pretty_print()
1865            .unwrap();
1866        assert!(rendered.contains("1.5"), "{rendered}");
1867
1868        for (query, expected_rows) in [
1869            ("raw_delta_histogram_bucket{stream=\"same\",le=~\"1|2\"}", 0),
1870            ("raw_delta_histogram_sum{stream=\"same\"}", 1),
1871            (
1872                "raw_delta_histogram_bucket{stream=\"changed\",le=~\"3|5\"}",
1873                2,
1874            ),
1875            ("raw_delta_histogram_sum{stream=\"changed\"}", 0),
1876            (
1877                "raw_delta_histogram_bucket{stream=\"boundless\",le=~\"3|5\"}",
1878                2,
1879            ),
1880        ] {
1881            let mut output = instance
1882                .do_query(&format!("TQL EVAL (480, 480, '1m') {query}"), ctx.clone())
1883                .await;
1884            let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1885                unreachable!()
1886            };
1887            let batches = RecordBatches::try_collect(stream).await.unwrap();
1888            assert_eq!(
1889                expected_rows,
1890                batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
1891                "{query}: {}",
1892                batches.pretty_print().unwrap()
1893            );
1894        }
1895
1896        let mut output = instance
1897            .do_query(
1898                "SELECT \
1899                   (SELECT COUNT(*) FROM raw_delta_histogram_bucket WHERE \"stream\" = 'new') AS buckets, \
1900                   (SELECT COUNT(*) FROM raw_delta_histogram_count WHERE \"stream\" = 'new') AS counts, \
1901                   (SELECT COUNT(*) FROM raw_delta_histogram_sum WHERE \"stream\" = 'new') AS sums",
1902                ctx.clone(),
1903            )
1904            .await;
1905        let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
1906            unreachable!()
1907        };
1908        let rendered = RecordBatches::try_collect(stream)
1909            .await
1910            .unwrap()
1911            .pretty_print()
1912            .unwrap();
1913        assert!(
1914            rendered.contains("| 1       | 1      | 0    |"),
1915            "{rendered}"
1916        );
1917
1918        let mut output = instance
1919            .do_query(
1920                "SELECT * FROM my_test_metric_my_ignored_unit ORDER BY greptime_timestamp",
1921                ctx.clone(),
1922            )
1923            .await;
1924        let output = output.remove(0).unwrap();
1925        let OutputData::Stream(stream) = output.data else {
1926            unreachable!()
1927        };
1928        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
1929        assert_eq!(
1930            recordbatches.pretty_print().unwrap(),
1931            "\
1932+----------------+---------------------+----------------+
1933| container_name | greptime_timestamp  | greptime_value |
1934+----------------+---------------------+----------------+
1935| testserver     | 1970-01-01T00:00:00 | 105.0          |
1936| testsevrer     | 1970-01-01T00:00:00 | 100.0          |
1937+----------------+---------------------+----------------+",
1938        );
1939
1940        let mut output = instance
1941            .do_query(
1942                "SELECT le, greptime_value FROM my_test_histo_my_ignored_unit_bucket order by le",
1943                ctx.clone(),
1944            )
1945            .await;
1946        let output = output.remove(0).unwrap();
1947        let OutputData::Stream(stream) = output.data else {
1948            unreachable!()
1949        };
1950        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
1951        assert_eq!(
1952            recordbatches.pretty_print().unwrap(),
1953            "\
1954+-----+----------------+
1955| le  | greptime_value |
1956+-----+----------------+
1957| 1   | 1.0            |
1958| 5   | 3.0            |
1959| inf | 4.0            |
1960+-----+----------------+",
1961        );
1962
1963        let mut output = instance
1964            .do_query(
1965                "SELECT * FROM my_test_histo_my_ignored_unit_sum",
1966                ctx.clone(),
1967            )
1968            .await;
1969        let output = output.remove(0).unwrap();
1970        let OutputData::Stream(stream) = output.data else {
1971            unreachable!()
1972        };
1973        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
1974        assert_eq!(
1975            recordbatches.pretty_print().unwrap(),
1976            "\
1977+------------+---------------------+----------------+
1978| host       | greptime_timestamp  | greptime_value |
1979+------------+---------------------+----------------+
1980| testserver | 1970-01-01T00:00:00 | 51.0           |
1981+------------+---------------------+----------------+",
1982        );
1983
1984        let mut output = instance
1985            .do_query(
1986                "SELECT * FROM my_test_histo_my_ignored_unit_count",
1987                ctx.clone(),
1988            )
1989            .await;
1990        let output = output.remove(0).unwrap();
1991        let OutputData::Stream(stream) = output.data else {
1992            unreachable!()
1993        };
1994        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
1995        assert_eq!(
1996            recordbatches.pretty_print().unwrap(),
1997            "\
1998+------------+---------------------+----------------+
1999| host       | greptime_timestamp  | greptime_value |
2000+------------+---------------------+----------------+
2001| testserver | 1970-01-01T00:00:00 | 4.0            |
2002+------------+---------------------+----------------+",
2003        );
2004    }
2005
2006    #[tokio::test(flavor = "multi_thread")]
2007    pub async fn test_otlp_metrics_into_microsecond_physical_table_on_standalone() {
2008        let standalone = GreptimeDbStandaloneBuilder::new("test_otlp_us_physical")
2009            .build()
2010            .await;
2011
2012        test_otlp_metrics_into_non_millisecond_physical_table(
2013            standalone.fe_instance(),
2014            "TIMESTAMP(6)",
2015            TimeUnit::Microsecond,
2016            1_704_067_200_123_456,
2017        )
2018        .await;
2019    }
2020
2021    #[tokio::test(flavor = "multi_thread")]
2022    pub async fn test_otlp_metrics_into_microsecond_physical_table_on_distributed() {
2023        let instance = tests::create_distributed_instance("test_otlp_us_physical_dist").await;
2024
2025        test_otlp_metrics_into_non_millisecond_physical_table(
2026            &instance.frontend(),
2027            "TIMESTAMP(6)",
2028            TimeUnit::Microsecond,
2029            1_704_067_200_123_456,
2030        )
2031        .await;
2032    }
2033
2034    #[tokio::test(flavor = "multi_thread")]
2035    pub async fn test_otlp_metrics_into_seconds_physical_table_on_standalone() {
2036        let standalone = GreptimeDbStandaloneBuilder::new("test_otlp_s_physical")
2037            .build()
2038            .await;
2039
2040        // Narrowing truncates: 1704067200123456789ns -> 1704067200s.
2041        test_otlp_metrics_into_non_millisecond_physical_table(
2042            standalone.fe_instance(),
2043            "TIMESTAMP(0)",
2044            TimeUnit::Second,
2045            1_704_067_200,
2046        )
2047        .await;
2048    }
2049
2050    #[tokio::test(flavor = "multi_thread")]
2051    pub async fn test_otlp_metrics_into_nanoseconds_physical_table_on_standalone() {
2052        let standalone = GreptimeDbStandaloneBuilder::new("test_otlp_ns_physical")
2053            .build()
2054            .await;
2055
2056        // Same unit: the full nanosecond precision is kept verbatim.
2057        test_otlp_metrics_into_non_millisecond_physical_table(
2058            standalone.fe_instance(),
2059            "TIMESTAMP(9)",
2060            TimeUnit::Nanosecond,
2061            1_704_067_200_123_456_789,
2062        )
2063        .await;
2064    }
2065
2066    /// Regression test for <https://github.com/GreptimeTeam/greptimedb/issues/9231>:
2067    /// a physical metric table pre-created with a non-millisecond time index
2068    /// accepts OTLP ingestion; the nanosecond samples are converted to the
2069    /// physical table's unit (kept verbatim for a nanosecond table,
2070    /// truncating the sub-unit part for micro/second tables).
2071    async fn test_otlp_metrics_into_non_millisecond_physical_table(
2072        instance: &Arc<Instance>,
2073        sql_ts_type: &str,
2074        expected_unit: TimeUnit,
2075        expected_value: i64,
2076    ) {
2077        let db = "otlp_non_ms_physical";
2078        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, db);
2079        // Route the request to the metric engine, like the OTLP HTTP handler
2080        // does with the default `prom_store.with_metric_engine = true`.
2081        ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx {
2082            with_metric_engine: true,
2083            ..Default::default()
2084        }));
2085        let ctx = Arc::new(ctx);
2086        assert!(
2087            SqlQueryHandler::do_query(
2088                instance.as_ref(),
2089                &format!("CREATE DATABASE IF NOT EXISTS {db}"),
2090                ctx.clone(),
2091            )
2092            .await
2093            .first()
2094            .unwrap()
2095            .is_ok()
2096        );
2097
2098        let mut output = instance
2099            .do_query(
2100                &format!(
2101                    "CREATE TABLE greptime_physical_table (\
2102                 greptime_timestamp {sql_ts_type} NOT NULL, \
2103                 greptime_value DOUBLE NULL, \
2104                 TIME INDEX (greptime_timestamp)) \
2105                 ENGINE = metric WITH ('physical_metric_table' = 'true')"
2106                ),
2107                ctx.clone(),
2108            )
2109            .await;
2110        assert!(output.remove(0).is_ok());
2111
2112        let request = ExportMetricsServiceRequest {
2113            resource_metrics: vec![ResourceMetrics {
2114                scope_metrics: vec![ScopeMetrics {
2115                    metrics: vec![Metric {
2116                        name: "my_gauge".to_string(),
2117                        data: Some(metric::Data::Gauge(Gauge {
2118                            data_points: vec![NumberDataPoint {
2119                                attributes: vec![keyvalue("host", "h1")],
2120                                time_unix_nano: 1_704_067_200_123_456_789,
2121                                value: Some(Value::AsDouble(1.0)),
2122                                ..Default::default()
2123                            }],
2124                        })),
2125                        ..Default::default()
2126                    }],
2127                    ..Default::default()
2128                }],
2129                ..Default::default()
2130            }],
2131        };
2132        instance.metrics(request, ctx.clone()).await.unwrap();
2133
2134        let mut output = instance
2135            .do_query("SELECT greptime_timestamp FROM my_gauge", ctx.clone())
2136            .await;
2137        let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
2138            unreachable!()
2139        };
2140        let batches = RecordBatches::try_collect(stream).await.unwrap().take();
2141        assert_eq!(batches[0].num_rows(), 1);
2142        // The logical table's time index keeps the physical table's unit.
2143        let ts_column = batches[0].column(0);
2144        assert_eq!(
2145            ts_column.data_type(),
2146            &DataType::Timestamp(expected_unit, None),
2147            "unexpected time index type"
2148        );
2149        let stored = match expected_unit {
2150            TimeUnit::Second => ts_column.as_primitive::<TimestampSecondType>().value(0),
2151            TimeUnit::Millisecond => ts_column
2152                .as_primitive::<TimestampMillisecondType>()
2153                .value(0),
2154            TimeUnit::Microsecond => ts_column
2155                .as_primitive::<TimestampMicrosecondType>()
2156                .value(0),
2157            TimeUnit::Nanosecond => ts_column.as_primitive::<TimestampNanosecondType>().value(0),
2158        };
2159        assert_eq!(stored, expected_value);
2160    }
2161
2162    fn build_request() -> ExportMetricsServiceRequest {
2163        let data_points = vec![
2164            NumberDataPoint {
2165                attributes: vec![keyvalue("container.name", "testsevrer")],
2166                time_unix_nano: 100,
2167                value: Some(Value::AsInt(100)),
2168                ..Default::default()
2169            },
2170            NumberDataPoint {
2171                attributes: vec![keyvalue("container.name", "testserver")],
2172                time_unix_nano: 105,
2173                value: Some(Value::AsInt(105)),
2174                ..Default::default()
2175            },
2176        ];
2177        let gauge = Gauge { data_points };
2178
2179        let histo_data_points = vec![HistogramDataPoint {
2180            attributes: vec![keyvalue("host", "testserver")],
2181            time_unix_nano: 100,
2182            count: 4,
2183            bucket_counts: vec![1, 2, 1],
2184            explicit_bounds: vec![1.0f64, 5.0f64],
2185            sum: Some(51f64),
2186            ..Default::default()
2187        }];
2188
2189        let histo = Histogram {
2190            data_points: histo_data_points,
2191            aggregation_temporality: 0,
2192        };
2193
2194        ExportMetricsServiceRequest {
2195            resource_metrics: vec![ResourceMetrics {
2196                scope_metrics: vec![ScopeMetrics {
2197                    metrics: vec![
2198                        Metric {
2199                            name: "my.test.metric".into(),
2200                            description: "my ignored desc".into(),
2201                            unit: "my ignored unit".into(),
2202                            metadata: vec![],
2203                            data: Some(metric::Data::Gauge(gauge)),
2204                        },
2205                        Metric {
2206                            name: "my.test.histo".into(),
2207                            description: "my ignored desc".into(),
2208                            unit: "my ignored unit".into(),
2209                            metadata: vec![],
2210                            data: Some(metric::Data::Histogram(histo)),
2211                        },
2212                    ],
2213                    scope: Some(InstrumentationScope {
2214                        attributes: vec![
2215                            keyvalue("scope", "otel"),
2216                            keyvalue("telemetry.sdk.name", "java"),
2217                        ],
2218                        ..Default::default()
2219                    }),
2220                    ..Default::default()
2221                }],
2222                resource: Some(Resource {
2223                    attributes: vec![keyvalue("resource", "greptimedb")],
2224                    dropped_attributes_count: 0,
2225                    entity_refs: vec![],
2226                }),
2227                ..Default::default()
2228            }],
2229        }
2230    }
2231
2232    fn build_sum_request(
2233        name: &str,
2234        temporality: AggregationTemporality,
2235        points: &[(u64, i64)],
2236    ) -> ExportMetricsServiceRequest {
2237        let data_points = points
2238            .iter()
2239            .map(|(seconds, value)| NumberDataPoint {
2240                attributes: vec![keyvalue("stream", "same")],
2241                time_unix_nano: *seconds * 1_000_000_000,
2242                value: Some(Value::AsInt(*value)),
2243                ..Default::default()
2244            })
2245            .collect();
2246        ExportMetricsServiceRequest {
2247            resource_metrics: vec![ResourceMetrics {
2248                scope_metrics: vec![ScopeMetrics {
2249                    metrics: vec![Metric {
2250                        name: name.to_string(),
2251                        data: Some(metric::Data::Sum(Sum {
2252                            data_points,
2253                            aggregation_temporality: temporality as i32,
2254                            is_monotonic: true,
2255                        })),
2256                        ..Default::default()
2257                    }],
2258                    ..Default::default()
2259                }],
2260                ..Default::default()
2261            }],
2262        }
2263    }
2264
2265    fn build_histogram_request(
2266        name: &str,
2267        points: Vec<HistogramDataPoint>,
2268    ) -> ExportMetricsServiceRequest {
2269        ExportMetricsServiceRequest {
2270            resource_metrics: vec![ResourceMetrics {
2271                scope_metrics: vec![ScopeMetrics {
2272                    metrics: vec![Metric {
2273                        name: name.to_string(),
2274                        data: Some(metric::Data::Histogram(Histogram {
2275                            data_points: points,
2276                            aggregation_temporality: AggregationTemporality::Delta as i32,
2277                        })),
2278                        ..Default::default()
2279                    }],
2280                    ..Default::default()
2281                }],
2282                ..Default::default()
2283            }],
2284        }
2285    }
2286
2287    fn keyvalue(key: &str, value: &str) -> KeyValue {
2288        KeyValue {
2289            key: key.into(),
2290            value: Some(AnyValue {
2291                value: Some(Val::StringValue(value.into())),
2292            }),
2293        }
2294    }
2295}