From 9d281b401fd174686c4be846c37752bda2993215 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Mon, 26 Oct 2026 17:04:11 +0800 Subject: [PATCH] perf(promql): decide set membership from a per-group presence bitmap The set kernel checked every filter series of a kept series' match group at every kept point, so a coarse `on()` / `on(...)` cost O(kept series x points x group size). Prometheus decides a step from the set of signatures present on the filter side. Build one bitmap per filter match group, one bit per grid point, from the filter side's rows, and look each kept sample up in O(1). Each bitmap is charged to the walk phase when its group is first met and is released with the group table before the sort phase. Results are unchanged. Signed-off-by: Dennis Zhuang --- src/promql/src/vector_match.rs | 76 +++++++++++++++-------- src/promql/src/vector_match/tests.rs | 93 ++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 27 deletions(-) diff --git a/src/promql/src/vector_match.rs b/src/promql/src/vector_match.rs index 66a534cd12b..7f8bdbd26a3 100644 --- a/src/promql/src/vector_match.rs +++ b/src/promql/src/vector_match.rs @@ -1223,7 +1223,7 @@ fn set_vectors_inner( /// The figures one set field's phases are charged from, read once from its operands. struct SetFieldMetadata { - /// The filter side's match groups. + /// The filter side's match-group table, without the presence bitmaps it holds. tables: usize, /// The widest key one row builds: the match key a kept series is looked up by, or the key a row /// is filed under. @@ -1241,7 +1241,6 @@ fn set_field_metadata( for series in filter.series { tables = tables .saturating_add(match_key_bytes(&series.metric, labels)) - .saturating_add(2 * std::mem::size_of::()) .saturating_add(GROUP_SLOT_BYTES); } // `or` files the filter side's rows too, so both sides' label sets bound a row's keys and the @@ -1276,8 +1275,8 @@ fn set_field( SetOp::Unless => (lhs, rhs, false), SetOp::Or => (rhs, lhs, false), }; - // The walk is one phase, entered before the group table exists. A kept series reads its - // candidates through one cursor each, so the widest group's cursors join it once it does. + // The walk is one phase, entered before the group table exists; each group's bitmap joins it + // as the group is first met. let metadata = match budget.as_deref_mut() { Some(budget) => { let metadata = set_field_metadata(labels, kept, filter); @@ -1286,11 +1285,12 @@ fn set_field( } None => None, }; - let filter_groups = group(filter, labels); - if let Some(budget) = budget.as_deref_mut() { - let widest = filter_groups.values().map(Vec::len).max().unwrap_or(0); - budget.grow(widest.saturating_mul(std::mem::size_of::>()))?; - } + let presence = group_presence( + filter, + labels, + grid.spec().len as usize, + budget.as_deref_mut(), + )?; // A set operator hands on its operands' label sets unchanged, so a row is filed under the label // set of the series it came from. let charged = budget.is_some(); @@ -1323,29 +1323,14 @@ fn set_field( } for (index, series) in kept.series.iter().enumerate() { let label_set = label_set(&series.metric); - let candidates = filter_groups.get(&group_key(&series.metric, labels)); - let candidate_indices: &[usize] = candidates.map_or(&[], Vec::as_slice); - // One forward cursor per candidate: `kept`'s positions only increase, so the row a - // candidate is read at only increases as well. - let mut candidate_cursors: Vec> = candidate_indices - .iter() - .map(|&candidate| filter.cursor(candidate)) - .collect(); + let bits = presence.get(&group_key(&series.metric, labels)); let mut cursor = kept.cursor(index); for (point, row) in kept.positions(index) { let sample = cursor.at(row).expect("a row never goes back"); if matches!(sample, SampleRef::Missing) { continue; } - let mut matched = false; - for (&candidate, candidate_cursor) in - candidate_indices.iter().zip(&mut candidate_cursors) - { - if filter.present(candidate, point, candidate_cursor) { - matched = true; - break; - } - } + let matched = bits.is_some_and(|bits| bits[point / 64] & (1 << (point % 64)) != 0); if matched == keep_matched { results.insert( &series.metric, @@ -1360,7 +1345,7 @@ fn set_field( } // The group table is gone before the sort phase, which is charged without it: the message the // sort may append and one slot per row it sorts, entered before it allocates either. - drop(filter_groups); + drop(presence); if let (Some(budget), Some(metadata)) = (budget.as_deref_mut(), &metadata) { budget.phase( metadata @@ -1382,6 +1367,43 @@ fn set_field( Ok(results) } +/// The points at which each match group of `operand` holds a sample, one bit per grid point. +/// +/// Prometheus decides a set operator at a step from the signatures present on the filter side, so +/// a kept sample needs only whether its group is present at its point: built once from the filter +/// side's rows, that is one lookup per kept sample however many series share the group. A group's +/// bitmap is charged as the group is first met, before it is allocated. +fn group_presence( + operand: &Operand<'_>, + labels: &MatchingLabels, + points: usize, + mut budget: Option<&mut Budget<'_>>, +) -> Result>> { + let words = points.div_ceil(64); + let mut presence: HashMap> = HashMap::new(); + for (index, series) in operand.series.iter().enumerate() { + let bits = match presence.entry(group_key(&series.metric, labels)) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + if let Some(budget) = budget.as_deref_mut() { + budget.grow(words.saturating_mul(std::mem::size_of::()))?; + } + entry.insert(vec![0; words]) + } + }; + let mut cursor = operand.cursor(index); + for (point, row) in operand.positions(index) { + if !matches!( + cursor.at(row).expect("a row never goes back"), + SampleRef::Missing + ) { + bits[point / 64] |= 1 << (point % 64); + } + } + } + Ok(presence) +} + /// Joins the per-field results into series with all fields, by label set. /// /// The merge is one phase, entered from the results the fields filed: the triples it groups, the diff --git a/src/promql/src/vector_match/tests.rs b/src/promql/src/vector_match/tests.rs index fecc9578e05..05bccd13ece 100644 --- a/src/promql/src/vector_match/tests.rs +++ b/src/promql/src/vector_match/tests.rs @@ -3442,3 +3442,96 @@ fn a_set_operation_holds_the_rows_it_keeps() { "`and` holds {and} bytes and `unless` {unless}" ); } + +/// Membership is read from each match group's presence, however many series share the group and +/// however sparse each of them is: one group here holds sixteen filter series that each carry a +/// few points, the grid spans several 64-point words, and every operator, with either side as the +/// filter, agrees with the step-by-step reference. +#[test] +fn set_membership_of_crowded_sparse_groups_matches_the_reference() { + let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 10, 150).unwrap()).unwrap()); + let points = grid.spec().len as usize; + let schema = schema(1, &grid); + let side = |rng: &mut StdRng, name: &str, jobs: &[(&str, usize)], density: f64| -> Side { + let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone())).unwrap(); + let (mut series, mut dense) = (Vec::new(), Vec::new()); + for (job, count) in jobs { + for member in 0..*count { + let labels = Labels::from([ + ("__name__".to_string(), name.to_string()), + ("job".to_string(), job.to_string()), + ("host".to_string(), format!("{name}{member}")), + ]); + let values: Dense = (0..points) + .map(|point| vec![rng.random_bool(density).then_some(point as f64)]) + .collect(); + builder.begin_series(); + for (point, row) in values.iter().enumerate() { + builder.push(point, row).unwrap(); + } + series.extend(builder.finish(metric(&labels)).unwrap()); + dense.push((labels, values)); + } + } + Side { + schema: schema.clone(), + series, + dense, + } + }; + let labels = MatchingLabels::On(vec![Arc::from("job")]); + for seed in 0..20 { + let mut rng = StdRng::seed_from_u64(seed); + let crowded = side(&mut rng, "r", &[("x", 16), ("y", 2)], 0.05); + let dense = side(&mut rng, "l", &[("x", 2), ("y", 2), ("z", 1)], 0.6); + for (lhs, rhs) in [(&dense, &crowded), (&crowded, &dense)] { + for op in [SetOp::And, SetOp::Or, SetOp::Unless] { + let expected = reference_set(op, &labels, &lhs.dense, &rhs.dense, points); + let actual = set_vectors( + op, + &labels, + &lhs.schema, + &lhs.series, + &rhs.schema, + &rhs.series, + ); + assert_output(actual, expected, seed); + } + } + } +} + +/// Each match group's presence bitmap is charged: one sample per series, so a longer grid adds +/// nothing but the bitmaps, which grow by one 64-bit word per group per 64 points. +#[test] +fn a_set_operation_charges_its_presence_bitmaps() { + const GROUPS: usize = 32; + let peak = |points: usize| { + let grid = budget_grid(points); + let schema = Arc::new(schema(1, &grid)); + let lhs = vec![budget_operand(&schema, "a", &[0])]; + let rhs: Vec = (0..GROUPS) + .map(|group| jobbed_operand(&schema, "b", &format!("j{group}"), &[0])) + .collect(); + let mut seen = Vec::new(); + set_vectors_with_reservation( + SetOp::Unless, + &MatchingLabels::On(vec![Arc::from("job")]), + &schema, + &lhs, + &schema, + &rhs, + &mut |total| { + seen.push(total); + Ok(()) + }, + ) + .unwrap(); + seen.into_iter().max().unwrap() + }; + let (short, long) = (peak(64), peak(64 * 65)); + assert!( + long >= short + GROUPS * 64 * std::mem::size_of::(), + "64 points peak at {short} bytes and 4160 at {long}" + ); +}