diff --git a/src/promql/src/vector_match.rs b/src/promql/src/vector_match.rs index 66a534cd12b..7f8bdbd26a3 100644 --- a/src/promql/src/vector_match.rs +++ b/src/promql/src/vector_match.rs @@ -1223,7 +1223,7 @@ fn set_vectors_inner( /// The figures one set field's phases are charged from, read once from its operands. struct SetFieldMetadata { - /// The filter side's match groups. + /// The filter side's match-group table, without the presence bitmaps it holds. tables: usize, /// The widest key one row builds: the match key a kept series is looked up by, or the key a row /// is filed under. @@ -1241,7 +1241,6 @@ fn set_field_metadata( for series in filter.series { tables = tables .saturating_add(match_key_bytes(&series.metric, labels)) - .saturating_add(2 * std::mem::size_of::()) .saturating_add(GROUP_SLOT_BYTES); } // `or` files the filter side's rows too, so both sides' label sets bound a row's keys and the @@ -1276,8 +1275,8 @@ fn set_field( SetOp::Unless => (lhs, rhs, false), SetOp::Or => (rhs, lhs, false), }; - // The walk is one phase, entered before the group table exists. A kept series reads its - // candidates through one cursor each, so the widest group's cursors join it once it does. + // The walk is one phase, entered before the group table exists; each group's bitmap joins it + // as the group is first met. let metadata = match budget.as_deref_mut() { Some(budget) => { let metadata = set_field_metadata(labels, kept, filter); @@ -1286,11 +1285,12 @@ fn set_field( } None => None, }; - let filter_groups = group(filter, labels); - if let Some(budget) = budget.as_deref_mut() { - let widest = filter_groups.values().map(Vec::len).max().unwrap_or(0); - budget.grow(widest.saturating_mul(std::mem::size_of::>()))?; - } + let presence = group_presence( + filter, + labels, + grid.spec().len as usize, + budget.as_deref_mut(), + )?; // A set operator hands on its operands' label sets unchanged, so a row is filed under the label // set of the series it came from. let charged = budget.is_some(); @@ -1323,29 +1323,14 @@ fn set_field( } for (index, series) in kept.series.iter().enumerate() { let label_set = label_set(&series.metric); - let candidates = filter_groups.get(&group_key(&series.metric, labels)); - let candidate_indices: &[usize] = candidates.map_or(&[], Vec::as_slice); - // One forward cursor per candidate: `kept`'s positions only increase, so the row a - // candidate is read at only increases as well. - let mut candidate_cursors: Vec> = candidate_indices - .iter() - .map(|&candidate| filter.cursor(candidate)) - .collect(); + let bits = presence.get(&group_key(&series.metric, labels)); let mut cursor = kept.cursor(index); for (point, row) in kept.positions(index) { let sample = cursor.at(row).expect("a row never goes back"); if matches!(sample, SampleRef::Missing) { continue; } - let mut matched = false; - for (&candidate, candidate_cursor) in - candidate_indices.iter().zip(&mut candidate_cursors) - { - if filter.present(candidate, point, candidate_cursor) { - matched = true; - break; - } - } + let matched = bits.is_some_and(|bits| bits[point / 64] & (1 << (point % 64)) != 0); if matched == keep_matched { results.insert( &series.metric, @@ -1360,7 +1345,7 @@ fn set_field( } // The group table is gone before the sort phase, which is charged without it: the message the // sort may append and one slot per row it sorts, entered before it allocates either. - drop(filter_groups); + drop(presence); if let (Some(budget), Some(metadata)) = (budget.as_deref_mut(), &metadata) { budget.phase( metadata @@ -1382,6 +1367,43 @@ fn set_field( Ok(results) } +/// The points at which each match group of `operand` holds a sample, one bit per grid point. +/// +/// Prometheus decides a set operator at a step from the signatures present on the filter side, so +/// a kept sample needs only whether its group is present at its point: built once from the filter +/// side's rows, that is one lookup per kept sample however many series share the group. A group's +/// bitmap is charged as the group is first met, before it is allocated. +fn group_presence( + operand: &Operand<'_>, + labels: &MatchingLabels, + points: usize, + mut budget: Option<&mut Budget<'_>>, +) -> Result>> { + let words = points.div_ceil(64); + let mut presence: HashMap> = HashMap::new(); + for (index, series) in operand.series.iter().enumerate() { + let bits = match presence.entry(group_key(&series.metric, labels)) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + if let Some(budget) = budget.as_deref_mut() { + budget.grow(words.saturating_mul(std::mem::size_of::()))?; + } + entry.insert(vec![0; words]) + } + }; + let mut cursor = operand.cursor(index); + for (point, row) in operand.positions(index) { + if !matches!( + cursor.at(row).expect("a row never goes back"), + SampleRef::Missing + ) { + bits[point / 64] |= 1 << (point % 64); + } + } + } + Ok(presence) +} + /// Joins the per-field results into series with all fields, by label set. /// /// The merge is one phase, entered from the results the fields filed: the triples it groups, the diff --git a/src/promql/src/vector_match/tests.rs b/src/promql/src/vector_match/tests.rs index fecc9578e05..05bccd13ece 100644 --- a/src/promql/src/vector_match/tests.rs +++ b/src/promql/src/vector_match/tests.rs @@ -3442,3 +3442,96 @@ fn a_set_operation_holds_the_rows_it_keeps() { "`and` holds {and} bytes and `unless` {unless}" ); } + +/// Membership is read from each match group's presence, however many series share the group and +/// however sparse each of them is: one group here holds sixteen filter series that each carry a +/// few points, the grid spans several 64-point words, and every operator, with either side as the +/// filter, agrees with the step-by-step reference. +#[test] +fn set_membership_of_crowded_sparse_groups_matches_the_reference() { + let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 10, 150).unwrap()).unwrap()); + let points = grid.spec().len as usize; + let schema = schema(1, &grid); + let side = |rng: &mut StdRng, name: &str, jobs: &[(&str, usize)], density: f64| -> Side { + let mut builder = GridSeriesBuilder::try_new(Arc::new(schema.clone())).unwrap(); + let (mut series, mut dense) = (Vec::new(), Vec::new()); + for (job, count) in jobs { + for member in 0..*count { + let labels = Labels::from([ + ("__name__".to_string(), name.to_string()), + ("job".to_string(), job.to_string()), + ("host".to_string(), format!("{name}{member}")), + ]); + let values: Dense = (0..points) + .map(|point| vec![rng.random_bool(density).then_some(point as f64)]) + .collect(); + builder.begin_series(); + for (point, row) in values.iter().enumerate() { + builder.push(point, row).unwrap(); + } + series.extend(builder.finish(metric(&labels)).unwrap()); + dense.push((labels, values)); + } + } + Side { + schema: schema.clone(), + series, + dense, + } + }; + let labels = MatchingLabels::On(vec![Arc::from("job")]); + for seed in 0..20 { + let mut rng = StdRng::seed_from_u64(seed); + let crowded = side(&mut rng, "r", &[("x", 16), ("y", 2)], 0.05); + let dense = side(&mut rng, "l", &[("x", 2), ("y", 2), ("z", 1)], 0.6); + for (lhs, rhs) in [(&dense, &crowded), (&crowded, &dense)] { + for op in [SetOp::And, SetOp::Or, SetOp::Unless] { + let expected = reference_set(op, &labels, &lhs.dense, &rhs.dense, points); + let actual = set_vectors( + op, + &labels, + &lhs.schema, + &lhs.series, + &rhs.schema, + &rhs.series, + ); + assert_output(actual, expected, seed); + } + } + } +} + +/// Each match group's presence bitmap is charged: one sample per series, so a longer grid adds +/// nothing but the bitmaps, which grow by one 64-bit word per group per 64 points. +#[test] +fn a_set_operation_charges_its_presence_bitmaps() { + const GROUPS: usize = 32; + let peak = |points: usize| { + let grid = budget_grid(points); + let schema = Arc::new(schema(1, &grid)); + let lhs = vec![budget_operand(&schema, "a", &[0])]; + let rhs: Vec = (0..GROUPS) + .map(|group| jobbed_operand(&schema, "b", &format!("j{group}"), &[0])) + .collect(); + let mut seen = Vec::new(); + set_vectors_with_reservation( + SetOp::Unless, + &MatchingLabels::On(vec![Arc::from("job")]), + &schema, + &lhs, + &schema, + &rhs, + &mut |total| { + seen.push(total); + Ok(()) + }, + ) + .unwrap(); + seen.into_iter().max().unwrap() + }; + let (short, long) = (peak(64), peak(64 * 65)); + assert!( + long >= short + GROUPS * 64 * std::mem::size_of::(), + "64 points peak at {short} bytes and 4160 at {long}" + ); +}