// Copyright 2025 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
use config::meta::stream::FileKey;
pub async fn file_list(
mode: &str,
stream: &str,
hour: &str,
group_size: &str,
) -> Result<(), anyhow::Error> {
let parts = stream.splitn(3, '/').collect::>();
let org = parts[0];
let stream_type = parts[1];
let stream_name = parts[2];
println!("Testing group stream files:");
println!("stream: {stream}");
println!("date hour: {hour}");
println!("strategy: {mode}");
println!("group size: {group_size}gb");
let file_list = crate::service::file_list::query_for_merge(
org,
stream_name,
stream_type.into(),
hour,
hour,
)
.await?;
println!("get files: {}", file_list.len());
let group_size = group_size.parse::().unwrap_or(5) * 1024 * 1024 * 1024;
let mode = mode.trim().to_lowercase();
let groups = match mode.as_str() {
"file_size" => group_by_file_size(file_list, group_size),
"file_time" => group_by_file_time(file_list, group_size),
"time_range" => group_by_time_range(file_list, group_size),
_ => {
return Err(anyhow::anyhow!("unsupported mode: {mode}"));
}
};
println!("groups: {}", groups.len());
Ok(())
}
fn group_by_file_size(mut file_list: Vec, group_size: i64) -> Vec> {
file_list.sort_by_key(|f| f.meta.original_size);
let mut groups = Vec::new();
let mut current_group = Vec::new();
let mut current_size = 0;
for file in file_list {
if !current_group.is_empty() && current_size + file.meta.original_size > group_size {
groups.push(current_group);
current_group = Vec::new();
current_size = 0;
}
current_size += file.meta.original_size;
current_group.push(file);
}
if !current_group.is_empty() {
groups.push(current_group);
}
groups
}
fn group_by_file_time(mut file_list: Vec, group_size: i64) -> Vec> {
file_list.sort_by_key(|f| f.meta.min_ts);
let mut groups = Vec::new();
let mut current_group = Vec::new();
let mut current_size = 0;
for file in file_list {
if !current_group.is_empty() && current_size + file.meta.original_size > group_size {
groups.push(current_group);
current_group = Vec::new();
current_size = 0;
}
current_size += file.meta.original_size;
current_group.push(file);
}
if !current_group.is_empty() {
groups.push(current_group);
}
groups
}
fn group_by_time_range(mut file_list: Vec, group_size: i64) -> Vec> {
// first group file by non-overlapping
file_list.sort_by_key(|f| f.meta.min_ts);
let mut groups: Vec> = Vec::new();
for file in file_list {
let mut inserted = None;
for (i, group) in groups.iter().enumerate() {
if group
.last()
.map(|f| file.meta.min_ts >= f.meta.max_ts)
.unwrap_or(false)
{
inserted = Some(i);
break;
}
}
if let Some(i) = inserted {
groups[i].push(file);
} else {
groups.push(vec![file]);
}
}
// then sort each group by file size
let mut new_groups = Vec::new();
for group in groups {
new_groups.extend(group_by_file_time(group, group_size));
}
new_groups
}