starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-06 08:09:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: update datafusion to 34 (#2222)

Browse Source
This commit is contained in:
Hengfei Yang
2023-12-18 10:49:08 +08:00
committed by GitHub
parent a862b8b87b
commit 27c8d2420f
24 changed files with 556 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/data/cli.rs
+8 -10
View File
@@ -29,7 +29,7 @@ pub struct Cli {
pub end_time: i64,
}
pub fn args() -> impl IntoIterator<Item=impl Into<clap::Arg>> {
pub fn args() -> impl IntoIterator<Item = impl Into<clap::Arg>> {
[
clap::Arg::new("stream").long("stream").alias("c").default_value("stream")
.help("for stream data, maybe we can export stream, users, metadata, but for now we can only support stream"),
@@ -87,18 +87,16 @@ fn handle_args(app: clap::ArgMatches) -> Cli {
let start = match app.get_one::<String>("start_time") {
Some(time) => time.parse::<i64>().unwrap(),
None => {
Local::now()
.date_naive()
.and_hms_milli_opt(0, 0, 0, 0)
.unwrap()
.timestamp_micros()
}
None => Local::now()
.date_naive()
.and_hms_milli_opt(0, 0, 0, 0)
.unwrap()
.timestamp_micros(),
};
let end = match app.get_one::<String>("end_time") {
Some(time) => time.parse::<i64>().unwrap(),
None => { (Local::now() - Duration::hours(1)).timestamp_micros() }
None => (Local::now() - Duration::hours(1)).timestamp_micros(),
};
Cli {
@@ -112,4 +110,4 @@ fn handle_args(app: clap::ArgMatches) -> Cli {
start_time: start,
end_time: end,
}
}
}
src/cli/data/export.rs
+20 -13
View File
@@ -13,21 +13,24 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::HashMap;
use std::fs::{self};
use std::fs::File;
use std::path::Path;
use std::{
collections::HashMap,
fs::{self, File},
path::Path,
};
use actix_web::web::Query;
use ahash::AHashMap;
use crate::cli::data::cli::Cli;
use crate::cli::data::Context;
use crate::common::infra::config::CONFIG;
use crate::common::meta::search::Request;
use crate::common::meta::StreamType;
use crate::common::utils::http::get_stream_type_from_request;
use crate::service::search as SearchService;
use crate::{
cli::data::{cli::Cli, Context},
common::{
infra::config::CONFIG,
meta::{search::Request, StreamType},
utils::http::get_stream_type_from_request,
},
service::search as SearchService,
};
pub struct Export {}
@@ -72,7 +75,11 @@ impl Context for Export {
}
let path = Path::new(c.data.as_str());
fs::create_dir_all(path)?;
let file = File::create(path.join(format!("{}.{}", chrono::Local::now().timestamp_micros(), c.file_type)))?;
let file = File::create(path.join(format!(
"{}.{}",
chrono::Local::now().timestamp_micros(),
c.file_type
)))?;
serde_json::to_writer_pretty(file, &res.hits)?;
Ok(true)
}
@@ -83,4 +90,4 @@ impl Context for Export {
}
})
}
}
}
src/cli/data/import.rs
+16 -8
View File
@@ -17,10 +17,11 @@ use std::fs;
use bytes::Bytes;
use crate::cli::data::cli::Cli;
use crate::cli::data::Context;
use crate::common::meta::ingestion::IngestionRequest;
use crate::service::logs;
use crate::{
cli::data::{cli::Cli, Context},
common::meta::ingestion::IngestionRequest,
service::logs,
};
pub struct Import {}
@@ -38,15 +39,22 @@ fn read_files_in_directory(c: Cli, dir_path: &str) -> Result<bool, anyhow::Error
let path = entry.path();
if path.is_file() {
let content = fs::read(&path)?;
if let Err(e) = logs::ingest::ingest(&c.org, &c.stream_name,
IngestionRequest::JSON(&Bytes::from(content)), 0).await {
if let Err(e) = logs::ingest::ingest(
&c.org,
&c.stream_name,
IngestionRequest::JSON(&Bytes::from(content)),
0,
)
.await
{
eprintln!("insert data fail {:?}: {:?}", path, e);
return Ok(false);
}
} else if path.is_dir() && !read_files_in_directory(c.clone(), &path.to_string_lossy())? {
} else if path.is_dir() && !read_files_in_directory(c.clone(), &path.to_string_lossy())?
{
return Ok(false);
}
}
Ok(true)
})
}
}
src/cli/data/mod.rs
+37 -26
View File
@@ -13,58 +13,69 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::cli::data::cli::Cli;
pub mod cli;
pub mod export;
pub mod import;
pub trait Context {
fn operator(c: Cli) -> Result<bool, anyhow::Error>;
fn operator(c: cli::Cli) -> Result<bool, anyhow::Error>;
}
#[cfg(test)]
mod tests {
use crate::cli::data::cli::Cli;
use crate::cli::data::Context;
use crate::cli::data::export::Export;
use crate::cli::data::import::Import;
use super::*;
#[test]
fn test_export_operator() {
let args = vec![
"openobserve",
"--c", "stream",
"--o", "default",
"--st", "logs",
"--s", "default",
"--t", "json",
"--f", "day",
"--d", "./json",
"--c",
"stream",
"--o",
"default",
"--st",
"logs",
"--s",
"default",
"--t",
"json",
"--f",
"day",
"--d",
"./json",
];
let cli = Cli::args(args);
let cli = cli::Cli::args(args);
if let Err(err) = Export::operator(cli.clone()) {
if let Err(err) = export::Export::operator(cli.clone()) {
println!("Error: {}", err);
}
}
#[test]
fn test_import_operator() {
let args = vec![
"openobserve",
"--c", "stream",
"--o", "default",
"--st", "logs",
"--s", "default",
"--t", "json",
"--f", "day",
"--d", "./json",
"--c",
"stream",
"--o",
"default",
"--st",
"logs",
"--s",
"default",
"--t",
"json",
"--f",
"day",
"--d",
"./json",
];
let cli = Cli::args(args);
let cli = cli::Cli::args(args);
if let Err(err) = Import::operator(cli.clone()) {
if let Err(err) = import::Import::operator(cli.clone()) {
println!("Error: {}", err);
}
}
}
}