From ffe2bf90c716c4a7b9a92069842c835cd652007b Mon Sep 17 00:00:00 2001 From: Hengfei Yang Date: Sun, 5 Feb 2023 09:27:41 +0800 Subject: [PATCH] perf: performance test (#10) --- deploy/build/Dockerfile-simd | 34 +++++++++++++++++++++++ deploy/build/Dockerfile-simd.aarch64 | 39 +++++++++++++++++++++++++++ deploy/build/buildspec-arm.yml | 23 ++++++++++++++++ deploy/build/buildspec.yml | 11 ++++++++ examples/regex.rs | 7 +++++ src/job/files.rs | 4 +-- src/service/search/datafusion/exec.rs | 12 ++++----- src/service/search/sql.rs | 4 +-- 8 files changed, 124 insertions(+), 10 deletions(-) create mode 100644 deploy/build/Dockerfile-simd create mode 100644 deploy/build/Dockerfile-simd.aarch64 create mode 100644 examples/regex.rs diff --git a/deploy/build/Dockerfile-simd b/deploy/build/Dockerfile-simd new file mode 100644 index 0000000000..26f4653ca6 --- /dev/null +++ b/deploy/build/Dockerfile-simd @@ -0,0 +1,34 @@ +FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder +WORKDIR /web +COPY ./web /web/ + +RUN npm install +RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build + +FROM public.ecr.aws/docker/library/rust:bullseye as builder +RUN apt-get update +RUN apt-get install -y protobuf-compiler +RUN apt-get install -y cmake curl + +RUN rustup toolchain install nightly-2023-01-26 +RUN rustup default nightly-2023-01-26 +RUN rustup target add x86_64-unknown-linux-gnu +# RUN diff -u <(rustc --print cfg) <(rustc -C target-cpu=native --print cfg) +RUN rustc --print cfg +RUN rustc -C target-cpu=native --print cfg + +WORKDIR /app +COPY . /app +COPY --from=webBuilder /web/dist web/dist + +# RUN cargo build --release +RUN RUSTFLAGS='-C target-cpu=native' cargo build --release --target x86_64-unknown-linux-gnu +RUN mv /app/target/x86_64-unknown-linux-gnu/release/zinc-observe /app/target/release/zinc-observe + +# FROM gcr.io/distroless/cc as runtime +FROM public.ecr.aws/debian/debian:bullseye-slim as runtime +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates +RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools +RUN update-ca-certificates +COPY --from=builder /app/target/release/zinc-observe / +CMD ["/zinc-observe"] diff --git a/deploy/build/Dockerfile-simd.aarch64 b/deploy/build/Dockerfile-simd.aarch64 new file mode 100644 index 0000000000..3e06f1166f --- /dev/null +++ b/deploy/build/Dockerfile-simd.aarch64 @@ -0,0 +1,39 @@ +FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder +WORKDIR /web +COPY ./web /web/ + +RUN npm install +RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build + +FROM public.ecr.aws/docker/library/rust:bullseye as builder +RUN apt-get update +RUN apt-get install -y protobuf-compiler +RUN apt-get install -y cmake curl + +RUN rustup toolchain install nightly-2023-01-26 +RUN rustup default nightly-2023-01-26 +RUN rustup target add aarch64-unknown-linux-gnu +# RUN diff -u <(rustc --print cfg) <(rustc -C target-cpu=native --print cfg) +RUN rustc --print cfg +RUN rustc -C target-cpu=native --print cfg + +WORKDIR /app +COPY . /app +COPY --from=webBuilder /web/dist web/dist + +# RUN cargo build --release +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ + CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \ + CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \ + RUSTFLAGS='-C target-cpu=native' +RUN cargo build --release --target aarch64-unknown-linux-gnu +RUN mv /app/target/aarch64-unknown-linux-gnu/release/zinc-observe /app/target/release/zinc-observe + +# FROM gcr.io/distroless/cc:latest-arm64 as runtime +# FROM gcr.io/distroless/cc:latest-arm64 as runtime +FROM public.ecr.aws/debian/debian:bullseye-slim as runtime +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates +RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools +RUN update-ca-certificates +COPY --from=builder /app/target/release/zinc-observe / +CMD ["/zinc-observe"] diff --git a/deploy/build/buildspec-arm.yml b/deploy/build/buildspec-arm.yml index 2cbb877eea..988179b15d 100644 --- a/deploy/build/buildspec-arm.yml +++ b/deploy/build/buildspec-arm.yml @@ -10,6 +10,29 @@ phases: - pwd - GIT_TAG="$(git describe --tags --abbrev=0)" - GIT_HASH="$(git rev-parse --short HEAD)" + + # simd version + - docker build -t zinc-observe:latest-arm64-simd -f deploy/build/Dockerfile-simd.aarch64 . + + - docker tag zinc-observe:latest-arm64-simd public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64-simd + - docker tag zinc-observe:latest-arm64-simd public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-arm64-simd + + - docker push public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64-simd + - docker push public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-arm64-simd + + - echo 'Pull amd64 image' + - docker pull public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd + - docker pull public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd + + - echo 'Create manifests' + - docker manifest create public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-arm64-simd + - docker manifest create public.ecr.aws/zinclabs/zinc-observe-dev:latest-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64-simd + + - echo 'Push manifests' + - docker manifest push public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-simd + - docker manifest push public.ecr.aws/zinclabs/zinc-observe-dev:latest-simd + + # common version - docker build -t zinc-observe:latest-arm64 -f deploy/build/Dockerfile.aarch64 . - docker tag zinc-observe:latest-arm64 public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64 diff --git a/deploy/build/buildspec.yml b/deploy/build/buildspec.yml index 336d4c9c9d..f70b806cda 100644 --- a/deploy/build/buildspec.yml +++ b/deploy/build/buildspec.yml @@ -10,6 +10,17 @@ phases: - pwd - GIT_TAG="$(git describe --tags --abbrev=0)" - GIT_HASH="$(git rev-parse --short HEAD)" + + # simd version + - docker build -t zinc-observe:latest-amd64-simd -f deploy/build/Dockerfile-simd . + + - docker tag zinc-observe:latest-amd64-simd public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd + - docker tag zinc-observe:latest-amd64-simd public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd + + - docker push public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd + - docker push public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd + + # common version - docker build -t zinc-observe:latest-amd64 -f deploy/build/Dockerfile . - docker tag zinc-observe:latest-amd64 public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64 diff --git a/examples/regex.rs b/examples/regex.rs new file mode 100644 index 0000000000..808b7c794e --- /dev/null +++ b/examples/regex.rs @@ -0,0 +1,7 @@ +fn main() { + let text = "(Hello_world), this is a test"; + let re = regex::Regex::new(r"(?i)\b(\w+)\b").unwrap(); + for cap in re.captures_iter(text) { + println!("Match: {:?}", cap); + } +} diff --git a/src/job/files.rs b/src/job/files.rs index af81077e7f..6eb8f64fa4 100644 --- a/src/job/files.rs +++ b/src/job/files.rs @@ -152,7 +152,7 @@ async fn upload_file( log::info!("[JOB] File upload begin: local: {}", path_str); let mut schema_reader = BufReader::new(&file); - let inferred_schema = infer_json_schema(&mut schema_reader, Some(4096)).unwrap(); + let inferred_schema = infer_json_schema(&mut schema_reader, Some(8192)).unwrap(); let arrow_schema = Arc::new(inferred_schema.clone()); drop(schema_reader); @@ -167,7 +167,7 @@ async fn upload_file( let mut buf_parquet = Vec::new(); let props = WriterProperties::builder() .set_compression(get_parquet_compression()) - .set_write_batch_size(4096) + .set_write_batch_size(8192) .set_data_pagesize_limit(1024 * 512) .set_max_row_group_size(1024 * 1024 * 256); let writer_props = props.build(); diff --git a/src/service/search/datafusion/exec.rs b/src/service/search/datafusion/exec.rs index 3ed0680ad9..0d669e0000 100644 --- a/src/service/search/datafusion/exec.rs +++ b/src/service/search/datafusion/exec.rs @@ -51,7 +51,7 @@ pub async fn sql( let runtime_env = create_runtime_env()?; let session_config = SessionConfig::new() .with_information_schema(schema.is_none()) - .with_batch_size(4096); + .with_batch_size(8192); let mut ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env)); // Configure listing options @@ -240,7 +240,7 @@ pub async fn merge( let runtime_env = create_runtime_env()?; let session_config = SessionConfig::new() .with_information_schema(true) - .with_batch_size(4096); + .with_batch_size(8192); let mut ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env)); // Configure listing options @@ -496,7 +496,7 @@ pub async fn convert_parquet_file( let runtime_env = create_runtime_env()?; let session_config = SessionConfig::new() .with_information_schema(false) - .with_batch_size(4096); + .with_batch_size(8192); let ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env)); // Configure listing options @@ -556,7 +556,7 @@ pub async fn convert_parquet_file( let batches = df.collect().await?; let props = WriterProperties::builder() .set_compression(get_parquet_compression()) - .set_write_batch_size(4096) + .set_write_batch_size(8192) .set_data_pagesize_limit(1024 * 512) .set_max_row_group_size(1024 * 1024 * 256); let writer_props = props.build(); @@ -589,7 +589,7 @@ pub async fn merge_parquet_files( let runtime_env = create_runtime_env()?; let session_config = SessionConfig::new() .with_information_schema(false) - .with_batch_size(4096); + .with_batch_size(8192); let ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env)); // Configure listing options @@ -651,7 +651,7 @@ pub async fn merge_parquet_files( let sort_column_id = schema.index_of(&CONFIG.common.time_stamp_col).unwrap(); let props = WriterProperties::builder() .set_compression(get_parquet_compression()) - .set_write_batch_size(4096) + .set_write_batch_size(8192) .set_data_pagesize_limit(1024 * 512) .set_max_row_group_size(1024 * 1024 * 256) .set_sorting_columns(Some( diff --git a/src/service/search/sql.rs b/src/service/search/sql.rs index dc56af39b4..963e9c07df 100644 --- a/src/service/search/sql.rs +++ b/src/service/search/sql.rs @@ -344,8 +344,8 @@ impl Sql { origin_sql = origin_sql.replace(item.0.as_str(), &fulltext_search); } // Hack: str_match - for key in ["str_match_no_case", "str_match", "match_no_case", "match"] { - let re_str_match = Regex::new(&format!(r"(?i){}\(([^\)]*)\)", key)).unwrap(); + for key in ["match", "match_no_case", "str_match", "str_match_no_case"] { + let re_str_match = Regex::new(&format!(r"(?i)\b{}\b\(([^\)]*)\)", key)).unwrap(); let re_fn = if key == "match" || key == "str_match" { "LIKE" } else {