perf: performance test (#10)

deploy/build/Dockerfile-simd

@@ -0,0 +1,34 @@
+FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder
+WORKDIR /web
+COPY ./web /web/
+
+RUN npm install
+RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build
+
+FROM public.ecr.aws/docker/library/rust:bullseye as builder
+RUN apt-get update
+RUN apt-get install -y protobuf-compiler
+RUN apt-get install -y cmake curl
+
+RUN rustup toolchain install nightly-2023-01-26
+RUN rustup default nightly-2023-01-26
+RUN rustup target add x86_64-unknown-linux-gnu
+# RUN diff -u <(rustc --print cfg) <(rustc -C target-cpu=native --print cfg)
+RUN rustc --print cfg
+RUN rustc -C target-cpu=native --print cfg
+
+WORKDIR /app
+COPY . /app
+COPY --from=webBuilder /web/dist web/dist
+
+# RUN cargo build --release
+RUN RUSTFLAGS='-C target-cpu=native' cargo build --release --target x86_64-unknown-linux-gnu
+RUN mv /app/target/x86_64-unknown-linux-gnu/release/zinc-observe /app/target/release/zinc-observe
+
+# FROM gcr.io/distroless/cc as runtime
+FROM public.ecr.aws/debian/debian:bullseye-slim as runtime
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
+RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools
+RUN update-ca-certificates
+COPY --from=builder /app/target/release/zinc-observe /
+CMD ["/zinc-observe"]

deploy/build/Dockerfile-simd.aarch64

@@ -0,0 +1,39 @@
+FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder
+WORKDIR /web
+COPY ./web /web/
+
+RUN npm install
+RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build
+
+FROM public.ecr.aws/docker/library/rust:bullseye as builder
+RUN apt-get update
+RUN apt-get install -y protobuf-compiler
+RUN apt-get install -y cmake curl
+
+RUN rustup toolchain install nightly-2023-01-26
+RUN rustup default nightly-2023-01-26
+RUN rustup target add aarch64-unknown-linux-gnu
+# RUN diff -u <(rustc --print cfg) <(rustc -C target-cpu=native --print cfg)
+RUN rustc --print cfg
+RUN rustc -C target-cpu=native --print cfg
+
+WORKDIR /app
+COPY . /app
+COPY --from=webBuilder /web/dist web/dist
+
+# RUN cargo build --release
+ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
+  CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
+  CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
+  RUSTFLAGS='-C target-cpu=native'
+RUN cargo build --release --target aarch64-unknown-linux-gnu
+RUN mv /app/target/aarch64-unknown-linux-gnu/release/zinc-observe /app/target/release/zinc-observe
+
+# FROM gcr.io/distroless/cc:latest-arm64 as runtime
+# FROM gcr.io/distroless/cc:latest-arm64 as runtime
+FROM public.ecr.aws/debian/debian:bullseye-slim as runtime
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
+RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools
+RUN update-ca-certificates
+COPY --from=builder /app/target/release/zinc-observe /
+CMD ["/zinc-observe"]

deploy/build/buildspec-arm.yml

@@ -10,6 +10,29 @@ phases:
       - pwd
       - GIT_TAG="$(git describe --tags --abbrev=0)"
       - GIT_HASH="$(git rev-parse --short HEAD)"
+
+      # simd version
+      - docker build -t zinc-observe:latest-arm64-simd -f deploy/build/Dockerfile-simd.aarch64 .
+      
+      - docker tag zinc-observe:latest-arm64-simd public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64-simd
+      - docker tag zinc-observe:latest-arm64-simd public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-arm64-simd
+      
+      - docker push public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64-simd
+      - docker push public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-arm64-simd
+
+      - echo 'Pull amd64 image'
+      - docker pull public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd
+      - docker pull public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd
+       
+      - echo 'Create manifests'
+      - docker manifest create public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-arm64-simd
+      - docker manifest create public.ecr.aws/zinclabs/zinc-observe-dev:latest-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd --amend public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64-simd
+      
+      - echo 'Push manifests'
+      - docker manifest push public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-simd
+      - docker manifest push public.ecr.aws/zinclabs/zinc-observe-dev:latest-simd
+
+      # common version
       - docker build -t zinc-observe:latest-arm64 -f deploy/build/Dockerfile.aarch64 .
       
       - docker tag zinc-observe:latest-arm64 public.ecr.aws/zinclabs/zinc-observe-dev:latest-arm64

deploy/build/buildspec.yml

@@ -10,6 +10,17 @@ phases:
       - pwd
       - GIT_TAG="$(git describe --tags --abbrev=0)"
       - GIT_HASH="$(git rev-parse --short HEAD)"
+
+      # simd version
+      - docker build -t zinc-observe:latest-amd64-simd -f deploy/build/Dockerfile-simd .
+      
+      - docker tag zinc-observe:latest-amd64-simd public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd
+      - docker tag zinc-observe:latest-amd64-simd public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd
+      
+      - docker push public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64-simd
+      - docker push public.ecr.aws/zinclabs/zinc-observe-dev:latest-amd64-simd
+
+      # common version
       - docker build -t zinc-observe:latest-amd64 -f deploy/build/Dockerfile .
       
       - docker tag zinc-observe:latest-amd64 public.ecr.aws/zinclabs/zinc-observe-dev:$GIT_TAG-$GIT_HASH-amd64

examples/regex.rs

@@ -0,0 +1,7 @@
+fn main() {
+    let text = "(Hello_world), this is a test";
+    let re = regex::Regex::new(r"(?i)\b(\w+)\b").unwrap();
+    for cap in re.captures_iter(text) {
+        println!("Match: {:?}", cap);
+    }
+}

src/job/files.rs

@@ -152,7 +152,7 @@ async fn upload_file(
     log::info!("[JOB] File upload begin: local: {}", path_str);
 
     let mut schema_reader = BufReader::new(&file);
-    let inferred_schema = infer_json_schema(&mut schema_reader, Some(4096)).unwrap();
+    let inferred_schema = infer_json_schema(&mut schema_reader, Some(8192)).unwrap();
     let arrow_schema = Arc::new(inferred_schema.clone());
     drop(schema_reader);
 
@@ -167,7 +167,7 @@ async fn upload_file(
     let mut buf_parquet = Vec::new();
     let props = WriterProperties::builder()
         .set_compression(get_parquet_compression())
-        .set_write_batch_size(4096)
+        .set_write_batch_size(8192)
         .set_data_pagesize_limit(1024 * 512)
         .set_max_row_group_size(1024 * 1024 * 256);
     let writer_props = props.build();

src/service/search/datafusion/exec.rs

@@ -51,7 +51,7 @@ pub async fn sql(
     let runtime_env = create_runtime_env()?;
     let session_config = SessionConfig::new()
         .with_information_schema(schema.is_none())
-        .with_batch_size(4096);
+        .with_batch_size(8192);
     let mut ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env));
 
     // Configure listing options
@@ -240,7 +240,7 @@ pub async fn merge(
     let runtime_env = create_runtime_env()?;
     let session_config = SessionConfig::new()
         .with_information_schema(true)
-        .with_batch_size(4096);
+        .with_batch_size(8192);
     let mut ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env));
 
     // Configure listing options
@@ -496,7 +496,7 @@ pub async fn convert_parquet_file(
     let runtime_env = create_runtime_env()?;
     let session_config = SessionConfig::new()
         .with_information_schema(false)
-        .with_batch_size(4096);
+        .with_batch_size(8192);
     let ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env));
 
     // Configure listing options
@@ -556,7 +556,7 @@ pub async fn convert_parquet_file(
     let batches = df.collect().await?;
     let props = WriterProperties::builder()
         .set_compression(get_parquet_compression())
-        .set_write_batch_size(4096)
+        .set_write_batch_size(8192)
         .set_data_pagesize_limit(1024 * 512)
         .set_max_row_group_size(1024 * 1024 * 256);
     let writer_props = props.build();
@@ -589,7 +589,7 @@ pub async fn merge_parquet_files(
     let runtime_env = create_runtime_env()?;
     let session_config = SessionConfig::new()
         .with_information_schema(false)
-        .with_batch_size(4096);
+        .with_batch_size(8192);
     let ctx = SessionContext::with_config_rt(session_config.clone(), Arc::new(runtime_env));
 
     // Configure listing options
@@ -651,7 +651,7 @@ pub async fn merge_parquet_files(
     let sort_column_id = schema.index_of(&CONFIG.common.time_stamp_col).unwrap();
     let props = WriterProperties::builder()
         .set_compression(get_parquet_compression())
-        .set_write_batch_size(4096)
+        .set_write_batch_size(8192)
         .set_data_pagesize_limit(1024 * 512)
         .set_max_row_group_size(1024 * 1024 * 256)
         .set_sorting_columns(Some(

src/service/search/sql.rs

@@ -344,8 +344,8 @@ impl Sql {
             origin_sql = origin_sql.replace(item.0.as_str(), &fulltext_search);
         }
         // Hack: str_match
-        for key in ["str_match_no_case", "str_match", "match_no_case", "match"] {
-            let re_str_match = Regex::new(&format!(r"(?i){}\(([^\)]*)\)", key)).unwrap();
+        for key in ["match", "match_no_case", "str_match", "str_match_no_case"] {
+            let re_str_match = Regex::new(&format!(r"(?i)\b{}\b\(([^\)]*)\)", key)).unwrap();
             let re_fn = if key == "match" || key == "str_match" {
                 "LIKE"
             } else {