starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-04 00:08:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

zinc-observe initial commit

Browse Source
This commit is contained in:
oasisk
2023-02-02 09:03:11 +05:30
parent 65cb40b2f8
commit 147362cb40
334 changed files with 104360 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+5780
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+107
View File
@@ -0,0 +1,107 @@
[package]
name = "zinc-oxide"
version = "0.1.0"
edition = "2021"
# exclude = ["/web"]
# include = ["/ui"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
mimalloc = []
[profile.release]
debug = false
debug-assertions = false
overflow-checks = false
strip = true
[dependencies]
anyhow = "1.0"
thiserror = "1.0"
log = "0.4.17"
futures = "0.3"
futures-util = "0.3.25"
tokio = { version = "1", features = ["full"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
simd-json = "0.7.0"
flatten-json-object = "0.6.1"
actix-web = "4.2"
actix-web-httpauth = "0.8"
actix-web-prometheus = "0.1.2"
actix-cors = "0.6.4"
prometheus = "0.13.3"
env_logger = "0.9"
dotenv = "0.15.0"
dotenv_config = "0.1.3"
prost = "0.11.2"
tonic = { version = "0.8", features = ["prost", "gzip"] }
sqlparser = { version = "0.30", features = ["serde"] }
arrow = { version = "31.0", features = ["simd", "ipc_compression"] }
arrow-schema = { version = "31.0", features = ["serde"] }
parquet = { version = "31.0", features = ["arrow", "async"] }
datafusion = { version = "17.0", features = ["simd"] }
datafusion-common = "17.0"
object_store = { version = "0.5.3", features = ["aws", "gcp"] }
etcd-client = { version = "0.10.2", features = ["tls"] }
sled = "0.34.7"
aws-config = "0.51.0"
aws-types = "0.51.0"
aws-sdk-s3 = "0.21.0"
zstd = "0.12.1"
chrono = "0.4"
rs-snowflake = "0.6.0"
lazy_static = "1.4.0"
async-trait = "0.1.57"
async_once = "0.2.6"
once_cell = "1.15.0"
glob = "0.3.0"
walkdir = "2"
bytes = "1.1"
memchr = "2.5.0"
rand = "0.8.5"
regex = "1"
regex-syntax = "0.6.27"
sys-info = "0.9.1"
get_if_addrs = "0.5.3"
lru = "0.8.1"
mimalloc = { version = "0.1", default-features = false }
uuid = { version = "1.2.2", features = ["v4", "fast-rng", "macro-diagnostics"] }
mlua = { version = "0.8.4", features = ["luajit52", "vendored" ,"serialize"] }
ahash = { version = "0.8.3", features = ["serde"] }
dashmap = { version = "5.4.0", features = ["serde"] }
argon2 = {version = "0.4.1" ,features = ["alloc", "password-hash"]}
snap = "1"
utoipa = { version = "2", features = ["actix_extras"] }
utoipa-swagger-ui = { version = "2", features = ["actix-web"] }
parking_lot = "0.12"
url = "2.2"
rsfs = "0.4.1"
http-auth-basic = "0.3.3"
opentelemetry = { version = "0.17", features = ["rt-tokio"] }
tracing = "0.1"
tracing-opentelemetry = "0.17"
tracing-subscriber = { version = "0.3", features = ["tracing-log" ,"env-filter"] }
opentelemetry-proto = { version = "0.1", features = ["gen-tonic","traces","with-serde","build-server"] }
opentelemetry-otlp = { version = "0.10", features = ["http-proto", "serialize" , "reqwest-client"] }
actix-web-opentelemetry = { version = "0.12", features = ["metrics"] }
actix-web-rust-embed-responder = { version = "2.1.1", default-features = false, features = ["support-rust-embed-for-web", "base64"] }
rust-embed-for-web="11.1.0"
awc = "3.0.1"
reqwest = "0.11"
[build-dependencies]
tonic-build = { version = "0.8", features = ["prost"] }
prost-build = { version = "0.11.2" }
[dev-dependencies]
boa_engine = { version = "0.15.0", features = ["console"] }
coverage-helper = "0.1"
v8 = "0.53.1"
serde_v8 = "0.66.0"
cluFlock = "1.2.7"
thrift = "0.16"
opentelemetry-prometheus = "0.10"
opentelemetry-proto = { version = "0.1", features = ["build-client"] }
opentelemetry-jaeger = { version = "0.16",features = ["reqwest_collector_client","collector_client", "rt-tokio-current-thread" , "isahc"] }
README.md
+61 -1
View File
@@ -1 +1,61 @@
# zinc-observe
# zinc-oxide
| | |
-----------|-----------------------------------------
| User ID | admin |
| Password | Complexpass#123 |
etcd-client which is used by zin-oxide needs protocol buffer compiler, please refer https://grpc.io/docs/protoc-installation/
For MacOS,using Homebrew
brew install protobuf
protoc --version
## Application structure
User -> API Interface -> Protocol{http|grpc|job} -> Services{logs|metrics|traces|users} -> Infra{db|storage|config} -> meta | common{thirdparty|function utils}
## init
1. register cluster
2. load cache
3. start move job
4. start file list
5. start grpc
6. start http
7. change node status to online
## Mimalloc
We use [Mimalloc](https://github.com/microsoft/mimalloc) replaced jemalloc.
But default we didn't enable it, if you want enable should add cargo flags like this:
```
RUSTFLAGS='-C target-cpu=native' cargo build --release --features "mimalloc"
```
if you want better control to release memory quickly, you can set some evironments
```
export MIMALLOC_VERBOSE=1
export MIMALLOC_PAGE_RESET=1
export MIMALLOC_DECOMMIT_DELAY=25
```
More options refer to: https://github.com/microsoft/mimalloc#environment-options
## Format code
```
cargo clippy -- -A clippy::uninlined_format_args
```
## check cargo build feature
```
diff -u <(rustc --print cfg) <(rustc -C target-cpu=native --print cfg)
```
build.rs
+87
View File
@@ -0,0 +1,87 @@
use std::io::Result;
fn main() -> Result<()> {
tonic_build::configure()
.type_attribute("FileList", "#[derive(Eq)]")
.type_attribute("FileList", "#[derive(serde::Serialize)]")
.type_attribute("FileKey", "#[derive(Eq)]")
.type_attribute("FileKey", "#[derive(serde::Serialize)]")
.type_attribute("FileDescriptor", "#[derive(Eq)]")
.type_attribute("FileDescriptor", "#[derive(serde::Serialize)]")
.type_attribute("FileMeta", "#[derive(Eq)]")
.type_attribute("FileMeta", "#[derive(serde::Serialize)]")
.type_attribute("Job", "#[derive(Eq)]")
.type_attribute("Job", "#[derive(serde::Serialize)]")
.type_attribute("Partition", "#[derive(Eq)]")
.type_attribute("SearchQuery", "#[derive(Eq)]")
.type_attribute("SearchRequest", "#[derive(Eq)]")
.type_attribute("SearchResponse", "#[derive(Eq)]")
.type_attribute("SearchResponse", "#[derive(serde::Serialize)]")
.type_attribute("SearchAggRequest", "#[derive(Eq)]")
.type_attribute("SearchAggRequest", "#[derive(serde::Serialize)]")
.type_attribute("SearchAggResponse", "#[derive(Eq)]")
.type_attribute("SearchAggResponse", "#[derive(serde::Serialize)]")
.compile(
&[
"proto/cluster/common.proto",
"proto/cluster/event.proto",
"proto/cluster/search.proto",
],
&["proto"],
)
.unwrap();
let mut config = prost_build::Config::new();
config
.type_attribute(
"MetricMetadata",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute(
"MetricType",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute("Sample", "#[derive(serde::Deserialize,serde::Serialize)]")
.type_attribute("Exemplar", "#[derive(serde::Deserialize,serde::Serialize)]")
.type_attribute(
"Histogram",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute(
"BucketSpan",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute(
"TimeSeries",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute("Label", "#[derive(serde::Deserialize,serde::Serialize)]")
.type_attribute("Labels", "#[derive(serde::Deserialize,serde::Serialize)]")
.type_attribute(
"LabelMatcher",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute(
"ReadHints",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute("Chunk", "#[derive(serde::Deserialize,serde::Serialize)]")
.type_attribute(
"ChunkedSeries",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute(
"zero_count",
"#[derive(serde::Deserialize,serde::Serialize)]",
)
.type_attribute("count", "#[derive(serde::Deserialize,serde::Serialize)]");
config.compile_protos(
&[
"proto/prometheus/remote.proto",
"proto/prometheus/types.proto",
],
&["proto"],
)?;
Ok(())
}
coverage.sh
Executable
+38
View File
@@ -0,0 +1,38 @@
#!/bin/bash
# cargo llvm-cov >report.json && ls -l | grep TOTAL report.json | xargs > coverage.txt
summary="$(RUSTFLAGS='-C target-cpu=native' cargo llvm-cov >report.json && ls -l | grep TOTAL report.json | xargs)"
echo "Coverage Summary $summary %"
region_cov="$(cut -d' ' -f4 <<<"$summary")"
region_cov=${region_cov//%/}
func_cov="$(cut -d' ' -f7 <<<"$summary")"
func_cov=${func_cov//%/}
line_cov="$(cut -d' ' -f10 <<<"$summary")"
line_cov=${line_cov//%/}
echo "region_cov $region_cov"
echo "func_cov $func_cov"
echo "line_cov $line_cov"
# enable threshold
#COVERAGE_THRESHOLD=80
FUNC_COV_THRESHOLD=40
LINE_COV_THRESHOLD=35
REGION_COV_THRESHOLD=25
# clean up
# find ./target -name llvm-cov-target -type d|xargs rm -fR
# clean up finished
func_diff=$(echo "$func_cov < $FUNC_COV_THRESHOLD" | bc)
line_diff=$(echo "$line_cov < $LINE_COV_THRESHOLD" | bc)
region_diff=$(echo "$region_cov < $REGION_COV_THRESHOLD" | bc)
if [ $func_diff -eq 1 ] || [ $line_diff -eq 1 ] || [ $region_diff -eq 1 ]; then
echo "Coverage is below threshold of function coverage $FUNC_COV_THRESHOLD% or line coverage $LINE_COV_THRESHOLD% or region coverage $REGION_COV_THRESHOLD%"
exit 1
else
echo "Coverage is above threshold of function coverage $FUNC_COV_THRESHOLD% & line coverage $LINE_COV_THRESHOLD% & region coverage $REGION_COV_THRESHOLD%"
exit 0
fi
deploy/build/Dockerfile
+31
View File
@@ -0,0 +1,31 @@
FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder
WORKDIR /web
COPY ./web /web/
RUN npm install
RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build
FROM public.ecr.aws/docker/library/rust:bullseye as builder
RUN apt-get update
RUN apt-get install -y protobuf-compiler
RUN apt-get install -y cmake curl
RUN rustup toolchain install nightly-2023-01-26
RUN rustup default nightly-2023-01-26
RUN rustup target add x86_64-unknown-linux-gnu
WORKDIR /app
COPY . /app
COPY --from=webBuilder /web/dist web/dist
# RUN cargo build --release
RUN RUSTFLAGS='-C target-feature=+aes,+avx,+avx2,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2' cargo build --release --target x86_64-unknown-linux-gnu
RUN mv /app/target/x86_64-unknown-linux-gnu/release/zinc-oxide /app/target/release/ziox
# FROM gcr.io/distroless/cc as runtime
FROM public.ecr.aws/debian/debian:bullseye-slim as runtime
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools
RUN update-ca-certificates
COPY --from=builder /app/target/release/ziox /
CMD ["/ziox"]
deploy/build/Dockerfile-tag.aarch64
+36
View File
@@ -0,0 +1,36 @@
FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder
WORKDIR /web
COPY ./web /web/
RUN npm install
RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build
FROM public.ecr.aws/docker/library/rust:bullseye as builder
RUN apt-get update
RUN apt-get install -y protobuf-compiler
RUN apt-get install -y cmake curl
RUN rustup toolchain install nightly-2023-01-26
RUN rustup default nightly-2023-01-26
RUN rustup target add aarch64-unknown-linux-gnu
WORKDIR /app
COPY . /app
COPY --from=webBuilder /web/dist web/dist
# RUN cargo build --release
ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
RUSTFLAGS='-C target-feature=+aes,+crc,+fhm,+lse,+neon,+jsconv,+sha2'
RUN cargo build --release --target aarch64-unknown-linux-gnu
RUN mv /app/target/aarch64-unknown-linux-gnu/release/zinc-oxide /app/target/release/ziox
FROM gcr.io/distroless/cc:latest-arm64 as runtime
# FROM public.ecr.aws/debian/debian:bullseye-slim as runtime
# RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
# RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools
# RUN update-ca-certificates
COPY --from=builder /app/target/release/ziox /
COPY --from=builder /lib/aarch64-linux-gnu/liblzma.so.5 /lib/aarch64-linux-gnu/liblzma.so.5
CMD ["/ziox"]
deploy/build/Dockerfile-tag.amd64
+32
View File
@@ -0,0 +1,32 @@
FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder
WORKDIR /web
COPY ./web /web/
RUN npm install
RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build
FROM public.ecr.aws/docker/library/rust:bullseye as builder
RUN apt-get update
RUN apt-get install -y protobuf-compiler
RUN apt-get install -y cmake curl
RUN rustup toolchain install nightly-2023-01-26
RUN rustup default nightly-2023-01-26
RUN rustup target add x86_64-unknown-linux-gnu
WORKDIR /app
COPY . /app
COPY --from=webBuilder /web/dist web/dist
# RUN cargo build --release
RUN RUSTFLAGS='-C target-feature=+aes,+avx,+avx2,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2' cargo build --release --target x86_64-unknown-linux-gnu
RUN mv /app/target/x86_64-unknown-linux-gnu/release/zinc-oxide /app/target/release/ziox
FROM gcr.io/distroless/cc as runtime
# FROM public.ecr.aws/debian/debian:bullseye-slim as runtime
# RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
# RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools
# RUN update-ca-certificates
COPY --from=builder /app/target/release/ziox /
COPY --from=builder /lib/x86_64-linux-gnu/liblzma.so.5 /lib/x86_64-linux-gnu/liblzma.so.5
CMD ["/ziox"]
deploy/build/Dockerfile.aarch64
+36
View File
@@ -0,0 +1,36 @@
FROM public.ecr.aws/docker/library/node:18.12.1-bullseye as webBuilder
WORKDIR /web
COPY ./web /web/
RUN npm install
RUN NODE_OPTIONS="--max-old-space-size=6192" npm run build
FROM public.ecr.aws/docker/library/rust:bullseye as builder
RUN apt-get update
RUN apt-get install -y protobuf-compiler
RUN apt-get install -y cmake curl
RUN rustup toolchain install nightly-2023-01-26
RUN rustup default nightly-2023-01-26
RUN rustup target add aarch64-unknown-linux-gnu
WORKDIR /app
COPY . /app
COPY --from=webBuilder /web/dist web/dist
# RUN cargo build --release
ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
RUSTFLAGS='-C target-feature=+aes,+crc,+fhm,+lse,+neon,+jsconv,+sha2'
RUN cargo build --release --target aarch64-unknown-linux-gnu
RUN mv /app/target/aarch64-unknown-linux-gnu/release/zinc-oxide /app/target/release/ziox
# FROM gcr.io/distroless/cc:latest-arm64 as runtime
# FROM gcr.io/distroless/cc:latest-arm64 as runtime
FROM public.ecr.aws/debian/debian:bullseye-slim as runtime
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
RUN apt-get install -y curl htop iftop sysstat procps lsof net-tools
RUN update-ca-certificates
COPY --from=builder /app/target/release/ziox /
CMD ["/ziox"]
deploy/build/buildspec-arm.yml
+59
View File
@@ -0,0 +1,59 @@
version: 0.2
phases:
pre_build:
commands:
- aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 058694856476.dkr.ecr.us-west-2.amazonaws.com
- aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/zinclabs
build:
commands:
- pwd
- GIT_TAG="$(git describe --tags --abbrev=0)"
- GIT_HASH="$(git rev-parse --short HEAD)"
- docker build -t ziox:latest-arm64 -f deploy/build/Dockerfile.aarch64 .
- docker tag ziox:latest-arm64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-arm64
- docker tag ziox:latest-arm64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-arm64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-arm64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-arm64
- echo 'Pull amd64 image'
- docker pull 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-amd64
- docker pull 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-amd64
- echo 'Create manifests'
- docker manifest create 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-amd64 --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-arm64
- docker manifest create 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-amd64 --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-arm64
- echo 'Push manifests'
- docker manifest push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev
- docker manifest push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest
# for ecr-public
- docker tag ziox:latest-arm64 public.ecr.aws/zinclabs/zinc-oxide:latest-arm64
- docker tag ziox:latest-arm64 public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev-arm64
- docker push public.ecr.aws/zinclabs/zinc-oxide:latest-arm64
- docker push public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev-arm64
- echo 'Pull amd64 image'
- docker pull 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-amd64
- docker pull 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-amd64
- docker tag 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-amd64 public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev-amd64
- docker tag 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-amd64 public.ecr.aws/zinclabs/zinc-oxide:latest-amd64
- docker push public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev-amd64
- docker push public.ecr.aws/zinclabs/zinc-oxide:latest-amd64
- echo 'Create manifests'
- docker manifest create public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev --amend public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev-amd64 --amend public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev-arm64
- docker manifest create public.ecr.aws/zinclabs/zinc-oxide:latest --amend public.ecr.aws/zinclabs/zinc-oxide:latest-amd64 --amend public.ecr.aws/zinclabs/zinc-oxide:latest-arm64
- echo 'Push manifests'
- docker manifest push public.ecr.aws/zinclabs/zinc-oxide:$GIT_TAG-$GIT_HASH-dev
- docker manifest push public.ecr.aws/zinclabs/zinc-oxide:latest
# cache:
# paths:
# - ./target
deploy/build/buildspec-tag-amd64.yml
+22
View File
@@ -0,0 +1,22 @@
version: 0.2
phases:
pre_build:
commands:
- aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 058694856476.dkr.ecr.us-west-2.amazonaws.com
build:
commands:
- pwd
- GIT_TAG="$(git describe --tags --abbrev=0)"
- docker build -t ziox:latest-amd64 -f deploy/build/Dockerfile-tag.amd64 .
- docker tag ziox:latest-amd64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-amd64
- docker tag ziox:latest-amd64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-amd64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-amd64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-amd64
# cache:
# paths:
# - ./target
deploy/build/buildspec-tag-arm64.yml
+34
View File
@@ -0,0 +1,34 @@
version: 0.2
phases:
pre_build:
commands:
- aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 058694856476.dkr.ecr.us-west-2.amazonaws.com
build:
commands:
- pwd
- GIT_TAG="$(git describe --tags --abbrev=0)"
- docker build -t ziox:latest-arm64 -f deploy/build/Dockerfile-tag.aarch64 .
- docker tag ziox:latest-arm64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-arm64
- docker tag ziox:latest-arm64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-arm64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-arm64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-arm64
-
- echo 'Pull amd64 image'
- docker pull 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-amd64
- docker pull 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-amd64
- echo 'Create manifests'
- docker manifest create 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-amd64 --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG-arm64
- docker manifest create 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-amd64 --amend 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest-arm64
- echo 'Push manifests'
- docker manifest push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:$GIT_TAG
- docker manifest push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox:latest
# cache:
# paths:
# - ./target
deploy/build/buildspec-test.yml
+21
View File
@@ -0,0 +1,21 @@
version: 0.2
phases:
pre_build:
commands:
- apt-get update
- apt-get install -y bc
- apt-get install -y protobuf-compiler
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- . "$HOME/.cargo/env"
- rustup toolchain install nightly-2023-01-26
- rustup toolchain list
- rustup default nightly-2023-01-26
- export RUSTFLAGS='-C target-cpu=native'
- cargo install cargo-llvm-cov
build:
commands:
- pwd
- cargo clippy -- -A clippy::uninlined_format_args
- ./coverage.sh
deploy/build/buildspec.yml
+24
View File
@@ -0,0 +1,24 @@
version: 0.2
phases:
pre_build:
commands:
- aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 058694856476.dkr.ecr.us-west-2.amazonaws.com
build:
commands:
- pwd
# - git config --global --add safe.directory /codebuild/output/srcDownload/src
- GIT_TAG="$(git describe --tags --abbrev=0)"
- GIT_HASH="$(git rev-parse --short HEAD)"
- docker build -t ziox:latest-amd64 -f deploy/build/Dockerfile .
- docker tag ziox:latest-amd64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-amd64
- docker tag ziox:latest-amd64 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-amd64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:$GIT_TAG-$GIT_HASH-dev-amd64
- docker push 058694856476.dkr.ecr.us-west-2.amazonaws.com/ziox-dev:latest-amd64
# cache:
# paths:
# - ./target
examples/actix-web.rs
+37
View File
@@ -0,0 +1,37 @@
use actix_web::{get, post, web, App, HttpResponse, HttpServer, Responder};
use opentelemetry_proto::{self, tonic::collector::trace::v1::ExportTraceServiceRequest};
use prost::Message;
#[get("/")]
async fn hello() -> impl Responder {
HttpResponse::Ok().body("Hello world!")
}
#[post("/echo")]
async fn echo(req_body: String) -> impl Responder {
HttpResponse::Ok().body(req_body)
}
async fn manual_hello() -> impl Responder {
HttpResponse::Ok().body("Hey there!")
}
#[actix_web::main]
async fn main() -> std::io::Result<()> {
HttpServer::new(|| {
App::new()
.service(hello)
.service(echo)
.route("/hey", web::get().to(manual_hello))
.service(web::resource("/v1/traces").to(trace))
})
.bind(("127.0.0.1", 4317))?
.run()
.await
}
async fn trace(body: actix_web::web::Bytes) -> String {
let request = ExportTraceServiceRequest::decode(body).expect("Invalid protobuf");
print!("{:?}", request);
format!("Done")
}
examples/datafusion.rs
+50
View File
@@ -0,0 +1,50 @@
use std::sync::Arc;
use datafusion::arrow::array::Int32Array;
use datafusion::arrow::array::StringArray;
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::datasource::MemTable;
use datafusion::error::Result;
use datafusion::from_slice::FromSlice;
use datafusion::prelude::{cast, col, SessionContext};
/// This example demonstrates how to use the DataFrame API against in-memory data.
#[tokio::main]
async fn main() -> Result<()> {
// define a schema.
let schema = Arc::new(Schema::new(vec![Field::new("f", DataType::Int32, false)]));
// define data.
let batch = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(Int32Array::from_slice([1, 10, 10, 100]))],
)?;
let schema2 = Arc::new(Schema::new(vec![Field::new("f", DataType::Utf8, false)]));
let batch2 = RecordBatch::try_new(
schema2.clone(),
vec![Arc::new(StringArray::from_slice(["2", "20", "20", "200"]))],
)?;
// declare a new context. In spark API, this corresponds to a new spark SQLsession
let ctx = SessionContext::new();
// declare a table in memory. In spark API, this corresponds to createDataFrame(...).
let provider = MemTable::try_new(schema.clone(), vec![vec![batch]])?;
ctx.register_table("t", Arc::new(provider))?;
let df = ctx.table("t").await?;
let df1 = df.select(vec![cast(col("f"), DataType::Utf8)])?;
let provider2 = MemTable::try_new(schema2.clone(), vec![vec![batch2]])?;
ctx.register_table("t2", Arc::new(provider2))?;
let df2 = ctx.table("t2").await?;
let new_df = df1.union(df2.clone())?;
ctx.register_table("final", new_df.into_view())?;
let result = ctx.sql("select * from final").await?;
println!("{:?}", result.schema());
result.show().await.unwrap();
Ok(())
}
examples/env_config.rs
+35
View File
@@ -0,0 +1,35 @@
use dotenv_config::EnvConfig;
#[derive(Debug, EnvConfig)]
struct Config {
#[env_config(default = "192.168.2.1")]
server_addr: String,
server_mode: bool,
#[env_config(name = "ZINC_FOO", default = true)]
foo: bool,
#[env_config(name = "ZINC_BAR", default = 123456)]
bar: Option<i64>,
rr: Redis,
}
#[derive(Debug, EnvConfig)]
struct Redis {
addr: String,
port: String,
auth: String,
#[env_config(name = "ZINC_REDIS_TIMEOUT", default = 30)]
timeout: i32,
}
fn main() {
let cfg = Config::init().unwrap();
println!("{:#?}", cfg);
println!("{}", cfg.server_addr);
println!("{}", cfg.server_mode);
println!("{}", cfg.foo);
println!("{:?}", cfg.bar);
println!("{:?}", cfg.rr.addr);
println!("{:?}", cfg.rr.port);
println!("{:?}", cfg.rr.auth);
println!("{:?}", cfg.rr.timeout);
}
examples/etcd.rs
+93
View File
@@ -0,0 +1,93 @@
use async_once::AsyncOnce;
use etcd_client::{Client, Error};
use lazy_static::lazy_static;
lazy_static! {
static ref CLIENT: AsyncOnce<Client> =
AsyncOnce::new(async { Client::connect(["localhost:2379"], None).await.unwrap() });
}
#[tokio::main]
async fn main() -> Result<(), Error> {
// let mut client = Client::connect(["localhost:2379"], None).await?;
let mut tasks = Vec::new();
for i in 0..10 {
let mut client = CLIENT.get().await.clone();
let i = i;
let task = tokio::task::spawn(async move {
let _ret = test(&mut client, i).await;
});
tasks.push(task);
}
for task in tasks {
task.await.unwrap();
}
// do 1000k write test
// let mut client = CLIENT.get().await.clone();
// write_1000k(&mut client).await?;
// read_1000k(&mut client).await?;
Ok(())
}
// async fn write_1000k(client: &mut Client) -> Result<(), Error> {
// for i in 0..1000000 {
// let key = format!(
// "/btest/default/olympics/2022/10/03/10/6982652937134804993_{}.parquet",
// i
// );
// let value = format!(
// r##"{{"min_ts":1664794181570777,"max_ts":1664794188309667,"records":60000,"original_size":13605746,"compressed_size":520701}}"##
// );
// client.put(key.as_bytes(), value.as_bytes(), None).await?;
// }
// Ok(())
// }
// #[derive(Clone, Debug, serde::Deserialize)]
// struct fileInfo {
// min_ts: u64,
// max_ts: u64,
// records: u64,
// original_size: u64,
// compressed_size: u64,
// }
// async fn read_1000k(client: &mut Client) -> Result<(), Error> {
// let key = "/btest/".to_string();
// let opt = GetOptions::new().with_prefix();
// let ret = client.get(key.as_bytes(), Some(opt)).await?;
// let mut files = HashMap::with_capacity(1000000);
// for kv in ret.kvs() {
// let key = String::from_utf8(kv.key().to_vec()).unwrap();
// // let value = String::from_utf8(kv.value().to_vec()).unwrap();
// let value:fileInfo = serde_json::from_str(kv.value_str().unwrap()).unwrap();
// // println!("key: {}, value: {}", key, value);
// for i in 0..10 {
// let key = format!("{}/{}", key, i);
// files.insert(key, value.clone());
// }
// }
// println!("files len: {}", files.len());
// tokio::time::sleep(tokio::time::Duration::from_secs(10)).await;
// Ok(())
// }
async fn test(client: &mut Client, i: usize) -> Result<(), Error> {
// let mut client = Client::connect(["localhost:2379"], None).await?;
// put kv
let key = format!("key_{}", i);
let val = format!("value_{}", i);
client.put(key.clone(), val, None).await?;
// get kv
let resp = client.get(key.clone(), None).await?;
if let Some(kv) = resp.kvs().first() {
println!("Get kv: {{{}: {}}}", kv.key_str()?, kv.value_str()?);
}
Ok(())
}
examples/event_client.rs
+48
View File
@@ -0,0 +1,48 @@
use cluster::event_client::EventClient;
use http_auth_basic::Credentials;
use tonic::{codec::CompressionEncoding, metadata::MetadataValue, transport::Channel, Request};
use zinc_oxide::infra::config::CONFIG;
pub mod cluster {
tonic::include_proto!("cluster");
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let credentials = Credentials::new(&CONFIG.auth.username, &CONFIG.auth.password);
let credentials = credentials.as_http_header();
let token: MetadataValue<_> = credentials.parse()?;
let org_id: MetadataValue<_> = "default".parse()?;
let channel = Channel::from_static("http://127.0.0.1:5081")
.connect()
.await?;
let mut client = EventClient::with_interceptor(channel, move |mut req: Request<()>| {
req.metadata_mut().insert("authorization", token.clone());
req.metadata_mut()
.insert(CONFIG.grpc.org_header_key.as_str(), org_id.clone());
Ok(req)
});
client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip);
let mut req_query = cluster::FileList::default();
req_query.items.push(cluster::FileKey {
key: "default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet".to_string(),
meta: Some(cluster::FileMeta::default()),
deleted: false,
});
req_query.items.push(cluster::FileKey {
key: "default/logs/olympics/2022/10/03/10/6982652937134804993_2.parquet".to_string(),
meta: Some(cluster::FileMeta::default()),
deleted: true,
});
let request = tonic::Request::new(req_query);
let response = client.send_file_list(request).await?;
println!("RESPONSE={:?}", response);
Ok(())
}
examples/generate_openapi.rs
+22
View File
@@ -0,0 +1,22 @@
use utoipa::OpenApi;
use zinc_oxide::handler::http::request::stream::*;
use zinc_oxide::meta::stream;
use zinc_oxide::meta::StreamType;
#[tokio::main]
async fn main() {
#[derive(OpenApi)]
#[openapi(
paths(list_streams),
components(schemas(
stream::ListStream,
stream::Stream,
StreamType,
stream::Stats,
stream::StreamProperty,
))
)]
struct ApiDoc;
println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());
}
examples/json.rs
+20
View File
@@ -0,0 +1,20 @@
use std::{
env,
fs::File,
io::{BufRead, BufReader},
process,
};
fn main() {
let file = env::args().nth(1).unwrap_or_else(|| {
eprintln!("Usage: {} <file>", env::args().next().unwrap());
process::exit(1)
});
let f = File::open(file).unwrap();
let reader = BufReader::new(f);
for line in reader.lines() {
let line = line.unwrap();
let data: serde_json::Value = serde_json::from_str(&line).unwrap();
println!("{:?}", data.is_null());
}
}
examples/lock_file.rs
+102
View File
@@ -0,0 +1,102 @@
use std::{
fs::OpenOptions,
io::{BufWriter, ErrorKind, Write},
thread,
time::Duration,
};
use cluFlock::{ExclusiveFlock, FlockLock};
fn main() {
println!("Checking lock");
let handle = thread::spawn(|| {
for i in 1..3 {
let file = OpenOptions::new()
.write(true)
.create(true)
.append(true)
.open("./test_file.json")
.unwrap();
let file_lock = ExclusiveFlock::try_lock(&file);
match file_lock {
Ok(_) => println!("{:?} locked using thread {}", file_lock, i),
Err(_) => println!(" ****** Create New File "),
}
println!("################# Checking if lock exists");
let is_locked = is_locked("./test_file.json").unwrap();
println!("################# lock exists {}", is_locked);
let mut writer = BufWriter::new(&file);
writer
.write(
format!("Writing using thread {} ", i)
.to_string()
.as_bytes(),
)
.unwrap();
writer.write(b"\n").unwrap();
thread::sleep(Duration::from_millis(5));
writer
.write(
format!("Somthing using thread after sleep {} ", i)
.to_string()
.as_bytes(),
)
.unwrap();
writer.write(b"\n").unwrap();
//file_lock.unlock().unwrap();
println!("unlocked by thread {}", i);
}
});
for i in 1..3 {
let file = OpenOptions::new()
.write(true)
.create(true)
.append(true)
.open("./test_file.json")
.unwrap();
let file_lock = ExclusiveFlock::wait_lock(&file).unwrap();
println!("{:?} locked using Main Attempt {}", file_lock, i);
let mut writer = BufWriter::new(&file);
writer
.write(format!("Writing Main Attempt{}", i).to_string().as_bytes())
.unwrap();
writer.write(b"\n").unwrap();
writer
.write(
format!("Somthing Main Attempt After sleep{} ", i)
.to_string()
.as_bytes(),
)
.unwrap();
writer.write(b"\n").unwrap();
thread::sleep(Duration::from_millis(10));
file_lock.unlock().unwrap();
println!("unlocked using Main Attempt {}", i);
}
handle.join().unwrap();
println!("Checking if lock exists before existing");
let is_locked = is_locked("./test_file.json").unwrap();
println!("lock exists {}", is_locked);
}
pub fn is_locked(path: &str) -> Result<bool, std::io::Error> {
let file = OpenOptions::new().read(true).write(false).open(path)?;
FlockLock::try_exclusive_lock_fn(
&file,
|_| Ok(false),
|e| match e.kind() {
ErrorKind::WouldBlock => Ok(true), // ignore err
_ => Err(e.into_err()), // into_err: FlockErr -> std::io::Error
},
)
}
examples/memchr.rs
+24
View File
@@ -0,0 +1,24 @@
use memchr::memmem;
use std::env;
use std::time::Instant;
fn main() {
let haystack = r##"{"Athlete":"PAHUD DE MORTANGES, Charles Ferdinand","City":"Los Angeles","Country":"NED","Discipline":"Eventing","Event":"Team","Gender":"Men","Medal":"Silver","Season":"summer","Sport":"Equestrian","Year":1932,"_timestamp":"2022-08-31T03:32:44.555229+00:00"}"##;
let haystackb = haystack.as_bytes();
let mode = env::args().nth(1).unwrap_or("std".to_string());
println!("arch: {:?}, mode: {}", env::consts::ARCH, mode);
let start_time = Instant::now();
if mode == "memchr" {
for _i in 1..1000000 {
memmem::find(haystackb, b"NED");
}
} else {
for _i in 1..1000000 {
haystack.contains("NED");
}
}
println!("Time elapsed is: {:?}", start_time.elapsed());
}
examples/prometheus.rs
+97
View File
@@ -0,0 +1,97 @@
use bytes::{Bytes, BytesMut};
use chrono::Utc;
use prost::Message;
use tokio::time::sleep;
use tokio::time::Duration;
pub mod prometheus_prot {
include!(concat!(env!("OUT_DIR"), "/prometheus.rs"));
}
#[tokio::main]
async fn main() {
let replicas = ["prom-k8s-0", "prom-k8s-2", "prom-k8s-0"];
let labels = [
"grafana_api_dashboard_save_milliseconds_count",
"kube_cronjob_info",
"alertmanager_config_hash",
];
let handle = tokio::spawn(async move {
for i in 1..10 {
sleep(Duration::from_secs(1)).await;
let client = reqwest::Client::new();
let mut loc_lable: Vec<prometheus_prot::Label> = vec![];
loc_lable.push(prometheus_prot::Label {
name: "__name__".to_string(),
value: labels[i % 3].to_string(),
});
loc_lable.push(prometheus_prot::Label {
name: "cluster".to_string(),
value: "prom-k8s".to_string(),
});
loc_lable.push(prometheus_prot::Label {
name: "__replica__".to_string(),
value: replicas[i % 3].to_string(),
});
let mut loc_samples: Vec<prometheus_prot::Sample> = vec![];
for i in 1..100 {
loc_samples.push(prometheus_prot::Sample {
value: i as f64,
timestamp: Utc::now().timestamp_micros(),
});
}
loc_samples.push(prometheus_prot::Sample {
value: f64::NEG_INFINITY,
timestamp: Utc::now().timestamp_micros(),
});
loc_samples.push(prometheus_prot::Sample {
value: f64::INFINITY,
timestamp: Utc::now().timestamp_micros(),
});
loc_samples.push(prometheus_prot::Sample {
value: 0.0_f64 / 0.0_f64,
timestamp: Utc::now().timestamp_micros(),
});
println!("Number of samples {}", loc_samples.len());
let loc_exemp: Vec<prometheus_prot::Exemplar> = vec![];
let loc_hist: Vec<prometheus_prot::Histogram> = vec![];
let ts = prometheus_prot::TimeSeries {
labels: loc_lable,
samples: loc_samples,
exemplars: loc_exemp,
histograms: loc_hist,
};
let metadata: Vec<prometheus_prot::MetricMetadata> = vec![];
let wr_req: prometheus_prot::WriteRequest = prometheus_prot::WriteRequest {
timeseries: vec![ts],
metadata,
};
let mut out = BytesMut::with_capacity(wr_req.encoded_len());
wr_req.encode(&mut out).expect("Out of memory");
let body = snap_block(out.into());
let req = client
.post("http://localhost:5080/api/nexus/prometheus/write")
.header("Authorization", "Basic YWRtaW46Q29tcGxleHBhc3MjMTIz")
.header("X-Prometheus-Remote-Write-Version", "0.1.0")
.header("Content-Encoding", "snappy")
.header("Content-Type", "application/x-protobuf")
.body(body);
let res = req.send().await.unwrap();
println!("{}", res.status());
}
});
let _out = handle.await.unwrap();
}
fn snap_block(data: Bytes) -> Vec<u8> {
snap::raw::Encoder::new()
.compress_vec(&data)
.expect("Out of memory")
}
examples/search_client.rs
+68
View File
@@ -0,0 +1,68 @@
use cluster::search_client::SearchClient;
use cluster::SearchRequest;
use http_auth_basic::Credentials;
use tonic::{codec::CompressionEncoding, metadata::MetadataValue, transport::Channel, Request};
use zinc_oxide::infra::config::CONFIG;
pub mod cluster {
tonic::include_proto!("cluster");
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let credentials = Credentials::new(&CONFIG.auth.username, &CONFIG.auth.password);
let credentials = credentials.as_http_header();
let token: MetadataValue<_> = credentials.parse()?;
let org_id: MetadataValue<_> = "default".parse()?;
let channel = Channel::from_static("http://127.0.0.1:5081")
.connect()
.await?;
let mut client = SearchClient::with_interceptor(channel, move |mut req: Request<()>| {
req.metadata_mut().insert("authorization", token.clone());
req.metadata_mut()
.insert(CONFIG.grpc.org_header_key.as_str(), org_id.clone());
Ok(req)
});
client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip);
let req_query = cluster::SearchQuery {
sql: "select * from k8s".to_string(),
sql_mode: "".to_string(),
from: 0,
size: 10,
start_time: 0,
end_time: 0,
track_total_hits: true,
};
// let mut aggs = AHashMap::new();
// aggs.insert("histogram".to_string(),"select histogram(_timestamp, '1 minute') AS key, count(*) AS num from query GROUP BY key ORDER BY key".to_string());
// let mut file_list = Vec::with_capacity(100000);
// for _ in 0..2 {
// file_list
// .push("default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet".to_string());
// file_list
// .push("default/logs/olympics/2022/10/03/10/6982655327841947649_1.parquet".to_string());
// file_list
// .push("default/logs/olympics/2022/10/03/11/6982665068089577473_1.parquet".to_string());
// }
let request = tonic::Request::new(SearchRequest {
job: Some(cluster::Job::default()),
org_id: "default".to_string(),
stype: cluster::SearchType::User.into(),
query: Some(req_query),
aggs: Vec::new(),
partition: None,
file_list: Vec::new(),
stream_type: "logs".to_string(),
});
let response = client.search(request).await?;
println!("{}", serde_json::to_string(&response.into_inner()).unwrap());
Ok(())
}
examples/simd-json.rs
+26
View File
@@ -0,0 +1,26 @@
use std::env;
fn main() -> Result<(), anyhow::Error> {
let jsonstr = br#"{"kubernetes.annotations.checksum/config":"7e233732c5d6164e2408296cc5d77749803905c56148fdfc4c92e497abd25093", "kubernetes.annotations.checksum/luascripts":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "kubernetes.annotations.kubernetes.io/psp":"eks.privileged", "kubernetes.container_image":"cr.fluentbit.io/fluent/fluent-bit:1.9.7", "kubernetes.container_name":"fluent-bit", "kubernetes.docker_id":"d0fe1d22906c6918c71322c700f6a9b00f2c47eb14c460a05c709694c7de1644", "kubernetes.host":"ip-10-1-81-177.us-west-2.compute.internal", "kubernetes.labels.app.kubernetes.io/instance":"fluent-bit", "kubernetes.labels.app.kubernetes.io/name":"fluent-bit", "kubernetes.labels.controller-revision-hash":"6db46d9688", "kubernetes.labels.pod-template-generation":"1", "kubernetes.namespace_name":"logging", "kubernetes.pod_id":"f3ce6981-3add-472b-bce5-a4e561d3019d", "kubernetes.pod_name":"fluent-bit-5x4b7", "requestId":"a01ef7fc-0001-bc3a-51d1-0183cfa1c1ec", "stream":"stderr", "time":"2022-10-13T04:37:08.253116988Z"}"#;
let mut jsonstr = jsonstr.to_vec();
let mode = env::args().nth(1).unwrap_or("std".to_string());
println!("arch: {:?}, mode: {}", env::consts::ARCH, mode);
let start_time = std::time::Instant::now();
let n = 1000000;
if mode == "simd" {
for _ in 0..n {
let _val: serde_json::Value = simd_json::serde::from_slice(&mut jsonstr).unwrap();
// println!("{}", val.get("kubernetes.pod_id").unwrap().as_str().unwrap());
}
} else {
for _ in 0..n {
let _val: serde_json::Value = serde_json::from_slice(&jsonstr).unwrap();
// println!("{}", val.get("kubernetes.pod_id").unwrap().as_str().unwrap());
}
}
println!("Time elapsed is: {:?}", start_time.elapsed());
Ok(())
}
examples/simple_udf_sql.rs
+152
View File
@@ -0,0 +1,152 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use datafusion::{
arrow::{
array::{ArrayRef, Float32Array, Float64Array},
datatypes::DataType,
record_batch::RecordBatch,
},
logical_expr::Volatility,
};
use datafusion::from_slice::FromSlice;
use datafusion::prelude::*;
use datafusion::{error::Result, physical_plan::functions::make_scalar_function};
use std::sync::Arc;
// create local execution context with an in-memory table
fn create_context() -> Result<SessionContext> {
use datafusion::arrow::datatypes::{Field, Schema};
use datafusion::datasource::MemTable;
// define a schema.
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Float32, false),
Field::new("b", DataType::Float64, false),
]));
// define data.
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Float32Array::from_slice([2.1, 3.1, 4.1, 5.1])),
Arc::new(Float64Array::from_slice([1.0, 2.0, 3.0, 4.0])),
],
)?;
// declare a new context. In spark API, this corresponds to a new spark SQLsession
let ctx = SessionContext::new();
// declare a table in memory. In spark API, this corresponds to createDataFrame(...).
let provider = MemTable::try_new(schema, vec![vec![batch]])?;
ctx.register_table("t", Arc::new(provider))?;
Ok(ctx)
}
/// In this example we will declare a single-type, single return type UDF that exponentiates f64, a^b
#[tokio::main]
async fn main() -> Result<()> {
let ctx = create_context()?;
// First, declare the actual implementation of the calculation
let pow = |args: &[ArrayRef]| {
// in DataFusion, all `args` and output are dynamically-typed arrays, which means that we need to:
// 1. cast the values to the type we want
// 2. perform the computation for every element in the array (using a loop or SIMD) and construct the result
// this is guaranteed by DataFusion based on the function's signature.
assert_eq!(args.len(), 2);
// 1. cast both arguments to f64. These casts MUST be aligned with the signature or this function panics!
let base = &args[0]
.as_any()
.downcast_ref::<Float64Array>()
.expect("cast failed");
let exponent = &args[1]
.as_any()
.downcast_ref::<Float64Array>()
.expect("cast failed");
// this is guaranteed by DataFusion. We place it just to make it obvious.
assert_eq!(exponent.len(), base.len());
// 2. perform the computation
let array = base
.iter()
.zip(exponent.iter())
.map(|(base, exponent)| {
match (base, exponent) {
// in arrow, any value can be null.
// Here we decide to make our UDF to return null when either base or exponent is null.
(Some(base), Some(exponent)) => Some(base.powf(exponent)),
_ => None,
}
})
.collect::<Float64Array>();
// `Ok` because no error occurred during the calculation (we should add one if exponent was [0, 1[ and the base < 0 because that panics!)
// `Arc` because arrays are immutable, thread-safe, trait objects.
Ok(Arc::new(array) as ArrayRef)
};
// the function above expects an `ArrayRef`, but DataFusion may pass a scalar to a UDF.
// thus, we use `make_scalar_function` to decorare the closure so that it can handle both Arrays and Scalar values.
let pow = make_scalar_function(pow);
// Next:
// * give it a name so that it shows nicely when the plan is printed
// * declare what input it expects
// * declare its return type
let pow = create_udf(
"pow",
// expects two f64
vec![DataType::Float64, DataType::Float64],
// returns f64
Arc::new(DataType::Float64),
Volatility::Immutable,
pow,
);
// at this point, we can use it or register it, depending on the use-case:
// * if the UDF is expected to be used throughout the program in different contexts,
// we can register it, and call it later:
ctx.register_udf(pow.clone()); // clone is only required in this example because we show both usages
// * if the UDF is expected to be used directly in the scope, `.call` it directly:
// let expr = pow.call(vec![col("a"), col("b")]);
// get a DataFrame from the context
let df = ctx.table("t").await?;
let df_with_all = df.with_column("_all", col("a") + col("b"))?;
println!("Printing schema");
println!("{}", df_with_all.schema());
ctx.register_table("t_new", df_with_all.into_view())?;
// println!("{:?}", ctx.tables().unwrap());
let result = ctx
.sql(
"SELECT a, b , _all FROM t_new ;
",
)
.await?;
println!("Printing results");
// print the results
result.show().await?;
Ok(())
}
examples/sqlparser.rs
+36
View File
@@ -0,0 +1,36 @@
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
fn main() {
let sql = "SELECT a, b, 123, myfunc(b) \
FROM table_1 \
WHERE a > b AND b < 100 AND (c = 1 or b = 2) \
ORDER BY a DESC, b ASC limit 10,0 ";
let dialect = GenericDialect {}; // or AnsiDialect, or your own dialect ...
let parse_result = Parser::parse_sql(&dialect, sql);
let ast = match parse_result {
Ok(stmt) => stmt,
Err(e) => {
println!("err {:?}", e);
vec![]
}
};
let query = ast
.iter()
.filter_map(|stmt| match stmt {
sqlparser::ast::Statement::Query(q) => Some(q),
_ => None,
})
.next()
.unwrap();
// println!("AST: {:?}", query);
println!("SQL->body: {:?}", query.body);
println!("SQL->order_by: {:?}", query.order_by);
println!("SQL->limit: {:?}", query.limit);
println!("SQL->offset: {:?}", query.offset);
}
examples/threads.rs
+29
View File
@@ -0,0 +1,29 @@
use std::sync::mpsc::channel;
use std::thread::{self, sleep};
use std::time;
fn main() {
let (tx, rx) = channel();
let receiver = thread::spawn(move || {
for received in rx {
println!("Got: {}", received);
}
// let value = rx.recv().expect("Unable to receive from channel");
// println!("{value}");
});
let sender = thread::spawn(move || {
for i in 0..100 {
tx.send(i).expect("Unable to send on channel");
println!("send {}", i);
let ten_millis = time::Duration::from_millis(10);
let _now = time::Instant::now();
sleep(ten_millis);
}
});
receiver.join().expect("The receiver thread has panicked");
sender.join().expect("The sender thread has panicked");
}
examples/time.rs
+80
View File
@@ -0,0 +1,80 @@
use chrono::{DateTime, Datelike, Duration, FixedOffset, NaiveDateTime, TimeZone, Timelike, Utc};
use std::ops::Sub;
fn main() {
let s = "2022-09-27T00:00:00Z";
let time = DateTime::parse_from_rfc3339(s).unwrap();
println!("{:?}, {:?}", time, time.timestamp_micros());
let s = "2022-08-29 18:23:23";
let fmt = "%Y-%m-%d %H:%M:%S";
let time = NaiveDateTime::parse_from_str(s, fmt).unwrap();
println!("{:?}", time);
let s = "2022-09-27T18:23:23+08:00";
let fmt = "%Y-%m-%dT%H:%M:%S%z";
let time = NaiveDateTime::parse_from_str(s, fmt).unwrap();
println!("{:?}", time);
/* let dt = Utc.with_ymd_and_hms(2022, 9, 12, 2, 0, 0).unwrap(); // `2014-07-08T09:10:11Z`
println!("{:?}, {:?}", dt, dt.timestamp_micros());
*/
let dt = Utc.timestamp_nanos(1661752800000000000);
println!("{:?}", dt);
let dt = Utc.timestamp_nanos(1661731871000000000);
println!("{:?}", dt);
let dt = Utc.timestamp_nanos(1662102671000000000);
println!("{:?}", dt);
let dt = parse_time("2022-08-29 18:23:23").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-08-29T18:23:23Z").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-08-29T18:23:23+08:00").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-10-13T04:38:52.260290946+08:00").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-10-13T04:38:52.260290946Z").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-10-13T04:38:52.260Z").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-10-13T04:38:52Z").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-10-13T04:38:52").unwrap();
println!("{:?}", dt.to_string());
let dt = parse_time("2022-10-13 04:38:52").unwrap();
println!("{:?}", dt.to_string());
let dt = chrono::Utc::now().timestamp_micros();
println!("{:?}", dt);
let now = Utc::now().sub(Duration::hours(1));
let offset: DateTime<Utc> = Utc
.with_ymd_and_hms(now.year(), now.month(), now.day(), now.hour(), 0, 0)
.unwrap();
println!("{:?}", now);
println!("{:?}", offset);
}
fn parse_time(s: &str) -> Result<DateTime<FixedOffset>, anyhow::Error> {
if s.contains('T') {
if s.len() == 19 {
let fmt = "%Y-%m-%dT%H:%M:%S";
let ret = Utc.datetime_from_str(s, fmt)?;
Ok(ret.into())
} else {
Ok(chrono::DateTime::parse_from_rfc3339(s)?)
}
} else if s.contains(',') {
Ok(chrono::DateTime::parse_from_rfc2822(s)?)
} else {
let fmt = "%Y-%m-%d %H:%M:%S";
let ret = Utc.datetime_from_str(s, fmt)?;
Ok(ret.into())
}
}
examples/tracing.rs
+73
View File
@@ -0,0 +1,73 @@
use opentelemetry::sdk::propagation::TraceContextPropagator;
use opentelemetry::trace::TraceError;
use opentelemetry::{global, sdk::trace as sdktrace};
use opentelemetry::{
trace::{TraceContextExt, Tracer},
Key,
};
use opentelemetry_otlp::WithExportConfig;
use std::collections::HashMap;
use std::error::Error;
use std::time::Duration;
/* fn _init_tracer() -> Result<sdktrace::Tracer, TraceError> {
opentelemetry_jaeger::new_pipeline()
.with_collector_endpoint("http://127.0.0.1:4317/v1/traces")
.with_service_name("trace-http-demo")
.install_batch(opentelemetry::runtime::TokioCurrentThread)
} */
fn init_tracer_otlp() -> Result<sdktrace::Tracer, TraceError> {
let mut headers = HashMap::new();
headers.insert(
"Authorization".to_string(),
"Basic YWRtaW46Q29tcGxleHBhc3MjMTIz".to_string(),
);
// Start a new jaeger trace pipeline
global::set_text_map_propagator(TraceContextPropagator::new());
let exporter = opentelemetry_otlp::new_exporter()
.http()
.with_headers(headers)
.with_endpoint("http://localhost:5080/api/org1/traces");
opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(exporter)
.install_batch(opentelemetry::runtime::Tokio)
}
const LEMONS_KEY: Key = Key::from_static_str("ex.com/lemons");
const ANOTHER_KEY: Key = Key::from_static_str("ex.com/another");
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error + Send + Sync + 'static>> {
let _ = init_tracer_otlp()?;
let tracer = global::tracer("ex.com/basic");
tracer.in_span("operation", |cx| {
let span = cx.span();
println!("{:?}", span.span_context());
span.add_event(
"Nice operation!".to_string(),
vec![Key::new("bogons").i64(100)],
);
span.add_event(
"Nice operation!".to_string(),
vec![Key::new("bogons").i64(100)],
);
span.set_attribute(ANOTHER_KEY.string("yes"));
tracer.in_span("Sub operation...", |cx| {
let span = cx.span();
span.set_attribute(LEMONS_KEY.string("five"));
span.add_event("Sub span event", vec![]);
});
});
// wait for 1 minutes so that we could see metrics being pushed via OTLP every 10 seconds.
tokio::time::sleep(Duration::from_secs(60)).await;
global::shutdown_tracer_provider();
Ok(())
}
examples/transform.rs
+126
View File
@@ -0,0 +1,126 @@
use boa_engine::Context;
use mlua::Lua;
#[tokio::main]
async fn main() {
let num_recs = 100000;
let log = r#"[2022-10-16T09:36:43Z INFO actix_web::middleware::logger] 10.1.10.35 "POST /api/zinc_next/_bulk HTTP/1.1" 200 80 "-" "Fluent-Bit" 0.002320"#.to_string();
use std::time::Instant;
let now = Instant::now();
lua_test(num_recs, log.clone()).await;
let elapsed = now.elapsed();
println!(" Lua Elapsed: {:.2?}", elapsed);
let now2 = Instant::now();
v8_test(num_recs, log.clone());
let elapsed2 = now2.elapsed();
println!("V8 Elapsed: {:.2?}", elapsed2);
/* let now1 = Instant::now();
boa_test(num_recs, log.clone());
let elapsed1 = now1.elapsed();
println!("Boa Elapsed: {:.2?}", elapsed1); */
}
pub fn boa_test(num_recs: i32, log: String) {
let mut context = Context::default();
let script = r#"
function pow(data) {
let resp = {}
resp["ipaddress"] = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/.exec(data)[0];
//resp["timestamp"] = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/.exec(data)[0];
//resp["log_level"] = /[A-Z]{4,5}/.exec(data)[0];
return resp["ipaddress"];
}
"#;
context.eval(script).unwrap();
let js_fn_str = format!("{}({:?})", "pow", log);
for _i in 0..num_recs {
let _value = context.eval(&js_fn_str).unwrap();
}
println!("exiting boa");
}
pub async fn lua_test(num_recs: i32, log: String) {
let lua = Lua::new();
let globals = lua.globals();
for _i in 0..num_recs {
globals.set("log", log.to_owned()).unwrap();
/* lua.scope(|scope| {
Ok({
lua.globals().set(
"pow",
scope.create_function_mut(|_, data: String| {
//print!("The maximum of the two numbers is {}", data);
rust_val = data:match("(%d+%.%d+%.%d+%.%d+%:-%d+)")
Ok(())
})?,
)?;
lua.load("pow(log)").exec().unwrap();
})
})
.unwrap(); */
let _res = lua
.load(
r#"
return log:match("(%d+%.%d+%.%d+%.%d+%:-%d+)")
"#,
)
.eval::<String>()
.unwrap();
//println!("Result from lua {}", res);
}
println!("exiting lua");
}
pub fn v8_test(num_recs: i32, log: String) {
let _js_fn_str = format!(
r#"var log = '{}';{};"#,
log, r#"/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/.exec(log)[0]"#
);
let js_func_script = r#"function pow(data) {
console.log(data)
let resp = {}
resp["ipaddress"] = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/.exec(data)[0];
resp["timestamp"] = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/.exec(data)[0];
resp["log_level"] = /[A-Z]{4,5}/.exec(data)[0];
return resp["ipaddress"];
}"#;
let platform = v8::new_default_platform(0, false).make_shared();
v8::V8::initialize_platform(platform);
v8::V8::initialize();
// Create a new Isolate and make it the current one.
let isolate = &mut v8::Isolate::new(v8::CreateParams::default());
// Create a stack-allocated handle scope.
let handle_scope = &mut v8::HandleScope::new(isolate);
// Create a new context.
let context = v8::Context::new(handle_scope);
// Enter the context for compiling and running the hello world script.
let scope = &mut v8::ContextScope::new(handle_scope, context);
for _i in 0..num_recs {
//let source_log = v8::String::new(scope, &log).unwrap();
//js_func.push_str("pow(source_log)");
let js_fn_str = format!(
r#"var log = '{}';{};{};"#,
log, js_func_script, r#"pow(log)"#
);
let source_reg = v8::String::new(scope, &js_fn_str).unwrap();
// scope.add_context_data(context, source_log);
let script = v8::Script::compile(scope, source_reg, None).unwrap();
// Run the script to get the result.
let result = script.run(scope).unwrap();
let _result = result.to_string(scope).unwrap();
//println!("Result from JS {}", _result.to_rust_string_lossy(scope));
}
println!("exiting V8");
}
proto/cluster/common.proto
+30
View File
@@ -0,0 +1,30 @@
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.zinc.oxide.cluster";
option java_outer_classname = "clusterProto";
package cluster;
message FileDescriptor {
FileMeta meta = 1;
StreamType file_type = 2;
}
message FileMeta {
int64 min_ts = 1; // microseconds
int64 max_ts = 2; // microseconds
uint64 records = 3;
uint64 original_size = 4;
uint64 compressed_size = 5;
}
enum StreamType {
LOGS = 0;
METRICS = 1;
TRACES = 2;
METADATA = 3;
FILELIST = 4;
}
message EmptyResponse {}
proto/cluster/event.proto
+25
View File
@@ -0,0 +1,25 @@
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.zinc.oxide.cluster";
option java_outer_classname = "clusterProto";
package cluster;
import "cluster/common.proto";
service Event {
rpc SendFileList (FileList) returns (EmptyResponse) {}
}
message FileList {
repeated FileKey items = 1;
}
message FileKey {
string key = 1;
FileMeta meta = 2;
bool deleted = 3;
}
proto/cluster/search.proto
+78
View File
@@ -0,0 +1,78 @@
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.zinc.oxide.cluster";
option java_outer_classname = "clusterProto";
package cluster;
service Search {
rpc Search (SearchRequest) returns (SearchResponse) {}
}
// Job information for a request
message Job {
string session_id = 1;
string job = 2;
int32 stage = 3;
int32 partition = 4;
}
message Partition {
int64 time_min = 1;
int64 time_max = 2;
}
// Search request type
enum SearchType {
USER = 0; // user input search request
CLUSTER = 1; // cluster dispatch search request
CACHE_ONLY = 2; // ingester node just search local cache
}
// Search request query
message SearchQuery {
string sql = 1;
string sql_mode = 2;
int32 from = 3;
int32 size = 4;
int64 start_time = 5;
int64 end_time = 6;
bool track_total_hits = 7;
}
// Search request
message SearchRequest {
Job job = 1;
string org_id = 2;
string stream_type = 3;
SearchType stype = 4;
SearchQuery query = 5;
Partition partition = 6;
repeated string file_list = 7;
repeated SearchAggRequest aggs = 8;
}
// The response message containing the greetings
message SearchResponse {
Job job = 1;
int32 took = 2;
int64 total = 3;
int32 from = 4;
int32 size = 5;
int32 file_count = 6;
int32 scan_size = 7; // unit: mb
bytes hits = 8;
repeated SearchAggResponse aggs = 9;
}
message SearchAggRequest {
string name = 1;
string sql = 2;
}
message SearchAggResponse {
string name = 1;
bytes hits = 2;
}
proto/gogoproto/gogo.proto
+144
View File
@@ -0,0 +1,144 @@
// Protocol Buffers for Go with Gadgets
//
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
// http://github.com/gogo/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
syntax = "proto2";
package gogoproto;
import "google/protobuf/descriptor.proto";
option java_package = "com.google.protobuf";
option java_outer_classname = "GoGoProtos";
option go_package = "github.com/gogo/protobuf/gogoproto";
extend google.protobuf.EnumOptions {
optional bool goproto_enum_prefix = 62001;
optional bool goproto_enum_stringer = 62021;
optional bool enum_stringer = 62022;
optional string enum_customname = 62023;
optional bool enumdecl = 62024;
}
extend google.protobuf.EnumValueOptions {
optional string enumvalue_customname = 66001;
}
extend google.protobuf.FileOptions {
optional bool goproto_getters_all = 63001;
optional bool goproto_enum_prefix_all = 63002;
optional bool goproto_stringer_all = 63003;
optional bool verbose_equal_all = 63004;
optional bool face_all = 63005;
optional bool gostring_all = 63006;
optional bool populate_all = 63007;
optional bool stringer_all = 63008;
optional bool onlyone_all = 63009;
optional bool equal_all = 63013;
optional bool description_all = 63014;
optional bool testgen_all = 63015;
optional bool benchgen_all = 63016;
optional bool marshaler_all = 63017;
optional bool unmarshaler_all = 63018;
optional bool stable_marshaler_all = 63019;
optional bool sizer_all = 63020;
optional bool goproto_enum_stringer_all = 63021;
optional bool enum_stringer_all = 63022;
optional bool unsafe_marshaler_all = 63023;
optional bool unsafe_unmarshaler_all = 63024;
optional bool goproto_extensions_map_all = 63025;
optional bool goproto_unrecognized_all = 63026;
optional bool gogoproto_import = 63027;
optional bool protosizer_all = 63028;
optional bool compare_all = 63029;
optional bool typedecl_all = 63030;
optional bool enumdecl_all = 63031;
optional bool goproto_registration = 63032;
optional bool messagename_all = 63033;
optional bool goproto_sizecache_all = 63034;
optional bool goproto_unkeyed_all = 63035;
}
extend google.protobuf.MessageOptions {
optional bool goproto_getters = 64001;
optional bool goproto_stringer = 64003;
optional bool verbose_equal = 64004;
optional bool face = 64005;
optional bool gostring = 64006;
optional bool populate = 64007;
optional bool stringer = 67008;
optional bool onlyone = 64009;
optional bool equal = 64013;
optional bool description = 64014;
optional bool testgen = 64015;
optional bool benchgen = 64016;
optional bool marshaler = 64017;
optional bool unmarshaler = 64018;
optional bool stable_marshaler = 64019;
optional bool sizer = 64020;
optional bool unsafe_marshaler = 64023;
optional bool unsafe_unmarshaler = 64024;
optional bool goproto_extensions_map = 64025;
optional bool goproto_unrecognized = 64026;
optional bool protosizer = 64028;
optional bool compare = 64029;
optional bool typedecl = 64030;
optional bool messagename = 64033;
optional bool goproto_sizecache = 64034;
optional bool goproto_unkeyed = 64035;
}
extend google.protobuf.FieldOptions {
optional bool nullable = 65001;
optional bool embed = 65002;
optional string customtype = 65003;
optional string customname = 65004;
optional string jsontag = 65005;
optional string moretags = 65006;
optional string casttype = 65007;
optional string castkey = 65008;
optional string castvalue = 65009;
optional bool stdtime = 65010;
optional bool stdduration = 65011;
optional bool wktpointer = 65012;
}
proto/googleapis/google/api/annotations.proto
+31
View File
@@ -0,0 +1,31 @@
// Copyright (c) 2015, Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
import "google/api/http.proto";
import "google/protobuf/descriptor.proto";
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "AnnotationsProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
extend google.protobuf.MethodOptions {
// See `HttpRule`.
HttpRule http = 72295728;
}
proto/googleapis/google/api/client.proto
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
import "google/protobuf/descriptor.proto";
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "ClientProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
extend google.protobuf.MethodOptions {
// A definition of a client library method signature.
//
// In client libraries, each proto RPC corresponds to one or more methods
// which the end user is able to call, and calls the underlying RPC.
// Normally, this method receives a single argument (a struct or instance
// corresponding to the RPC request object). Defining this field will
// add one or more overloads providing flattened or simpler method signatures
// in some languages.
//
// The fields on the method signature are provided as a comma-separated
// string.
//
// For example, the proto RPC and annotation:
//
// rpc CreateSubscription(CreateSubscriptionRequest)
// returns (Subscription) {
// option (google.api.method_signature) = "name,topic";
// }
//
// Would add the following Java overload (in addition to the method accepting
// the request object):
//
// public final Subscription createSubscription(String name, String topic)
//
// The following backwards-compatibility guidelines apply:
//
// * Adding this annotation to an unannotated method is backwards
// compatible.
// * Adding this annotation to a method which already has existing
// method signature annotations is backwards compatible if and only if
// the new method signature annotation is last in the sequence.
// * Modifying or removing an existing method signature annotation is
// a breaking change.
// * Re-ordering existing method signature annotations is a breaking
// change.
repeated string method_signature = 1051;
}
extend google.protobuf.ServiceOptions {
// The hostname for this service.
// This should be specified with no prefix or protocol.
//
// Example:
//
// service Foo {
// option (google.api.default_host) = "foo.googleapi.com";
// ...
// }
string default_host = 1049;
// OAuth scopes needed for the client.
//
// Example:
//
// service Foo {
// option (google.api.oauth_scopes) = \
// "https://www.googleapis.com/auth/cloud-platform";
// ...
// }
//
// If there is more than one scope, use a comma-separated string:
//
// Example:
//
// service Foo {
// option (google.api.oauth_scopes) = \
// "https://www.googleapis.com/auth/cloud-platform,"
// "https://www.googleapis.com/auth/monitoring";
// ...
// }
string oauth_scopes = 1050;
}
proto/googleapis/google/api/field_behavior.proto
+84
View File
@@ -0,0 +1,84 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
import "google/protobuf/descriptor.proto";
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "FieldBehaviorProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
extend google.protobuf.FieldOptions {
// A designation of a specific field behavior (required, output only, etc.)
// in protobuf messages.
//
// Examples:
//
// string name = 1 [(google.api.field_behavior) = REQUIRED];
// State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// google.protobuf.Duration ttl = 1
// [(google.api.field_behavior) = INPUT_ONLY];
// google.protobuf.Timestamp expire_time = 1
// [(google.api.field_behavior) = OUTPUT_ONLY,
// (google.api.field_behavior) = IMMUTABLE];
repeated google.api.FieldBehavior field_behavior = 1052;
}
// An indicator of the behavior of a given field (for example, that a field
// is required in requests, or given as output but ignored as input).
// This **does not** change the behavior in protocol buffers itself; it only
// denotes the behavior and may affect how API tooling handles the field.
//
// Note: This enum **may** receive new values in the future.
enum FieldBehavior {
// Conventional default for enums. Do not use this.
FIELD_BEHAVIOR_UNSPECIFIED = 0;
// Specifically denotes a field as optional.
// While all fields in protocol buffers are optional, this may be specified
// for emphasis if appropriate.
OPTIONAL = 1;
// Denotes a field as required.
// This indicates that the field **must** be provided as part of the request,
// and failure to do so will cause an error (usually `INVALID_ARGUMENT`).
REQUIRED = 2;
// Denotes a field as output only.
// This indicates that the field is provided in responses, but including the
// field in a request does nothing (the server *must* ignore it and
// *must not* throw an error as a result of the field's presence).
OUTPUT_ONLY = 3;
// Denotes a field as input only.
// This indicates that the field is provided in requests, and the
// corresponding field is not included in output.
INPUT_ONLY = 4;
// Denotes a field as immutable.
// This indicates that the field may be set once in a request to create a
// resource, but may not be changed thereafter.
IMMUTABLE = 5;
// Denotes that a (repeated) field is an unordered list.
// This indicates that the service may provide the elements of the list
// in any arbitrary order, rather than the order the user originally
// provided. Additionally, the list's order may or may not be stable.
UNORDERED_LIST = 6;
}
proto/googleapis/google/api/http.proto
+375
View File
@@ -0,0 +1,375 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
option cc_enable_arenas = true;
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "HttpProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
// Defines the HTTP configuration for an API service. It contains a list of
// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method
// to one or more HTTP REST API methods.
message Http {
// A list of HTTP configuration rules that apply to individual API methods.
//
// **NOTE:** All service configuration rules follow "last one wins" order.
repeated HttpRule rules = 1;
// When set to true, URL path parameters will be fully URI-decoded except in
// cases of single segment matches in reserved expansion, where "%2F" will be
// left encoded.
//
// The default behavior is to not decode RFC 6570 reserved characters in multi
// segment matches.
bool fully_decode_reserved_expansion = 2;
}
// # gRPC Transcoding
//
// gRPC Transcoding is a feature for mapping between a gRPC method and one or
// more HTTP REST endpoints. It allows developers to build a single API service
// that supports both gRPC APIs and REST APIs. Many systems, including [Google
// APIs](https://github.com/googleapis/googleapis),
// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC
// Gateway](https://github.com/grpc-ecosystem/grpc-gateway),
// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature
// and use it for large scale production services.
//
// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies
// how different portions of the gRPC request message are mapped to the URL
// path, URL query parameters, and HTTP request body. It also controls how the
// gRPC response message is mapped to the HTTP response body. `HttpRule` is
// typically specified as an `google.api.http` annotation on the gRPC method.
//
// Each mapping specifies a URL path template and an HTTP method. The path
// template may refer to one or more fields in the gRPC request message, as long
// as each field is a non-repeated field with a primitive (non-message) type.
// The path template controls how fields of the request message are mapped to
// the URL path.
//
// Example:
//
// service Messaging {
// rpc GetMessage(GetMessageRequest) returns (Message) {
// option (google.api.http) = {
// get: "/v1/{name=messages/*}"
// };
// }
// }
// message GetMessageRequest {
// string name = 1; // Mapped to URL path.
// }
// message Message {
// string text = 1; // The resource content.
// }
//
// This enables an HTTP REST to gRPC mapping as below:
//
// HTTP | gRPC
// -----|-----
// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")`
//
// Any fields in the request message which are not bound by the path template
// automatically become HTTP query parameters if there is no HTTP request body.
// For example:
//
// service Messaging {
// rpc GetMessage(GetMessageRequest) returns (Message) {
// option (google.api.http) = {
// get:"/v1/messages/{message_id}"
// };
// }
// }
// message GetMessageRequest {
// message SubMessage {
// string subfield = 1;
// }
// string message_id = 1; // Mapped to URL path.
// int64 revision = 2; // Mapped to URL query parameter `revision`.
// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`.
// }
//
// This enables a HTTP JSON to RPC mapping as below:
//
// HTTP | gRPC
// -----|-----
// `GET /v1/messages/123456?revision=2&sub.subfield=foo` |
// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield:
// "foo"))`
//
// Note that fields which are mapped to URL query parameters must have a
// primitive type or a repeated primitive type or a non-repeated message type.
// In the case of a repeated type, the parameter can be repeated in the URL
// as `...?param=A&param=B`. In the case of a message type, each field of the
// message is mapped to a separate parameter, such as
// `...?foo.a=A&foo.b=B&foo.c=C`.
//
// For HTTP methods that allow a request body, the `body` field
// specifies the mapping. Consider a REST update method on the
// message resource collection:
//
// service Messaging {
// rpc UpdateMessage(UpdateMessageRequest) returns (Message) {
// option (google.api.http) = {
// patch: "/v1/messages/{message_id}"
// body: "message"
// };
// }
// }
// message UpdateMessageRequest {
// string message_id = 1; // mapped to the URL
// Message message = 2; // mapped to the body
// }
//
// The following HTTP JSON to RPC mapping is enabled, where the
// representation of the JSON in the request body is determined by
// protos JSON encoding:
//
// HTTP | gRPC
// -----|-----
// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id:
// "123456" message { text: "Hi!" })`
//
// The special name `*` can be used in the body mapping to define that
// every field not bound by the path template should be mapped to the
// request body. This enables the following alternative definition of
// the update method:
//
// service Messaging {
// rpc UpdateMessage(Message) returns (Message) {
// option (google.api.http) = {
// patch: "/v1/messages/{message_id}"
// body: "*"
// };
// }
// }
// message Message {
// string message_id = 1;
// string text = 2;
// }
//
//
// The following HTTP JSON to RPC mapping is enabled:
//
// HTTP | gRPC
// -----|-----
// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id:
// "123456" text: "Hi!")`
//
// Note that when using `*` in the body mapping, it is not possible to
// have HTTP parameters, as all fields not bound by the path end in
// the body. This makes this option more rarely used in practice when
// defining REST APIs. The common usage of `*` is in custom methods
// which don't use the URL at all for transferring data.
//
// It is possible to define multiple HTTP methods for one RPC by using
// the `additional_bindings` option. Example:
//
// service Messaging {
// rpc GetMessage(GetMessageRequest) returns (Message) {
// option (google.api.http) = {
// get: "/v1/messages/{message_id}"
// additional_bindings {
// get: "/v1/users/{user_id}/messages/{message_id}"
// }
// };
// }
// }
// message GetMessageRequest {
// string message_id = 1;
// string user_id = 2;
// }
//
// This enables the following two alternative HTTP JSON to RPC mappings:
//
// HTTP | gRPC
// -----|-----
// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")`
// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id:
// "123456")`
//
// ## Rules for HTTP mapping
//
// 1. Leaf request fields (recursive expansion nested messages in the request
// message) are classified into three categories:
// - Fields referred by the path template. They are passed via the URL path.
// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP
// request body.
// - All other fields are passed via the URL query parameters, and the
// parameter name is the field path in the request message. A repeated
// field can be represented as multiple query parameters under the same
// name.
// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields
// are passed via URL path and HTTP request body.
// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all
// fields are passed via URL path and URL query parameters.
//
// ### Path template syntax
//
// Template = "/" Segments [ Verb ] ;
// Segments = Segment { "/" Segment } ;
// Segment = "*" | "**" | LITERAL | Variable ;
// Variable = "{" FieldPath [ "=" Segments ] "}" ;
// FieldPath = IDENT { "." IDENT } ;
// Verb = ":" LITERAL ;
//
// The syntax `*` matches a single URL path segment. The syntax `**` matches
// zero or more URL path segments, which must be the last part of the URL path
// except the `Verb`.
//
// The syntax `Variable` matches part of the URL path as specified by its
// template. A variable template must not contain other variables. If a variable
// matches a single path segment, its template may be omitted, e.g. `{var}`
// is equivalent to `{var=*}`.
//
// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL`
// contains any reserved character, such characters should be percent-encoded
// before the matching.
//
// If a variable contains exactly one path segment, such as `"{var}"` or
// `"{var=*}"`, when such a variable is expanded into a URL path on the client
// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The
// server side does the reverse decoding. Such variables show up in the
// [Discovery
// Document](https://developers.google.com/discovery/v1/reference/apis) as
// `{var}`.
//
// If a variable contains multiple path segments, such as `"{var=foo/*}"`
// or `"{var=**}"`, when such a variable is expanded into a URL path on the
// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded.
// The server side does the reverse decoding, except "%2F" and "%2f" are left
// unchanged. Such variables show up in the
// [Discovery
// Document](https://developers.google.com/discovery/v1/reference/apis) as
// `{+var}`.
//
// ## Using gRPC API Service Configuration
//
// gRPC API Service Configuration (service config) is a configuration language
// for configuring a gRPC service to become a user-facing product. The
// service config is simply the YAML representation of the `google.api.Service`
// proto message.
//
// As an alternative to annotating your proto file, you can configure gRPC
// transcoding in your service config YAML files. You do this by specifying a
// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same
// effect as the proto annotation. This can be particularly useful if you
// have a proto that is reused in multiple services. Note that any transcoding
// specified in the service config will override any matching transcoding
// configuration in the proto.
//
// Example:
//
// http:
// rules:
// # Selects a gRPC method and applies HttpRule to it.
// - selector: example.v1.Messaging.GetMessage
// get: /v1/messages/{message_id}/{sub.subfield}
//
// ## Special notes
//
// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the
// proto to JSON conversion must follow the [proto3
// specification](https://developers.google.com/protocol-buffers/docs/proto3#json).
//
// While the single segment variable follows the semantics of
// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String
// Expansion, the multi segment variable **does not** follow RFC 6570 Section
// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion
// does not expand special characters like `?` and `#`, which would lead
// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding
// for multi segment variables.
//
// The path variables **must not** refer to any repeated or mapped field,
// because client libraries are not capable of handling such variable expansion.
//
// The path variables **must not** capture the leading "/" character. The reason
// is that the most common use case "{var}" does not capture the leading "/"
// character. For consistency, all path variables must share the same behavior.
//
// Repeated message fields must not be mapped to URL query parameters, because
// no client library can support such complicated mapping.
//
// If an API needs to use a JSON array for request or response body, it can map
// the request or response body to a repeated field. However, some gRPC
// Transcoding implementations may not support this feature.
message HttpRule {
// Selects a method to which this rule applies.
//
// Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
string selector = 1;
// Determines the URL pattern is matched by this rules. This pattern can be
// used with any of the {get|put|post|delete|patch} methods. A custom method
// can be defined using the 'custom' field.
oneof pattern {
// Maps to HTTP GET. Used for listing and getting information about
// resources.
string get = 2;
// Maps to HTTP PUT. Used for replacing a resource.
string put = 3;
// Maps to HTTP POST. Used for creating a resource or performing an action.
string post = 4;
// Maps to HTTP DELETE. Used for deleting a resource.
string delete = 5;
// Maps to HTTP PATCH. Used for updating a resource.
string patch = 6;
// The custom pattern is used for specifying an HTTP method that is not
// included in the `pattern` field, such as HEAD, or "*" to leave the
// HTTP method unspecified for this rule. The wild-card rule is useful
// for services that provide content to Web (HTML) clients.
CustomHttpPattern custom = 8;
}
// The name of the request field whose value is mapped to the HTTP request
// body, or `*` for mapping all request fields not captured by the path
// pattern to the HTTP body, or omitted for not having any HTTP request body.
//
// NOTE: the referred field must be present at the top-level of the request
// message type.
string body = 7;
// Optional. The name of the response field whose value is mapped to the HTTP
// response body. When omitted, the entire response message will be used
// as the HTTP response body.
//
// NOTE: The referred field must be present at the top-level of the response
// message type.
string response_body = 12;
// Additional HTTP bindings for the selector. Nested bindings must
// not contain an `additional_bindings` field themselves (that is,
// the nesting may only be one level deep).
repeated HttpRule additional_bindings = 11;
}
// A custom pattern is used for defining custom HTTP verb.
message CustomHttpPattern {
// The name of this custom HTTP verb.
string kind = 1;
// The path matched by this custom verb.
string path = 2;
}
proto/googleapis/google/api/resource.proto
+299
View File
@@ -0,0 +1,299 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
import "google/protobuf/descriptor.proto";
option cc_enable_arenas = true;
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "ResourceProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
extend google.protobuf.FieldOptions {
// An annotation that describes a resource reference, see
// [ResourceReference][].
google.api.ResourceReference resource_reference = 1055;
}
extend google.protobuf.FileOptions {
// An annotation that describes a resource definition without a corresponding
// message; see [ResourceDescriptor][].
repeated google.api.ResourceDescriptor resource_definition = 1053;
}
extend google.protobuf.MessageOptions {
// An annotation that describes a resource definition, see
// [ResourceDescriptor][].
google.api.ResourceDescriptor resource = 1053;
}
// A simple descriptor of a resource type.
//
// ResourceDescriptor annotates a resource message (either by means of a
// protobuf annotation or use in the service config), and associates the
// resource's schema, the resource type, and the pattern of the resource name.
//
// Example:
//
// message Topic {
// // Indicates this message defines a resource schema.
// // Declares the resource type in the format of {service}/{kind}.
// // For Kubernetes resources, the format is {api group}/{kind}.
// option (google.api.resource) = {
// type: "pubsub.googleapis.com/Topic"
// name_descriptor: {
// pattern: "projects/{project}/topics/{topic}"
// parent_type: "cloudresourcemanager.googleapis.com/Project"
// parent_name_extractor: "projects/{project}"
// }
// };
// }
//
// The ResourceDescriptor Yaml config will look like:
//
// resources:
// - type: "pubsub.googleapis.com/Topic"
// name_descriptor:
// - pattern: "projects/{project}/topics/{topic}"
// parent_type: "cloudresourcemanager.googleapis.com/Project"
// parent_name_extractor: "projects/{project}"
//
// Sometimes, resources have multiple patterns, typically because they can
// live under multiple parents.
//
// Example:
//
// message LogEntry {
// option (google.api.resource) = {
// type: "logging.googleapis.com/LogEntry"
// name_descriptor: {
// pattern: "projects/{project}/logs/{log}"
// parent_type: "cloudresourcemanager.googleapis.com/Project"
// parent_name_extractor: "projects/{project}"
// }
// name_descriptor: {
// pattern: "folders/{folder}/logs/{log}"
// parent_type: "cloudresourcemanager.googleapis.com/Folder"
// parent_name_extractor: "folders/{folder}"
// }
// name_descriptor: {
// pattern: "organizations/{organization}/logs/{log}"
// parent_type: "cloudresourcemanager.googleapis.com/Organization"
// parent_name_extractor: "organizations/{organization}"
// }
// name_descriptor: {
// pattern: "billingAccounts/{billing_account}/logs/{log}"
// parent_type: "billing.googleapis.com/BillingAccount"
// parent_name_extractor: "billingAccounts/{billing_account}"
// }
// };
// }
//
// The ResourceDescriptor Yaml config will look like:
//
// resources:
// - type: 'logging.googleapis.com/LogEntry'
// name_descriptor:
// - pattern: "projects/{project}/logs/{log}"
// parent_type: "cloudresourcemanager.googleapis.com/Project"
// parent_name_extractor: "projects/{project}"
// - pattern: "folders/{folder}/logs/{log}"
// parent_type: "cloudresourcemanager.googleapis.com/Folder"
// parent_name_extractor: "folders/{folder}"
// - pattern: "organizations/{organization}/logs/{log}"
// parent_type: "cloudresourcemanager.googleapis.com/Organization"
// parent_name_extractor: "organizations/{organization}"
// - pattern: "billingAccounts/{billing_account}/logs/{log}"
// parent_type: "billing.googleapis.com/BillingAccount"
// parent_name_extractor: "billingAccounts/{billing_account}"
//
// For flexible resources, the resource name doesn't contain parent names, but
// the resource itself has parents for policy evaluation.
//
// Example:
//
// message Shelf {
// option (google.api.resource) = {
// type: "library.googleapis.com/Shelf"
// name_descriptor: {
// pattern: "shelves/{shelf}"
// parent_type: "cloudresourcemanager.googleapis.com/Project"
// }
// name_descriptor: {
// pattern: "shelves/{shelf}"
// parent_type: "cloudresourcemanager.googleapis.com/Folder"
// }
// };
// }
//
// The ResourceDescriptor Yaml config will look like:
//
// resources:
// - type: 'library.googleapis.com/Shelf'
// name_descriptor:
// - pattern: "shelves/{shelf}"
// parent_type: "cloudresourcemanager.googleapis.com/Project"
// - pattern: "shelves/{shelf}"
// parent_type: "cloudresourcemanager.googleapis.com/Folder"
message ResourceDescriptor {
// A description of the historical or future-looking state of the
// resource pattern.
enum History {
// The "unset" value.
HISTORY_UNSPECIFIED = 0;
// The resource originally had one pattern and launched as such, and
// additional patterns were added later.
ORIGINALLY_SINGLE_PATTERN = 1;
// The resource has one pattern, but the API owner expects to add more
// later. (This is the inverse of ORIGINALLY_SINGLE_PATTERN, and prevents
// that from being necessary once there are multiple patterns.)
FUTURE_MULTI_PATTERN = 2;
}
// A flag representing a specific style that a resource claims to conform to.
enum Style {
// The unspecified value. Do not use.
STYLE_UNSPECIFIED = 0;
// This resource is intended to be "declarative-friendly".
//
// Declarative-friendly resources must be more strictly consistent, and
// setting this to true communicates to tools that this resource should
// adhere to declarative-friendly expectations.
//
// Note: This is used by the API linter (linter.aip.dev) to enable
// additional checks.
DECLARATIVE_FRIENDLY = 1;
}
// The resource type. It must be in the format of
// {service_name}/{resource_type_kind}. The `resource_type_kind` must be
// singular and must not include version numbers.
//
// Example: `storage.googleapis.com/Bucket`
//
// The value of the resource_type_kind must follow the regular expression
// /[A-Za-z][a-zA-Z0-9]+/. It should start with an upper case character and
// should use PascalCase (UpperCamelCase). The maximum number of
// characters allowed for the `resource_type_kind` is 100.
string type = 1;
// Optional. The relative resource name pattern associated with this resource
// type. The DNS prefix of the full resource name shouldn't be specified here.
//
// The path pattern must follow the syntax, which aligns with HTTP binding
// syntax:
//
// Template = Segment { "/" Segment } ;
// Segment = LITERAL | Variable ;
// Variable = "{" LITERAL "}" ;
//
// Examples:
//
// - "projects/{project}/topics/{topic}"
// - "projects/{project}/knowledgeBases/{knowledge_base}"
//
// The components in braces correspond to the IDs for each resource in the
// hierarchy. It is expected that, if multiple patterns are provided,
// the same component name (e.g. "project") refers to IDs of the same
// type of resource.
repeated string pattern = 2;
// Optional. The field on the resource that designates the resource name
// field. If omitted, this is assumed to be "name".
string name_field = 3;
// Optional. The historical or future-looking state of the resource pattern.
//
// Example:
//
// // The InspectTemplate message originally only supported resource
// // names with organization, and project was added later.
// message InspectTemplate {
// option (google.api.resource) = {
// type: "dlp.googleapis.com/InspectTemplate"
// pattern:
// "organizations/{organization}/inspectTemplates/{inspect_template}"
// pattern: "projects/{project}/inspectTemplates/{inspect_template}"
// history: ORIGINALLY_SINGLE_PATTERN
// };
// }
History history = 4;
// The plural name used in the resource name and permission names, such as
// 'projects' for the resource name of 'projects/{project}' and the permission
// name of 'cloudresourcemanager.googleapis.com/projects.get'. It is the same
// concept of the `plural` field in k8s CRD spec
// https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
//
// Note: The plural form is required even for singleton resources. See
// https://aip.dev/156
string plural = 5;
// The same concept of the `singular` field in k8s CRD spec
// https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
// Such as "project" for the `resourcemanager.googleapis.com/Project` type.
string singular = 6;
// Style flag(s) for this resource.
// These indicate that a resource is expected to conform to a given
// style. See the specific style flags for additional information.
repeated Style style = 10;
}
// Defines a proto annotation that describes a string field that refers to
// an API resource.
message ResourceReference {
// The resource type that the annotated field references.
//
// Example:
//
// message Subscription {
// string topic = 2 [(google.api.resource_reference) = {
// type: "pubsub.googleapis.com/Topic"
// }];
// }
//
// Occasionally, a field may reference an arbitrary resource. In this case,
// APIs use the special value * in their resource reference.
//
// Example:
//
// message GetIamPolicyRequest {
// string resource = 2 [(google.api.resource_reference) = {
// type: "*"
// }];
// }
string type = 1;
// The resource type of a child collection that the annotated field
// references. This is useful for annotating the `parent` field that
// doesn't have a fixed resource type.
//
// Example:
//
// message ListLogEntriesRequest {
// string parent = 1 [(google.api.resource_reference) = {
// child_type: "logging.googleapis.com/LogEntry"
// };
// }
string child_type = 2;
}
proto/googleapis/google/pubsub/v1/pubsub.proto
+1316
View File
File diff suppressed because it is too large Load Diff
proto/googleapis/google/pubsub/v1/schema.proto
+289
View File
@@ -0,0 +1,289 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.pubsub.v1;
import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/protobuf/empty.proto";
option cc_enable_arenas = true;
option csharp_namespace = "Google.Cloud.PubSub.V1";
option go_package = "google.golang.org/genproto/googleapis/pubsub/v1;pubsub";
option java_multiple_files = true;
option java_outer_classname = "SchemaProto";
option java_package = "com.google.pubsub.v1";
option php_namespace = "Google\\Cloud\\PubSub\\V1";
option ruby_package = "Google::Cloud::PubSub::V1";
// Service for doing schema-related operations.
//
// EXPERIMENTAL: The Schema service is in development and may not work yet.
service SchemaService {
option (google.api.default_host) = "pubsub.googleapis.com";
option (google.api.oauth_scopes) =
"https://www.googleapis.com/auth/cloud-platform,"
"https://www.googleapis.com/auth/pubsub";
// Creates a schema.
rpc CreateSchema(CreateSchemaRequest) returns (Schema) {
option (google.api.http) = {
post: "/v1/{parent=projects/*}/schemas"
body: "schema"
};
option (google.api.method_signature) = "parent,schema,schema_id";
}
// Gets a schema.
rpc GetSchema(GetSchemaRequest) returns (Schema) {
option (google.api.http) = {
get: "/v1/{name=projects/*/schemas/*}"
};
option (google.api.method_signature) = "name";
}
// Lists schemas in a project.
rpc ListSchemas(ListSchemasRequest) returns (ListSchemasResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*}/schemas"
};
option (google.api.method_signature) = "parent";
}
// Deletes a schema.
rpc DeleteSchema(DeleteSchemaRequest) returns (google.protobuf.Empty) {
option (google.api.http) = {
delete: "/v1/{name=projects/*/schemas/*}"
};
option (google.api.method_signature) = "name";
}
// Validates a schema.
rpc ValidateSchema(ValidateSchemaRequest) returns (ValidateSchemaResponse) {
option (google.api.http) = {
post: "/v1/{parent=projects/*}/schemas:validate"
body: "*"
};
option (google.api.method_signature) = "parent,schema";
}
// Validates a message against a schema.
rpc ValidateMessage(ValidateMessageRequest)
returns (ValidateMessageResponse) {
option (google.api.http) = {
post: "/v1/{parent=projects/*}/schemas:validateMessage"
body: "*"
};
}
}
// A schema resource.
message Schema {
option (google.api.resource) = {
type: "pubsub.googleapis.com/Schema"
pattern: "projects/{project}/schemas/{schema}"
};
// Possible schema definition types.
enum Type {
// Default value. This value is unused.
TYPE_UNSPECIFIED = 0;
// A Protocol Buffer schema definition.
PROTOCOL_BUFFER = 1;
// An Avro schema definition.
AVRO = 2;
}
// Required. Name of the schema.
// Format is `projects/{project}/schemas/{schema}`.
string name = 1 [(google.api.field_behavior) = REQUIRED];
// The type of the schema definition.
Type type = 2;
// The definition of the schema. This should contain a string representing
// the full definition of the schema that is a valid schema definition of
// the type specified in `type`.
string definition = 3;
}
// Request for the CreateSchema method.
message CreateSchemaRequest {
// Required. The name of the project in which to create the schema.
// Format is `projects/{project-id}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
child_type: "pubsub.googleapis.com/Schema"
}
];
// Required. The schema object to create.
//
// This schema's `name` parameter is ignored. The schema object returned
// by CreateSchema will have a `name` made using the given `parent` and
// `schema_id`.
Schema schema = 2 [(google.api.field_behavior) = REQUIRED];
// The ID to use for the schema, which will become the final component of
// the schema's resource name.
//
// See https://cloud.google.com/pubsub/docs/admin#resource_names for resource
// name constraints.
string schema_id = 3;
}
// View of Schema object fields to be returned by GetSchema and ListSchemas.
enum SchemaView {
// The default / unset value.
// The API will default to the BASIC view.
SCHEMA_VIEW_UNSPECIFIED = 0;
// Include the name and type of the schema, but not the definition.
BASIC = 1;
// Include all Schema object fields.
FULL = 2;
}
// Request for the GetSchema method.
message GetSchemaRequest {
// Required. The name of the schema to get.
// Format is `projects/{project}/schemas/{schema}`.
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" }
];
// The set of fields to return in the response. If not set, returns a Schema
// with `name` and `type`, but not `definition`. Set to `FULL` to retrieve all
// fields.
SchemaView view = 2;
}
// Request for the `ListSchemas` method.
message ListSchemasRequest {
// Required. The name of the project in which to list schemas.
// Format is `projects/{project-id}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "cloudresourcemanager.googleapis.com/Project"
}
];
// The set of Schema fields to return in the response. If not set, returns
// Schemas with `name` and `type`, but not `definition`. Set to `FULL` to
// retrieve all fields.
SchemaView view = 2;
// Maximum number of schemas to return.
int32 page_size = 3;
// The value returned by the last `ListSchemasResponse`; indicates that
// this is a continuation of a prior `ListSchemas` call, and that the
// system should return the next page of data.
string page_token = 4;
}
// Response for the `ListSchemas` method.
message ListSchemasResponse {
// The resulting schemas.
repeated Schema schemas = 1;
// If not empty, indicates that there may be more schemas that match the
// request; this value should be passed in a new `ListSchemasRequest`.
string next_page_token = 2;
}
// Request for the `DeleteSchema` method.
message DeleteSchemaRequest {
// Required. Name of the schema to delete.
// Format is `projects/{project}/schemas/{schema}`.
string name = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" }
];
}
// Request for the `ValidateSchema` method.
message ValidateSchemaRequest {
// Required. The name of the project in which to validate schemas.
// Format is `projects/{project-id}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "cloudresourcemanager.googleapis.com/Project"
}
];
// Required. The schema object to validate.
Schema schema = 2 [(google.api.field_behavior) = REQUIRED];
}
// Response for the `ValidateSchema` method.
message ValidateSchemaResponse {}
// Request for the `ValidateMessage` method.
message ValidateMessageRequest {
// Required. The name of the project in which to validate schemas.
// Format is `projects/{project-id}`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "cloudresourcemanager.googleapis.com/Project"
}
];
oneof schema_spec {
// Name of the schema against which to validate.
//
// Format is `projects/{project}/schemas/{schema}`.
string name = 2 [
(google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" }
];
// Ad-hoc schema against which to validate
Schema schema = 3;
}
// Message to validate against the provided `schema_spec`.
bytes message = 4;
// The encoding expected for messages
Encoding encoding = 5;
}
// Response for the `ValidateMessage` method.
message ValidateMessageResponse {}
// Possible encoding types for messages.
enum Encoding {
// Unspecified
ENCODING_UNSPECIFIED = 0;
// JSON encoding
JSON = 1;
// Binary encoding, as defined by the schema type. For some schema types,
// binary encoding may not be available.
BINARY = 2;
}
proto/prometheus/proto_model.proto
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package io.prometheus.client;
option java_package = "io.prometheus.client";
message LabelPair {
optional string name = 1;
optional string value = 2;
}
enum MetricType {
COUNTER = 0;
GAUGE = 1;
SUMMARY = 2;
UNTYPED = 3;
HISTOGRAM = 4;
}
message Gauge {
optional double value = 1;
}
message Counter {
optional double value = 1;
}
message Quantile {
optional double quantile = 1;
optional double value = 2;
}
message Summary {
optional uint64 sample_count = 1;
optional double sample_sum = 2;
repeated Quantile quantile = 3;
}
message Untyped {
optional double value = 1;
}
message Histogram {
optional uint64 sample_count = 1;
optional double sample_sum = 2;
repeated Bucket bucket = 3; // Ordered in increasing order of upper_bound, +Inf bucket is optional.
}
message Bucket {
optional uint64 cumulative_count = 1; // Cumulative in increasing order.
optional double upper_bound = 2; // Inclusive.
}
message Metric {
repeated LabelPair label = 1;
optional Gauge gauge = 2;
optional Counter counter = 3;
optional Summary summary = 4;
optional Untyped untyped = 5;
optional Histogram histogram = 7;
optional int64 timestamp_ms = 6;
}
message MetricFamily {
optional string name = 1;
optional string help = 2;
optional MetricType type = 3;
repeated Metric metric = 4;
}
proto/prometheus/remote.proto
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2016 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package prometheus;
option go_package = "prompb";
import "prometheus/types.proto";
import "gogoproto/gogo.proto";
message WriteRequest {
repeated prometheus.TimeSeries timeseries = 1 [(gogoproto.nullable) = false];
// Cortex uses this field to determine the source of the write request.
// We reserve it to avoid any compatibility issues.
reserved 2;
repeated prometheus.MetricMetadata metadata = 3 [(gogoproto.nullable) = false];
}
// ReadRequest represents a remote read request.
message ReadRequest {
repeated Query queries = 1;
enum ResponseType {
// Server will return a single ReadResponse message with matched series that includes list of raw samples.
// It's recommended to use streamed response types instead.
//
// Response headers:
// Content-Type: "application/x-protobuf"
// Content-Encoding: "snappy"
SAMPLES = 0;
// Server will stream a delimited ChunkedReadResponse message that
// contains XOR or HISTOGRAM(!) encoded chunks for a single series.
// Each message is following varint size and fixed size bigendian
// uint32 for CRC32 Castagnoli checksum.
//
// Response headers:
// Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse"
// Content-Encoding: ""
STREAMED_XOR_CHUNKS = 1;
}
// accepted_response_types allows negotiating the content type of the response.
//
// Response types are taken from the list in the FIFO order. If no response type in `accepted_response_types` is
// implemented by server, error is returned.
// For request that do not contain `accepted_response_types` field the SAMPLES response type will be used.
repeated ResponseType accepted_response_types = 2;
}
// ReadResponse is a response when response_type equals SAMPLES.
message ReadResponse {
// In same order as the request's queries.
repeated QueryResult results = 1;
}
message Query {
int64 start_timestamp_ms = 1;
int64 end_timestamp_ms = 2;
repeated prometheus.LabelMatcher matchers = 3;
prometheus.ReadHints hints = 4;
}
message QueryResult {
// Samples within a time series must be ordered by time.
repeated prometheus.TimeSeries timeseries = 1;
}
// ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS.
// We strictly stream full series after series, optionally split by time. This means that a single frame can contain
// partition of the single series, but once a new series is started to be streamed it means that no more chunks will
// be sent for previous one. Series are returned sorted in the same way TSDB block are internally.
message ChunkedReadResponse {
repeated prometheus.ChunkedSeries chunked_series = 1;
// query_index represents an index of the query from ReadRequest.queries these chunks relates to.
int64 query_index = 2;
}
proto/prometheus/types.proto
+186
View File
@@ -0,0 +1,186 @@
// Copyright 2017 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package prometheus;
option go_package = "prompb";
import "gogoproto/gogo.proto";
message MetricMetadata {
enum MetricType {
UNKNOWN = 0;
COUNTER = 1;
GAUGE = 2;
HISTOGRAM = 3;
GAUGEHISTOGRAM = 4;
SUMMARY = 5;
INFO = 6;
STATESET = 7;
}
// Represents the metric type, these match the set from Prometheus.
// Refer to model/textparse/interface.go for details.
MetricType type = 1;
string metric_family_name = 2;
string help = 4;
string unit = 5;
}
message Sample {
double value = 1;
// timestamp is in ms format, see model/timestamp/timestamp.go for
// conversion from time.Time to Prometheus timestamp.
int64 timestamp = 2;
}
message Exemplar {
// Optional, can be empty.
repeated Label labels = 1 [(gogoproto.nullable) = false];
double value = 2;
// timestamp is in ms format, see model/timestamp/timestamp.go for
// conversion from time.Time to Prometheus timestamp.
int64 timestamp = 3;
}
// A native histogram, also known as a sparse histogram.
// Original design doc:
// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit
// The appendix of this design doc also explains the concept of float
// histograms. This Histogram message can represent both, the usual
// integer histogram as well as a float histogram.
message Histogram {
enum ResetHint {
UNKNOWN = 0; // Need to test for a counter reset explicitly.
YES = 1; // This is the 1st histogram after a counter reset.
NO = 2; // There was no counter reset between this and the previous Histogram.
GAUGE = 3; // This is a gauge histogram where counter resets don't happen.
}
oneof count { // Count of observations in the histogram.
uint64 count_int = 1;
double count_float = 2;
}
double sum = 3; // Sum of observations in the histogram.
// The schema defines the bucket schema. Currently, valid numbers
// are -4 <= n <= 8. They are all for base-2 bucket schemas, where 1
// is a bucket boundary in each case, and then each power of two is
// divided into 2^n logarithmic buckets. Or in other words, each
// bucket boundary is the previous boundary times 2^(2^-n). In the
// future, more bucket schemas may be added using numbers < -4 or >
// 8.
sint32 schema = 4;
double zero_threshold = 5; // Breadth of the zero bucket.
oneof zero_count { // Count in zero bucket.
uint64 zero_count_int = 6;
double zero_count_float = 7;
}
// Negative Buckets.
repeated BucketSpan negative_spans = 8;
// Use either "negative_deltas" or "negative_counts", the former for
// regular histograms with integer counts, the latter for float
// histograms.
repeated sint64 negative_deltas = 9; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
repeated double negative_counts = 10; // Absolute count of each bucket.
// Positive Buckets.
repeated BucketSpan positive_spans = 11;
// Use either "positive_deltas" or "positive_counts", the former for
// regular histograms with integer counts, the latter for float
// histograms.
repeated sint64 positive_deltas = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
repeated double positive_counts = 13; // Absolute count of each bucket.
ResetHint reset_hint = 14;
// timestamp is in ms format, see model/timestamp/timestamp.go for
// conversion from time.Time to Prometheus timestamp.
int64 timestamp = 15;
}
// A BucketSpan defines a number of consecutive buckets with their
// offset. Logically, it would be more straightforward to include the
// bucket counts in the Span. However, the protobuf representation is
// more compact in the way the data is structured here (with all the
// buckets in a single array separate from the Spans).
message BucketSpan {
sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative).
uint32 length = 2; // Length of consecutive buckets.
}
// TimeSeries represents samples and labels for a single time series.
message TimeSeries {
// For a timeseries to be valid, and for the samples and exemplars
// to be ingested by the remote system properly, the labels field is required.
repeated Label labels = 1 [(gogoproto.nullable) = false];
repeated Sample samples = 2 [(gogoproto.nullable) = false];
repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false];
repeated Histogram histograms = 4 [(gogoproto.nullable) = false];
}
message Label {
string name = 1;
string value = 2;
}
message Labels {
repeated Label labels = 1 [(gogoproto.nullable) = false];
}
// Matcher specifies a rule, which can match or set of labels or not.
message LabelMatcher {
enum Type {
EQ = 0;
NEQ = 1;
RE = 2;
NRE = 3;
}
Type type = 1;
string name = 2;
string value = 3;
}
message ReadHints {
int64 step_ms = 1; // Query step size in milliseconds.
string func = 2; // String representation of surrounding function or aggregation.
int64 start_ms = 3; // Start time in milliseconds.
int64 end_ms = 4; // End time in milliseconds.
repeated string grouping = 5; // List of label names used in aggregation.
bool by = 6; // Indicate whether it is without or by.
int64 range_ms = 7; // Range vector selector range in milliseconds.
}
// Chunk represents a TSDB chunk.
// Time range [min, max] is inclusive.
message Chunk {
int64 min_time_ms = 1;
int64 max_time_ms = 2;
// We require this to match chunkenc.Encoding.
enum Encoding {
UNKNOWN = 0;
XOR = 1;
HISTOGRAM = 2;
}
Encoding type = 3;
bytes data = 4;
}
// ChunkedSeries represents single, encoded time series.
message ChunkedSeries {
// Labels should be sorted.
repeated Label labels = 1 [(gogoproto.nullable) = false];
// Chunks will be in start time order and may overlap.
repeated Chunk chunks = 2 [(gogoproto.nullable) = false];
}
src/.DS_Store
Vendored
BIN
View File
Binary file not shown.
src/common/auth.rs
+26
View File
@@ -0,0 +1,26 @@
use argon2::{password_hash::SaltString, Algorithm, Argon2, Params, PasswordHasher, Version};
use crate::infra::config::PASSWORD_HASH;
pub fn get_hash(pass: &str, salt: &str) -> String {
let key = format!("{}{}", pass, salt);
let hash = PASSWORD_HASH.get(&key);
match hash {
Some(ret_hash) => ret_hash.value().to_string(),
None => {
let t_cost = 4;
let m_cost = 2048;
let p_cost = 2;
let params = Params::new(m_cost, t_cost, p_cost, None).unwrap();
let ctx = Argon2::new(Algorithm::Argon2d, Version::V0x10, params);
let password = pass.as_bytes();
let salt_string = SaltString::b64_encode(salt.as_bytes()).unwrap();
let password_hash = ctx
.hash_password(password, &salt_string)
.unwrap()
.to_string();
PASSWORD_HASH.insert(key, password_hash.clone());
password_hash
}
}
}
src/common/file.rs
+35
View File
@@ -0,0 +1,35 @@
use std::{
fs::{File, Metadata},
io::{Read, Write},
};
pub fn get_file_meta(file: &str) -> Result<Metadata, std::io::Error> {
let file = File::open(file)?;
file.metadata()
}
pub fn get_file_contents(file: &str) -> Result<Vec<u8>, std::io::Error> {
let mut file = File::open(file)?;
let mut contents: Vec<u8> = Vec::new();
file.read_to_end(&mut contents)?;
Ok(contents)
}
pub fn put_file_contents(file: &str, contents: &[u8]) -> Result<(), std::io::Error> {
let mut file = File::create(file)?;
file.write_all(contents)?;
Ok(())
}
pub fn delete_file(file: &str) -> Result<(), std::io::Error> {
std::fs::remove_file(file)?;
Ok(())
}
pub fn scan_files(pattern: &str) -> Vec<String> {
let files: Vec<String> = glob::glob(pattern)
.unwrap()
.map(|m| m.unwrap().to_str().unwrap().to_string())
.collect();
files
}
src/common/http.rs
+25
View File
@@ -0,0 +1,25 @@
use crate::meta::StreamType;
use actix_web::web::Query;
use ahash::AHashMap;
use std::io::{Error, ErrorKind};
pub fn get_stream_type_from_request(
query: &Query<AHashMap<String, String>>,
) -> Result<Option<StreamType>, Error> {
let stream_type = match query.get("type") {
Some(s) => match s.to_lowercase().as_str() {
"logs" => Some(StreamType::Logs),
"metrics" => Some(StreamType::Metrics),
"traces" => Some(StreamType::Traces),
_ => {
return Err(Error::new(
ErrorKind::Other,
"'type' query param with value 'logs' ,'metrics' or 'traces' allowed",
));
}
},
None => None,
};
Ok(stream_type)
}
src/common/json.rs
+151
View File
@@ -0,0 +1,151 @@
use flatten_json_object::{ArrayFormatting, Flattener};
pub type Value = serde_json::Value;
#[inline(always)]
#[cfg(target_arch = "x86_64")]
pub fn to_string<T>(value: &T) -> Result<String, simd_json::Error>
where
T: ?Sized + serde::Serialize,
{
simd_json::to_string(value)
}
#[inline(always)]
#[cfg(target_arch = "x86_64")]
pub fn to_vec<T>(value: &T) -> Result<Vec<u8>, simd_json::Error>
where
T: ?Sized + serde::Serialize,
{
simd_json::to_vec(value)
}
#[inline(always)]
#[cfg(target_arch = "x86_64")]
pub fn from_str<'a, T>(s: &'a str) -> Result<T, serde_json::Error>
where
T: serde::Deserialize<'a>,
{
from_slice(s.as_bytes())
}
#[inline(always)]
#[cfg(target_arch = "x86_64")]
pub fn from_slice<'a, T>(v: &'a [u8]) -> Result<T, serde_json::Error>
where
T: serde::Deserialize<'a>,
{
serde_json::from_slice(v)
}
#[inline(always)]
#[cfg(not(target_arch = "x86_64"))]
pub fn to_string<T>(value: &T) -> Result<String, serde_json::Error>
where
T: ?Sized + serde::Serialize,
{
serde_json::to_string(value)
}
#[inline(always)]
#[cfg(not(target_arch = "x86_64"))]
pub fn to_vec<T>(value: &T) -> Result<Vec<u8>, serde_json::Error>
where
T: ?Sized + serde::Serialize,
{
serde_json::to_vec(value)
}
#[inline(always)]
#[cfg(not(target_arch = "x86_64"))]
pub fn from_str<'a, T>(s: &'a str) -> Result<T, serde_json::Error>
where
T: serde::Deserialize<'a>,
{
from_slice(s.as_bytes())
}
#[inline(always)]
#[cfg(not(target_arch = "x86_64"))]
pub fn from_slice<'a, T>(v: &'a [u8]) -> Result<T, serde_json::Error>
where
T: serde::Deserialize<'a>,
{
serde_json::from_slice(v)
}
#[inline(always)]
pub fn flatten_json(obj: &Value) -> Value {
Flattener::new()
.set_key_separator(".")
.set_array_formatting(ArrayFormatting::Surrounded {
start: "[".to_string(),
end: "]".to_string(),
})
.set_preserve_empty_arrays(false)
.set_preserve_empty_objects(false)
.flatten(obj)
.unwrap()
}
pub fn unflatten_json(obj: &Value) -> Value {
if !obj.is_object() {
return obj.to_owned();
}
let mut unflattened = serde_json::Map::new();
for (key, value) in obj.as_object().unwrap() {
let mut current = &mut unflattened;
let mut parts = key.split('.');
let last = parts.next_back().unwrap();
for part in parts {
let old = current.get(part);
if let Some(old) = old {
if !old.is_object() {
let old = old.to_owned();
current.insert(part.to_string(), Value::Object(serde_json::Map::new()));
current = current.get_mut(part).unwrap().as_object_mut().unwrap();
current.insert("".to_string(), old);
} else {
current = current.get_mut(part).unwrap().as_object_mut().unwrap();
}
} else {
current.insert(part.to_string(), Value::Object(serde_json::Map::new()));
current = current.get_mut(part).unwrap().as_object_mut().unwrap();
}
}
current.insert(last.to_string(), value.clone());
}
Value::Object(unflattened)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_flatten_json() {
let obj = json!({"key": "value", "nested_key": {"key": "value", "foo": "bar"}});
assert_eq!(
flatten_json(&obj),
json!({"key": "value", "nested_key.key": "value", "nested_key.foo": "bar"}),
);
}
#[test]
fn test_unflatten_json1() {
let obj = json!({"key1": "value1", "nested_key.key2": "value2", "nested_key.foo": "bar"});
assert_eq!(
unflatten_json(&obj),
json!({"key1": "value1", "nested_key": {"key2": "value2", "foo": "bar"}}),
);
}
#[test]
fn test_unflatten_json2() {
let obj =
json!({"key1": "value1", "nested_key.key2": "value2", "nested_key.key2.foo": "bar"});
assert_eq!(
unflatten_json(&obj),
json!({"key1": "value1", "nested_key": {"key2": {"":"value2", "foo": "bar"}}}),
);
}
}
src/common/mod.rs
+9
View File
@@ -0,0 +1,9 @@
pub mod auth;
pub mod file;
pub mod http;
pub mod json;
pub mod notification;
pub mod str;
pub mod stream;
pub mod time;
pub mod utils;
src/common/notification.rs
+15
View File
@@ -0,0 +1,15 @@
use std::error::Error as StdError;
pub async fn send_notification(
url: &str,
payload: serde_json::Value,
) -> Result<(), Box<dyn StdError>> {
let client = reqwest::Client::new();
let loc_url = url::Url::parse(url).unwrap();
let req = client
.post(loc_url)
.header("Content-type", "application/json")
.json(&payload);
let _ = req.send().await.unwrap();
Ok(())
}
src/common/str.rs
+23
View File
@@ -0,0 +1,23 @@
#[inline(always)]
#[cfg(target_arch = "x86_64")]
pub fn find(haystack: &str, needle: &str) -> bool {
memchr::memmem::find(haystack.as_bytes(), needle.as_bytes()).is_some()
}
#[inline(always)]
#[cfg(not(target_arch = "x86_64"))]
pub fn find(haystack: &str, needle: &str) -> bool {
haystack.contains(needle)
}
#[cfg(test)]
mod test_utils {
use super::*;
#[test]
fn test_find() {
let haystack = "This is Zinc-oxide";
let needle = "oxide";
let result = find(haystack, needle);
assert_eq!(result, true)
}
}
src/common/stream.rs
+14
View File
@@ -0,0 +1,14 @@
use actix_web::HttpResponse;
use std::io::{Error, ErrorKind};
pub fn stream_type_query_param_error() -> Result<HttpResponse, Error> {
/* return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some("only 'type' query param with value 'logs' or 'metrics' allowed".to_string()),
))); */
Err(Error::new(
ErrorKind::Other,
"only 'type' query param with value 'logs' or 'metrics' allowed",
))
}
src/common/time.rs
+74
View File
@@ -0,0 +1,74 @@
use chrono::{DateTime, FixedOffset, TimeZone, Utc};
pub fn parse_str_to_time(s: &str) -> Result<DateTime<FixedOffset>, anyhow::Error> {
if s.contains('T') {
if s.len() == 19 {
let fmt = "%Y-%m-%dT%H:%M:%S";
let ret = Utc.datetime_from_str(s, fmt)?;
Ok(ret.into())
} else {
Ok(chrono::DateTime::parse_from_rfc3339(s)?)
}
} else if s.contains(',') {
Ok(chrono::DateTime::parse_from_rfc2822(s)?)
} else {
let fmt = "%Y-%m-%d %H:%M:%S";
let ret = Utc.datetime_from_str(s, fmt)?;
Ok(ret.into())
}
}
pub fn parse_str_to_timestamp_micros(v: &str) -> Result<i64, anyhow::Error> {
let n: i64 = match v.parse() {
Ok(i) => i,
Err(_) => match parse_str_to_time(v) {
Ok(v) => {
return Ok(v.timestamp_micros());
}
Err(_) => {
return Err(anyhow::anyhow!("invalid time format [string]"));
}
},
};
parse_i64_to_timestamp_micros(n)
}
pub fn parse_i64_to_timestamp_micros(v: i64) -> Result<i64, anyhow::Error> {
if v == 0 {
return Ok(0);
}
if v > (1e18 as i64) {
// nanoseconds
Ok(v / 1000)
} else if v > (1e15 as i64) {
// microseconds
Ok(v)
} else if v > (1e12 as i64) {
// milliseconds
Ok(v * 1000)
} else if v > (1e9 as i64) {
// seconds
Ok(v * 1000 * 1000)
} else {
Err(anyhow::anyhow!("Invalid time format [timestamp:value]"))
}
}
pub fn parse_timestamp_micro_from_value(v: &serde_json::Value) -> Result<i64, anyhow::Error> {
let n = match v {
serde_json::Value::String(s) => parse_str_to_timestamp_micros(s)?,
serde_json::Value::Number(n) => {
if n.is_i64() {
n.as_i64().unwrap()
} else if n.is_u64() {
n.as_u64().unwrap() as i64
} else if n.is_f64() {
n.as_f64().unwrap() as i64
} else {
return Err(anyhow::anyhow!("Invalid time format [timestamp]"));
}
}
_ => return Err(anyhow::anyhow!("Invalid time format [type]")),
};
parse_i64_to_timestamp_micros(n)
}
src/common/utils.rs
+149
View File
@@ -0,0 +1,149 @@
use datafusion::{
arrow::{datatypes::Schema, json, record_batch::RecordBatch},
datasource::MemTable,
prelude::SessionContext,
};
use serde_json::Value;
use std::sync::Arc;
use crate::infra::config::CONFIG;
use crate::meta::common::FileMeta;
use crate::meta::StreamType;
pub fn increment_stream_file_num_v1(file_name: &str) -> u32 {
let last_file_suffix_stream = &file_name.rfind('_').unwrap() + 1;
let last_file_suffix =
&file_name[last_file_suffix_stream..file_name.len() - CONFIG.common.file_ext_json.len()];
last_file_suffix.parse::<u32>().unwrap() + 1
}
pub fn get_stream_file_num_v1(file_name: &str) -> u32 {
let last_file_suffix_stream = &file_name.rfind('_').unwrap() + 1;
let last_file_suffix =
&file_name[last_file_suffix_stream..file_name.len() - CONFIG.common.file_ext_json.len()];
last_file_suffix.parse::<u32>().unwrap()
}
pub fn get_file_name_v1(org_id: &str, stream_name: &str, suffix: u32) -> String {
// creates file name like "./data/olympics/olympics#2022#09#13#13_1.json"
format!(
"{}{}/{}/{}/{}_{}{}",
&CONFIG.common.data_wal_dir,
org_id,
StreamType::Logs,
stream_name,
stream_name,
suffix,
&CONFIG.common.file_ext_json
)
}
pub async fn populate_file_meta(
schema: Arc<Schema>,
batch: Vec<Vec<RecordBatch>>,
file_meta: &mut FileMeta,
) {
let ctx = SessionContext::new();
let provider = MemTable::try_new(schema.clone(), batch).unwrap();
ctx.register_table("temp", Arc::new(provider)).unwrap();
let sql = format!(
"SELECT min({0}) as min, max({0}) as max ,count({0}) as num_records FROM temp ;",
CONFIG.common.time_stamp_col
);
let df = ctx.sql(sql.as_str()).await.unwrap();
let batches = df.collect().await.unwrap();
let json_rows = json::writer::record_batches_to_json_rows(&batches[..]).unwrap();
let mut result: Vec<Value> = json_rows
.into_iter()
.map(serde_json::Value::Object)
.collect();
let record = result.pop().unwrap();
file_meta.min_ts = record.get("min").unwrap().as_i64().unwrap();
file_meta.max_ts = record.get("max").unwrap().as_i64().unwrap();
file_meta.records = record.get("num_records").unwrap().as_u64().unwrap();
}
#[inline]
pub fn is_local_disk_storage() -> bool {
CONFIG.common.local_mode && CONFIG.common.local_mode_storage.eq("disk")
}
#[cfg(test)]
mod test_utils {
use super::*;
use datafusion::{
arrow::{
array::{Int64Array, StringArray},
datatypes::{DataType, Field},
},
from_slice::FromSlice,
};
#[test]
fn test_increment_stream_file_num_v1() {
let suffix_nums = vec![1, 9, 11, 78, 100, 234, 546];
for i in 0..suffix_nums.len() {
let suffix = increment_stream_file_num_v1(&format!(
"./data/WAL/nexus/logs/olympics/1663064862606912_{}.json",
suffix_nums[i]
));
assert_eq!(suffix, suffix_nums[i] + 1);
}
}
#[test]
fn test_get_file_name_v1() {
let file_key = get_file_name_v1(&"nexus".to_owned(), &"Olympics".to_owned(), 2);
assert_eq!(
file_key.as_str(),
"./data/wal/nexus/logs/Olympics/Olympics_2.json"
);
}
#[test]
fn test_get_stream_file_num_v1() {
let file_key = get_stream_file_num_v1("./data/WAL/logs/nexus/Olympics/Olympics_2.json");
assert_eq!(file_key, 2);
}
#[actix_web::test]
async fn test_populate_file_meta() {
// define a schema.
let val: i64 = 1666093521151350;
let schema = Arc::new(Schema::new(vec![
Field::new("log", DataType::Utf8, false),
Field::new("pod_id", DataType::Int64, false),
Field::new("_timestamp", DataType::Int64, false),
]));
// define data.
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(StringArray::from_slice(&["a", "b", "c", "d"])),
Arc::new(Int64Array::from_slice(&[1, 2, 1, 2])),
Arc::new(Int64Array::from_slice(&[
val - 100,
val - 10,
val - 90,
val,
])),
],
)
.unwrap();
//let file_name = path.file_name();
let mut file_meta = FileMeta {
min_ts: 0,
max_ts: 0,
records: 0,
original_size: 1000,
compressed_size: 700,
};
populate_file_meta(schema, vec![vec![batch]], &mut file_meta).await;
assert_eq!(file_meta.records, 4);
assert_eq!(file_meta.min_ts, val - 100);
}
}
src/handler/grpc/auth/mod.rs
+37
View File
@@ -0,0 +1,37 @@
use crate::{
common::auth::get_hash,
infra::config::{CONFIG, USERS},
};
use http_auth_basic::Credentials;
use tonic::{Request, Status};
pub fn check_auth(req: Request<()>) -> Result<Request<()>, Status> {
if !req.metadata().contains_key(&CONFIG.grpc.org_header_key)
&& !req.metadata().contains_key("authorization")
{
return Err(Status::unauthenticated("No valid auth token"));
}
let token = req
.metadata()
.get("authorization")
.unwrap()
.to_str()
.unwrap()
.to_string();
let credentials = match Credentials::from_header(token) {
Ok(c) => c,
Err(_) => return Err(Status::unauthenticated("No valid auth token")),
};
let user = USERS.get(&CONFIG.auth.username).unwrap();
let in_pass = get_hash(&credentials.password, &user.salt);
if credentials.user_id.eq(&CONFIG.auth.username)
&& (user.password.eq(&credentials.password) || user.password.eq(&in_pass))
{
Ok(req)
} else {
Err(Status::unauthenticated("No valid auth token"))
}
}
src/handler/grpc/mod.rs
+81
View File
@@ -0,0 +1,81 @@
use uuid::Uuid;
use crate::meta;
pub mod auth;
pub mod request;
pub mod cluster_rpc {
tonic::include_proto!("cluster");
}
impl From<meta::search::Request> for cluster_rpc::SearchRequest {
fn from(req: meta::search::Request) -> Self {
let req_query = cluster_rpc::SearchQuery {
sql: req.query.sql.clone(),
sql_mode: req.query.sql_mode.clone(),
from: req.query.from as i32,
size: req.query.size as i32,
start_time: req.query.start_time,
end_time: req.query.end_time,
track_total_hits: req.query.track_total_hits,
};
let job = cluster_rpc::Job {
session_id: Uuid::new_v4().to_string(),
job: "".to_string(),
stage: 0,
partition: 0,
};
let mut aggs = Vec::new();
for (name, sql) in req.aggs {
aggs.push(cluster_rpc::SearchAggRequest { name, sql });
}
cluster_rpc::SearchRequest {
job: Some(job),
org_id: "".to_string(),
stype: cluster_rpc::SearchType::User.into(),
query: Some(req_query),
aggs,
partition: None,
file_list: vec![],
stream_type: "".to_string(),
}
}
}
impl From<&meta::common::FileMeta> for cluster_rpc::FileMeta {
fn from(req: &meta::common::FileMeta) -> Self {
cluster_rpc::FileMeta {
min_ts: req.min_ts,
max_ts: req.max_ts,
records: req.records,
original_size: req.original_size,
compressed_size: req.compressed_size,
}
}
}
impl From<&cluster_rpc::FileMeta> for meta::common::FileMeta {
fn from(req: &cluster_rpc::FileMeta) -> Self {
meta::common::FileMeta {
min_ts: req.min_ts,
max_ts: req.max_ts,
records: req.records,
original_size: req.original_size,
compressed_size: req.compressed_size,
}
}
}
impl From<&meta::common::FileKey> for cluster_rpc::FileKey {
fn from(req: &meta::common::FileKey) -> Self {
cluster_rpc::FileKey {
key: req.key.clone(),
meta: Some(cluster_rpc::FileMeta::from(&req.meta)),
deleted: req.deleted,
}
}
}
src/handler/grpc/request/event.rs
+44
View File
@@ -0,0 +1,44 @@
use opentelemetry::global;
use tonic::{Request, Response, Status};
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::handler::grpc::cluster_rpc::event_server::Event;
use crate::handler::grpc::cluster_rpc::EmptyResponse;
use crate::handler::grpc::cluster_rpc::FileList;
use crate::meta::common::FileMeta;
use crate::service::db::file_list;
#[derive(Default)]
pub struct Eventer {}
#[tonic::async_trait]
impl Event for Eventer {
#[tracing::instrument(name = "grpc:event:SendFileList:enter", skip(self, req))]
async fn send_file_list(
&self,
req: Request<FileList>,
) -> Result<Response<EmptyResponse>, Status> {
let parent_cx = global::get_text_map_propagator(|prop| {
prop.extract(&super::MetadataMap(req.metadata()))
});
tracing::Span::current().set_parent(parent_cx);
let req = req.get_ref();
for file in req.items.iter() {
log::info!("received event:file, {:?}", file);
if let Err(e) = file_list::progress(
&file.key,
FileMeta::from(file.meta.as_ref().unwrap()),
file.deleted,
)
.await
{
return Err(Status::internal(e.to_string()));
}
}
let result = EmptyResponse {};
Ok(Response::new(result))
}
}
src/handler/grpc/request/mod.rs
+25
View File
@@ -0,0 +1,25 @@
use opentelemetry::propagation::Extractor;
pub mod event;
pub mod search;
pub mod traces;
pub struct MetadataMap<'a>(&'a tonic::metadata::MetadataMap);
impl<'a> Extractor for MetadataMap<'a> {
/// Get a value for a key from the MetadataMap. If the value can't be converted to &str, returns None
fn get(&self, key: &str) -> Option<&str> {
self.0.get(key).and_then(|metadata| metadata.to_str().ok())
}
/// Collect all the keys from the MetadataMap.
fn keys(&self) -> Vec<&str> {
self.0
.keys()
.map(|key| match key {
tonic::metadata::KeyRef::Ascii(v) => v.as_str(),
tonic::metadata::KeyRef::Binary(v) => v.as_str(),
})
.collect::<Vec<_>>()
}
}
src/handler/grpc/request/search.rs
+34
View File
@@ -0,0 +1,34 @@
use opentelemetry::global;
use tonic::{Request, Response, Status};
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::handler::grpc::cluster_rpc::search_server::Search;
use crate::handler::grpc::cluster_rpc::SearchRequest;
use crate::handler::grpc::cluster_rpc::SearchResponse;
use crate::service::search as SearchService;
#[derive(Default)]
pub struct Searcher {}
#[tonic::async_trait]
impl Search for Searcher {
#[tracing::instrument(name = "grpc:search:enter", skip(self, req))]
async fn search(
&self,
req: Request<SearchRequest>,
) -> Result<Response<SearchResponse>, Status> {
let parent_cx = global::get_text_map_propagator(|prop| {
prop.extract(&super::MetadataMap(req.metadata()))
});
tracing::Span::current().set_parent(parent_cx);
let req = req.get_ref();
let result = match SearchService::exec::search(req).await {
Ok(res) => res,
Err(err) => {
return Err(Status::internal(err.to_string()));
}
};
Ok(Response::new(result))
}
}
src/handler/grpc/request/traces.rs
+41
View File
@@ -0,0 +1,41 @@
use opentelemetry_proto::tonic::collector::trace::v1::{
trace_service_server::TraceService, ExportTraceServiceRequest, ExportTraceServiceResponse,
};
use tonic::Status;
use tonic::{codegen::*, Response};
use crate::infra::config::CONFIG;
use crate::service::traces::handle_trace_request;
#[derive(Default)]
pub struct TraceServer {}
#[async_trait]
impl TraceService for TraceServer {
async fn export(
&self,
request: tonic::Request<ExportTraceServiceRequest>,
) -> Result<tonic::Response<ExportTraceServiceResponse>, tonic::Status> {
let metadata = request.metadata().clone();
let msg = format!(
"Please specify orgnization id with header key '{}' ",
&CONFIG.grpc.org_header_key
);
if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
return Err(Status::invalid_argument(msg));
}
let in_req = request.into_inner();
let org_id = metadata.get(&CONFIG.grpc.org_header_key);
if org_id.is_none() {
return Err(Status::invalid_argument(msg));
}
let thread_id = Arc::new(0);
let resp = handle_trace_request(org_id.unwrap().to_str().unwrap(), thread_id, in_req).await;
if resp.is_ok() {
return Ok(Response::new(ExportTraceServiceResponse {}));
} else {
Err(Status::internal(resp.err().unwrap().to_string()))
}
}
}
src/handler/http/auth/mod.rs
+85
View File
@@ -0,0 +1,85 @@
use actix_web::{
dev::ServiceRequest,
error::{ErrorForbidden, ErrorUnauthorized},
Error,
};
use actix_web_httpauth::extractors::basic::BasicAuth;
use crate::common::auth::get_hash;
use crate::infra::config::CONFIG;
use crate::meta::user::Role;
use crate::service::users;
pub async fn validator(
req: ServiceRequest,
credentials: BasicAuth,
) -> Result<ServiceRequest, (Error, ServiceRequest)> {
let path = req.request().path().strip_prefix("/api/").unwrap();
match validate_credentials(
credentials.user_id(),
credentials.password().unwrap().trim(),
path,
)
.await
{
Ok(res) => {
if res {
Ok(req)
} else {
Err((ErrorUnauthorized("Unauthorized Access"), req))
}
}
Err(err) => Err((err, req)),
}
}
pub async fn validate_credentials(
user_id: &str,
user_password: &str,
path: &str,
) -> Result<bool, Error> {
let mut user = match path.find('/') {
Some(index) => {
let org_id = &path[0..index];
users::get_user(Some(org_id), user_id).await
}
None => users::get_user(None, user_id).await,
};
//this is only applicable for super admin user
if user.is_none() && is_admin_user(user_id).await {
user = users::get_user(None, user_id).await;
if user.is_none() {
return Ok(false);
}
} else {
return Ok(false);
}
let user = user.unwrap();
let in_pass = get_hash(user_password, &user.salt);
if !user.password.eq(&in_pass) {
return Ok(false);
}
if !path.contains("/user")
|| (path.contains("/user") && (user.role.eq(&Role::Admin) || user.role.eq(&Role::Root)))
{
Ok(true)
} else {
Err(ErrorForbidden("Not allowed"))
}
}
pub async fn is_admin_user(user_id: &str) -> bool {
user_id.eq(&CONFIG.auth.username)
}
#[cfg(test)]
mod test_utils {
use super::*;
#[actix_web::test]
async fn test_validate_credentials() {
let res = validate_credentials(&CONFIG.auth.username, &CONFIG.auth.password, "index").await;
assert_eq!(res.is_ok(), true)
}
}
src/handler/http/mod.rs
+4
View File
@@ -0,0 +1,4 @@
#![cfg_attr(coverage_nightly, feature(no_coverage))]
pub mod auth;
pub mod request;
pub mod router;
src/handler/http/request/alerts/mod.rs
+36
View File
@@ -0,0 +1,36 @@
use crate::{meta::alert::Alert, service::alerts};
use actix_web::{delete, get, post, web, HttpResponse, Responder};
use std::io::Error;
#[post("/{org_id}/{stream_name}/alerts/{alert_name}")]
pub async fn save_alert(
path: web::Path<(String, String, String)>,
alert: web::Json<Alert>,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name, name) = path.into_inner();
alerts::save_alert(org_id, stream_name, name, alert.into_inner()).await
}
#[get("/{org_id}/{stream_name}/alerts")]
async fn list_stream_alerts(path: web::Path<(String, String)>) -> impl Responder {
let (org_id, stream_name) = path.into_inner();
alerts::list_alert(org_id, Some(stream_name.as_str())).await
}
#[get("/{org_id}/alerts")]
async fn list_alerts(path: web::Path<String>) -> impl Responder {
let org_id = path.into_inner();
alerts::list_alert(org_id, None).await
}
#[get("/{org_id}/{stream_name}/alerts/{alert_name}")]
async fn get_alert(path: web::Path<(String, String, String)>) -> impl Responder {
let (org_id, stream_name, name) = path.into_inner();
alerts::get_alert(org_id, stream_name, name).await
}
#[delete("/{org_id}/{stream_name}/alerts/{alert_name}")]
async fn delete_alert(path: web::Path<(String, String, String)>) -> impl Responder {
let (org_id, stream_name, name) = path.into_inner();
alerts::delete_alert(org_id, stream_name, name).await
}
src/handler/http/request/dashboards/mod.rs
+30
View File
@@ -0,0 +1,30 @@
use actix_web::{delete, get, post, web, HttpResponse, Responder};
use std::io::Error;
use crate::service::dashboards;
#[post("/{org_id}/dashboards/{name}")]
pub async fn save_dashboard(
path: web::Path<(String, String)>,
details: web::Json<String>,
) -> Result<HttpResponse, Error> {
let (org_id, name) = path.into_inner();
dashboards::save_dashboard(&org_id, &name, &details).await
}
#[get("/{org_id}/dashboards")]
async fn list_dashboards(org_id: web::Path<String>) -> impl Responder {
dashboards::list_dashboards(&org_id.into_inner()).await
}
#[get("/{org_id}/dashboards/{name}")]
async fn get_dashboard(path: web::Path<(String, String)>) -> impl Responder {
let (org_id, name) = path.into_inner();
dashboards::get_dashboard(&org_id, &name).await
}
#[delete("/{org_id}/dashboards/{name}")]
async fn delete_dashboard(path: web::Path<(String, String)>) -> impl Responder {
let (org_id, name) = path.into_inner();
dashboards::delete_dashboard(&org_id, &name).await
}
src/handler/http/request/document/data_ingest.rs
+48
View File
@@ -0,0 +1,48 @@
use crate::service::logs;
use actix_web::{post, web, HttpResponse};
use prometheus::GaugeVec;
use std::io::Error;
#[utoipa::path(
context_path = "/api",
responses(
(status = 200, description = "multi ingest api", body = IngestionResponse , example = json!({"code": 200,"status": [{"name": "olympics","successful": 3,"failed": 0}]}))
)
)]
#[post("/{org_id}/_bulk")]
pub async fn bulk_ingest(
org_id: web::Path<String>,
body: actix_web::web::Bytes,
thread_id: web::Data<usize>,
ingest_stats: web::Data<GaugeVec>,
) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
logs::bulk::ingest(&org_id, body, thread_id, ingest_stats).await
}
#[utoipa::path(
context_path = "/api",
responses(
(status = 200, description = "multi ingest api", body = IngestionResponse , example = json!({"code": 200,"status": [{"name": "olympics","successful": 3,"failed": 0}]}))
)
)]
#[post("/{org_id}/{stream_name}/_multi")]
pub async fn multi_ingest(
path: web::Path<(String, String)>,
body: actix_web::web::Bytes,
thread_id: web::Data<usize>,
ingest_stats: web::Data<GaugeVec>,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name) = path.into_inner();
logs::multi::ingest(&org_id, &stream_name, body, thread_id, ingest_stats).await
}
#[post("/{org_id}/{stream_name}/_json")]
pub async fn json_ingest(
path: web::Path<(String, String)>,
body: actix_web::web::Bytes,
thread_id: web::Data<usize>,
ingest_stats: web::Data<GaugeVec>,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name) = path.into_inner();
logs::json::ingest(&org_id, &stream_name, body, thread_id, ingest_stats).await
}
src/handler/http/request/document/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod data_ingest;
src/handler/http/request/functions/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod udf;
src/handler/http/request/functions/udf.rs
+47
View File
@@ -0,0 +1,47 @@
use actix_web::{delete, get, post, web, HttpResponse, Responder};
use std::io::Error;
use crate::{meta::transform::Transform, service::transform};
#[post("/{org_id}/functions/{name}")]
pub async fn save_function(
path: web::Path<(String, String)>,
js_func: web::Json<Transform>,
) -> Result<HttpResponse, Error> {
let (org_id, name) = path.into_inner();
let transform = js_func.into_inner();
transform::register_transform(org_id, None, name, transform).await
}
#[get("/{org_id}/functions")]
async fn list_functions(org_id: web::Path<String>) -> impl Responder {
transform::list_transform(org_id.into_inner(), None).await
}
#[delete("/{org_id}/functions/{name}")]
async fn delete_function(path: web::Path<(String, String)>) -> impl Responder {
let (org_id, name) = path.into_inner();
transform::delete_transform(org_id, None, name).await
}
#[post("/{org_id}/{stream_name}/functions/{name}")]
pub async fn save_stream_function(
path: web::Path<(String, String, String)>,
js_func: web::Json<Transform>,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name, name) = path.into_inner();
let transform = js_func.into_inner();
transform::register_transform(org_id, Some(stream_name), name, transform).await
}
#[get("/{org_id}/{stream_name}/functions")]
async fn list_stream_function(path: web::Path<(String, String)>) -> impl Responder {
let (org_id, stream_name) = path.into_inner();
transform::list_transform(org_id, Some(stream_name)).await
}
#[delete("/{org_id}/{stream_name}/functions/{name}")]
async fn delete_stream_function(path: web::Path<(String, String, String)>) -> impl Responder {
let (org_id, stream_name, name) = path.into_inner();
transform::delete_transform(org_id, Some(stream_name), name).await
}
src/handler/http/request/health/mod.rs
+51
View File
@@ -0,0 +1,51 @@
use actix_web::{get, HttpResponse};
use ahash::AHashMap;
use serde::Serialize;
use serde_json::{json, Value};
use std::io::Error;
use crate::infra::{cache, cluster, config};
#[derive(Clone, Debug, Serialize)]
struct HealthzResponse {
status: String,
}
#[get("/healthz")]
pub async fn healthz() -> Result<HttpResponse, Error> {
Ok(HttpResponse::Ok().json(HealthzResponse {
status: "ok".to_string(),
}))
}
#[get("/cache/status")]
pub async fn cache_status() -> Result<HttpResponse, Error> {
let mut stats: AHashMap<&str, Value> = AHashMap::new();
stats.insert("LOCAL_NODE_UUID", json!(cluster::LOCAL_NODE_UUID.clone()));
stats.insert(
"LOCAL_NODE_NAME",
json!(&config::CONFIG.common.instance_name),
);
stats.insert("LOCAL_NODE_ROLE", json!(&config::CONFIG.common.node_role));
stats.insert("NODES", json!(cluster::NODES.clone()));
stats.insert(
"STREAM_TRANSFORMS",
json!(config::STREAM_TRANSFORMS.clone()),
);
stats.insert("QUERY_TRANSFORMS", json!(config::QUERY_TRANSFORMS.clone()));
stats.insert("STREAM_STATS", json!({"stream_num": cache::stats::get_stream_stats_len(), "mem_size": cache::stats::get_stream_stats_in_memory_size()}));
let (max_size, cur_size) = cache::file_data::stats();
stats.insert(
"FILE_DATA",
json!({"memory_limit":max_size,"mem_size": cur_size}),
);
let (file_list_num, files_num, mem_size) = cache::file_list::get_file_num().unwrap();
stats.insert(
"FILE_LIST",
json!({"file_list_num":file_list_num, "files_num":files_num, "mem_size":mem_size}),
);
Ok(HttpResponse::Ok().json(stats))
}
src/handler/http/request/mod.rs
+11
View File
@@ -0,0 +1,11 @@
pub mod alerts;
pub mod dashboards;
pub mod document;
pub mod functions;
pub mod health;
pub mod organization;
pub mod prom;
pub mod search;
pub mod stream;
pub mod traces;
pub mod users;
src/handler/http/request/organization/mod.rs
+85
View File
@@ -0,0 +1,85 @@
use actix_web::{get, web, HttpResponse, Result};
use serde::Serialize;
use std::io::Error;
use crate::{handler::http::auth::is_admin_user, infra::config::USERS};
#[derive(Serialize)]
struct Organization {
identifier: String,
label: String,
}
#[derive(Serialize)]
struct User {
first_name: String,
last_name: String,
email: String,
}
#[derive(Serialize)]
struct OrganizationDetails {
id: i64,
identifier: String,
name: String,
user_email: String,
ingest_threshold: i64,
search_threshold: i64,
org_type: String,
user_obj: User,
}
#[derive(Serialize)]
struct OrganizationResponse {
data: [OrganizationDetails; 1],
}
#[get("/organizarions_by_username/{user_name}")]
pub async fn organizarions_by_username(
user_name: web::Path<String>,
) -> Result<HttpResponse, Error> {
let mut orgs = Vec::new();
let user_name = user_name.to_string();
if is_admin_user(&user_name).await {
let obj = Organization {
identifier: "default".to_string(),
label: "Default".to_string(),
};
orgs.push(obj);
} else {
for user in USERS.iter() {
if user.key().contains(format!("/{}", user_name).as_str()) {
orgs.push(Organization {
identifier: user.key().split('/').collect::<Vec<&str>>()[0].to_string(),
label: user.key().split('/').collect::<Vec<&str>>()[0].to_string(),
});
}
}
}
Ok(HttpResponse::Ok().json(orgs))
}
#[get("/{org_id}/organizations")]
pub async fn organizations() -> Result<HttpResponse, Error> {
println!("Inside organizations");
let user_detail = User {
first_name: "admin".to_string(),
last_name: "admin".to_string(),
email: "admin".to_string(),
};
let obj = OrganizationDetails {
id: 1,
identifier: "default".to_string(),
name: "Default".to_string(),
user_email: "admin".to_string(),
ingest_threshold: 9383939382,
search_threshold: 9383939382,
org_type: "default".to_string(),
user_obj: user_detail,
};
let org_response = OrganizationResponse { data: [obj] };
Ok(HttpResponse::Ok().json(org_response))
}
src/handler/http/request/prom/mod.rs
+32
View File
@@ -0,0 +1,32 @@
use crate::{meta, service::metrics};
use actix_web::{http, post, web, HttpRequest, HttpResponse};
use std::io::Error;
#[post("/{org_id}/prometheus/write")]
pub async fn prometheus_write(
org_id: web::Path<String>,
thread_id: web::Data<usize>,
req: HttpRequest,
body: actix_web::web::Bytes,
) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
/* for header in _req.headers() {
println!("{:?}", header)
} */
let content_type = req.headers().get("Content-Type").unwrap().to_str().unwrap();
if content_type.eq("application/x-protobuf") {
metrics::prometheus_write_proto(&org_id, thread_id, body).await
} else {
Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some("Bad Request".to_string()),
)),
)
}
/* else if {
prom::prometheus_write(&org_id, body).await
} else if content_type.eq("text/plain") || content_type.eq("application/openmetrics-text") {
prom::prometheus_write_text(&org_id, body).await
} */
}
src/handler/http/request/search/mod.rs
+173
View File
@@ -0,0 +1,173 @@
use actix_web::http::{self, StatusCode};
use actix_web::{get, post, web, HttpRequest, HttpResponse};
use ahash::AHashMap as HashMap;
use chrono::Duration;
use std::io::Error;
use std::sync::Mutex;
use crate::common::http::get_stream_type_from_request;
use crate::infra::config::{CONFIG, LOCKER};
use crate::meta;
use crate::meta::http::HttpResponse as MetaHttpResponse;
use crate::meta::StreamType;
use crate::service::search as SearchService;
#[utoipa::path(
context_path = "/api",
responses(
(status = 200, description = "search api", body = Response , example = json!({"code": 200,"status": [{"name": "olympics","successful": 3,"failed": 0}]}))
)
)]
#[post("/{org_id}/_search")]
pub async fn search(
org_id: web::Path<String>,
in_req: HttpRequest,
body: actix_web::web::Bytes,
) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
let query = web::Query::<HashMap<String, String>>::from_query(in_req.query_string()).unwrap();
let stream_type = match get_stream_type_from_request(&query) {
Ok(v) => v.unwrap_or(StreamType::Logs),
Err(e) => {
return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some(e.to_string()),
)))
}
};
let req: meta::search::Request = serde_json::from_slice(&body)?;
// get a local search queue lock
let locker = LOCKER
.entry("search/local_queue".to_string())
.or_insert(Mutex::new(true));
let _locker = locker.lock();
// do search
match SearchService::search(&org_id, stream_type, &req).await {
Ok(res) => Ok(HttpResponse::Ok().json(res)),
Err(err) => {
log::error!("search error: {:?}", err);
Ok(
HttpResponse::InternalServerError().json(meta::http::HttpResponse::error(
StatusCode::INTERNAL_SERVER_ERROR.into(),
Some(err.to_string()),
)),
)
}
}
}
#[utoipa::path(
context_path = "/api",
responses(
(status = 200, description = "search around api", body = Response , example = json!({"code": 200,"status": [{"name": "olympics","successful": 3,"failed": 0}]}))
)
)]
#[get("/{org_id}/{stream_name}/_around")]
pub async fn around(
path: web::Path<(String, String)>,
in_req: HttpRequest,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name) = path.into_inner();
let query = web::Query::<HashMap<String, String>>::from_query(in_req.query_string()).unwrap();
let stream_type = match get_stream_type_from_request(&query) {
Ok(v) => v.unwrap_or(StreamType::Logs),
Err(e) => {
return Ok(HttpResponse::BadRequest().json(MetaHttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some(e.to_string()),
)))
}
};
let around_key = match query.get("key") {
Some(v) => v.parse::<i64>().unwrap_or(0),
None => 0,
};
let around_size = match query.get("size") {
Some(v) => v.parse::<usize>().unwrap_or(0),
None => 0,
};
// get a local search queue lock
let locker = LOCKER
.entry("search/local_queue".to_string())
.or_insert(Mutex::new(true));
let _locker = locker.lock();
// search forward
let req = meta::search::Request {
query: meta::search::Query {
sql: format!(
"SELECT * FROM {} ORDER BY {} DESC",
stream_name, CONFIG.common.time_stamp_col
),
from: 0,
size: around_size / 2,
start_time: around_key - Duration::seconds(300).num_microseconds().unwrap(),
end_time: around_key,
sql_mode: "context".to_string(),
track_total_hits: false,
},
aggs: ahash::AHashMap::new(),
};
let resp_forward = match SearchService::search(&org_id, stream_type, &req).await {
Ok(res) => res,
Err(err) => {
log::error!("search error: {:?}", err);
return Ok(
HttpResponse::InternalServerError().json(meta::http::HttpResponse::error(
StatusCode::INTERNAL_SERVER_ERROR.into(),
Some(err.to_string()),
)),
);
}
};
// search backward
let req = meta::search::Request {
query: meta::search::Query {
sql: format!(
"SELECT * FROM {} ORDER BY {} ASC",
stream_name, CONFIG.common.time_stamp_col
),
from: 0,
size: around_size / 2,
start_time: around_key,
end_time: around_key + Duration::seconds(300).num_microseconds().unwrap(),
sql_mode: "context".to_string(),
track_total_hits: false,
},
aggs: ahash::AHashMap::new(),
};
let resp_backward = match SearchService::search(&org_id, stream_type, &req).await {
Ok(res) => res,
Err(err) => {
log::error!("search error: {:?}", err);
return Ok(
HttpResponse::InternalServerError().json(meta::http::HttpResponse::error(
StatusCode::INTERNAL_SERVER_ERROR.into(),
Some(err.to_string()),
)),
);
}
};
// merge
let mut resp = meta::search::Response::default();
let hits_num = resp_backward.hits.len();
for i in 0..hits_num {
resp.hits
.push(resp_backward.hits[hits_num - 1 - i].to_owned());
}
let hits_num = resp_forward.hits.len();
for i in 0..hits_num {
resp.hits.push(resp_forward.hits[i].to_owned());
}
resp.total = resp.hits.len();
resp.took = resp_forward.took + resp_backward.took;
resp.scan_size = resp_forward.scan_size + resp_backward.scan_size;
Ok(HttpResponse::Ok().json(resp))
}
src/handler/http/request/stream/mod.rs
+164
View File
@@ -0,0 +1,164 @@
use crate::common::http::get_stream_type_from_request;
use crate::meta::{self, stream::StreamSettings, StreamType};
use crate::service::stream;
use actix_web::{
get,
http::{self, header::ContentType},
post, web, HttpRequest, HttpResponse, Responder,
};
use ahash::AHashMap;
use std::io::Error;
use std::io::ErrorKind;
#[get("/{org_id}/{stream_name}/schema")]
async fn stream_schema(
path: web::Path<(String, String)>,
req: HttpRequest,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name) = path.into_inner();
let query = web::Query::<AHashMap<String, String>>::from_query(req.query_string()).unwrap();
let stream_type = match get_stream_type_from_request(&query) {
Ok(v) => v,
Err(e) => {
return Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some(e.to_string()),
)),
)
}
};
match stream_type {
Some(stream_type_loc) => {
stream::get_stream(org_id.as_str(), stream_name.as_str(), stream_type_loc).await
}
/* Some(StreamType::Logs) => {
stream::get_stream(org_id.as_str(), stream_name.as_str(), StreamType::Logs).await
}
Some(StreamType::Metrics) => {
stream::get_stream(org_id.as_str(), stream_name.as_str(), StreamType::Metrics).await
}
Some(StreamType::Traces) => {
stream::get_stream(org_id.as_str(), stream_name.as_str(), StreamType::Traces).await
} */
None => stream::get_stream(org_id.as_str(), stream_name.as_str(), StreamType::Logs).await,
}
}
#[post("/{org_id}/{stream_name}/settings")]
async fn stream_settings(
path: web::Path<(String, String)>,
settings: web::Json<StreamSettings>,
req: HttpRequest,
) -> Result<HttpResponse, Error> {
let (org_id, stream_name) = path.into_inner();
let query = web::Query::<AHashMap<String, String>>::from_query(req.query_string()).unwrap();
let stream_type = match get_stream_type_from_request(&query) {
Ok(v) => v,
Err(e) => {
return Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some(e.to_string()),
)),
)
}
};
match stream_type {
Some(steam_type_loc) => {
stream::save_stream_settings(
org_id.as_str(),
stream_name.as_str(),
steam_type_loc,
settings.into_inner(),
)
.await
}
None => {
stream::save_stream_settings(
org_id.as_str(),
stream_name.as_str(),
StreamType::Logs,
settings.into_inner(),
)
.await
}
}
}
#[utoipa::path(
context_path = "/api",
responses(
(status = 200, description = "List streams", body = ListStream)
)
)]
#[get("/{org_id}/streams")]
async fn list_streams(org_id: web::Path<String>, req: HttpRequest) -> impl Responder {
let query = web::Query::<AHashMap<String, String>>::from_query(req.query_string()).unwrap();
let stream_type = match get_stream_type_from_request(&query) {
Ok(v) => v,
Err(e) => {
return Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some(e.to_string()),
)),
)
}
};
let fetch_schema = match query.get("fetchSchema") {
Some(s) => match s.to_lowercase().as_str() {
"true" => true,
"false" => false,
_ => {
return Err(Error::new(
ErrorKind::Other,
" 'fetchSchema' query param with value 'true' or 'false' allowed",
));
}
},
None => false,
};
match stream_type {
/* Some(StreamType::Logs) => {
stream::list_streams(org_id.as_str(), Some(StreamType::Logs), fetch_schema).await
}
Some(StreamType::Metrics) => {
stream::list_streams(org_id.as_str(), Some(StreamType::Metrics), fetch_schema).await
}
Some(StreamType::Traces) => {
stream::list_streams(org_id.as_str(), Some(StreamType::Traces), fetch_schema).await
} */
Some(stream_type_loc) => {
stream::list_streams(org_id.as_str(), Some(stream_type_loc), fetch_schema).await
}
None => stream::list_streams(org_id.as_str(), None, fetch_schema).await,
}
}
#[get("/{org_id}/")]
async fn org_index(_org_id: web::Path<String>, req: HttpRequest) -> impl Responder {
// eg.1: User-Agent:[elastic-transport-ruby/8.0.1 (RUBY_VERSION: 3.1.2; linux x86_64; Faraday v1.10.0)]
let mut version = "0.0.0";
let user_agent = match req.headers().get("User-Agent") {
Some(user_agent) => user_agent.to_str().unwrap(),
None => "",
};
if user_agent.contains("elastic") {
let re = regex::Regex::new(r"(\d+\.\d+\.\d+)").unwrap();
version = match re.captures(user_agent) {
Some(caps) => caps.get(1).unwrap().as_str(),
None => "8.1.0",
};
}
let es_info = r##"{"name":"Elasticsearch","cluster_name":"N/A","cluster_uuid":"N/A","version":{"number":"0.0.0","build_flavor":"default","build_hash":"0","build_date":"0","build_snapshot":false,"lucene_version":"N/A","minimum_wire_version":"N/A","minimum_index_compatibility":"N/A"},"tagline":"You Know, for Search"}"##;
let es_info = es_info.replace("0.0.0", version);
HttpResponse::Ok()
.content_type(ContentType::json())
.insert_header(("X-Elastic-Product", "Elasticsearch"))
.body(es_info)
}
src/handler/http/request/traces/mod.rs
+30
View File
@@ -0,0 +1,30 @@
use crate::{meta, service::traces::otlp_http};
use actix_web::{http, post, web, HttpRequest, HttpResponse};
use std::io::Error;
pub const CONTENT_TYPE_JSON: &str = "application/json";
pub const CONTENT_TYPE_PROTO: &str = "application/x-protobuf";
#[post("/{org_id}/traces")]
pub async fn traces_write(
org_id: web::Path<String>,
thread_id: web::Data<usize>,
req: HttpRequest,
body: actix_web::web::Bytes,
) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
let content_type = req.headers().get("Content-Type").unwrap().to_str().unwrap();
if content_type.eq(CONTENT_TYPE_PROTO) {
otlp_http::traces_proto(&org_id, thread_id.into_inner(), body).await
} else if content_type.eq(CONTENT_TYPE_JSON) {
otlp_http::traces_json(&org_id, thread_id.into_inner(), body).await
} else {
Ok(
HttpResponse::BadRequest().json(meta::http::HttpResponse::error(
http::StatusCode::BAD_REQUEST.into(),
Some("Bad Request".to_string()),
)),
)
}
}
src/handler/http/request/users/mod.rs
+68
View File
@@ -0,0 +1,68 @@
use crate::handler::http::auth::validate_credentials;
use crate::infra::config::CONFIG;
use crate::meta::user::Role;
use crate::meta::user::SignInUser;
use crate::{meta::user::User, service::users};
use actix_web::{delete, get, post, web, HttpResponse};
use ahash::AHashMap;
use serde_json::Value;
use std::io::Error;
#[post("/{org_id}/users")]
pub async fn post_user(
org_id: web::Path<String>,
user: web::Json<User>,
) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
let user = user.into_inner();
users::post_user(&org_id, user).await
}
#[get("/{org_id}/users")]
pub async fn list_users(org_id: web::Path<String>) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
users::list_user(&org_id).await
}
#[delete("/{org_id}/users/{user_name}")]
pub async fn delete_user(path: web::Path<(String, String)>) -> Result<HttpResponse, Error> {
let (org_id, name) = path.into_inner();
users::delete_user(&org_id, &name).await
}
#[post("/{org_id}/authentication")]
pub async fn authentication(
org_id: web::Path<String>,
user: web::Json<SignInUser>,
) -> Result<HttpResponse, Error> {
let org_id = org_id.into_inner();
let mut ret: AHashMap<&str, Value> = AHashMap::new();
match validate_credentials(
&user.name,
&user.password,
&format!("{}/authentication", &org_id),
)
.await
{
Ok(v) => {
if v {
if user.name == CONFIG.auth.username {
ret.insert("role", Value::String(format!("{:?}", &Role::Admin)));
} else if let Some(user) = users::get_user(Some(&org_id), &user.name).await {
// println!("{:?}", format!("{:?}", user.role));
ret.insert("role", Value::String(format!("{:?}", user.role)));
}
ret.insert("status", Value::Bool(true));
} else {
ret.insert("status", Value::Bool(false));
ret.insert("message", Value::String("Invalid credentials".to_string()));
}
}
Err(_e) => {
ret.insert("status", Value::Bool(false));
ret.insert("message", Value::String("Invalid credentials".to_string()));
}
};
Ok(HttpResponse::Ok().json(ret))
}
src/handler/http/router/mod.rs
+113
View File
@@ -0,0 +1,113 @@
use actix_cors::Cors;
use actix_web::{http::header, web};
use actix_web_httpauth::middleware::HttpAuthentication;
use std::sync::Arc;
use utoipa::OpenApi;
use utoipa_swagger_ui::{SwaggerUi, Url};
use super::auth::validator;
use super::request::alerts::*;
use super::request::dashboards::*;
use super::request::document::data_ingest::*;
use super::request::functions::udf::*;
use super::request::health::{cache_status, healthz};
use super::request::organization::*;
use super::request::prom::*;
use super::request::search;
use super::request::stream::*;
use super::request::traces::*;
use super::request::users::*;
use crate::infra::config::CONFIG;
use crate::meta::ingestion::{IngestionResponse, StreamStatus};
use crate::meta::stream;
use crate::meta::StreamType;
pub mod ui;
pub fn get_routes(cfg: &mut web::ServiceConfig) {
#[derive(OpenApi)]
#[openapi(
paths(list_streams, bulk_ingest, multi_ingest),
components(schemas(
stream::ListStream,
stream::Stream,
StreamType,
stream::Stats,
stream::StreamProperty,
IngestionResponse,
StreamStatus,
//SearchResponse
)),
tags(
(name = "Ingestion", description = "Todo management endpoints.")
)
)]
struct ApiDoc;
let auth = HttpAuthentication::basic(validator);
cfg.service(healthz);
let cors = Cors::default()
.send_wildcard()
.allowed_methods(vec!["HEAD", "GET", "POST", "PUT", "OPTIONS", "DELETE"])
.allowed_headers(vec![
header::AUTHORIZATION,
header::ACCEPT,
header::CONTENT_TYPE,
])
.allow_any_origin()
.max_age(3600);
let cors = Arc::new(cors);
cfg.service(
web::scope("/auth")
.wrap(cors.clone())
.service(organizarions_by_username)
.service(authentication),
);
cfg.service(
web::scope("/api")
.wrap(auth)
.wrap(cors)
.service(cache_status)
.service(bulk_ingest)
.service(multi_ingest)
.service(json_ingest)
.service(search::search)
.service(search::around)
.service(stream_schema)
.service(stream_settings)
.service(list_streams)
.service(org_index)
.service(save_function)
.service(list_functions)
.service(delete_function)
.service(save_stream_function)
.service(list_stream_function)
.service(delete_stream_function)
.service(list_users)
.service(delete_user)
.service(post_user)
.service(prometheus_write)
.service(save_dashboard)
.service(get_dashboard)
.service(list_dashboards)
.service(delete_dashboard)
.service(traces_write)
.service(organizations)
.service(save_alert)
.service(get_alert)
.service(list_alerts)
.service(list_stream_alerts)
.service(delete_alert),
);
cfg.service(SwaggerUi::new("/swagger-ui/{_:.*}").urls(vec![(
Url::new("api", "/api-doc/openapi1.json"),
ApiDoc::openapi(),
)]));
if CONFIG.common.ui_enabled {
cfg.service(ui::serve);
}
}
src/handler/http/router/ui.rs
+38
View File
@@ -0,0 +1,38 @@
use actix_web::{route, web};
use actix_web_rust_embed_responder::{
Compress, EmbedResponse, EmbedableFileResponse, IntoResponse,
};
use rust_embed_for_web::RustEmbed;
#[derive(RustEmbed)]
#[folder = "web/dist/"]
struct WebAssets;
pub const UI_PAGES: [&str; 9] = [
"login",
"about",
"users",
"logs",
"logstream",
"ingestion",
"ingestion/fluentbit",
"ingestion/fluentd",
"ingestion/vector",
];
#[route("/{path:.*}", method = "GET", method = "HEAD")]
pub async fn serve(path: web::Path<String>) -> EmbedResponse<EmbedableFileResponse> {
let mut path = if path.is_empty() {
"index.html"
} else {
path.as_str()
};
if UI_PAGES.contains(&path) {
path = "index.html";
}
WebAssets::get(path)
.into_response()
.use_compression(Compress::Never)
}
src/handler/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod grpc;
pub mod http;
src/infra/cache/file_data.rs
Vendored
+124
View File
@@ -0,0 +1,124 @@
use bytes::Bytes;
use lru::LruCache;
use std::cmp::max;
use std::sync::RwLock;
use crate::infra::config::CONFIG;
use crate::infra::storage;
lazy_static! {
pub static ref FILES: RwLock<FileData> = RwLock::new(FileData::new());
}
pub struct FileData {
max_size: usize,
cur_size: usize,
data: LruCache<String, Bytes>,
}
impl Default for FileData {
fn default() -> Self {
Self::new()
}
}
impl FileData {
pub fn new() -> FileData {
FileData {
max_size: CONFIG.memory_cache.max_size,
cur_size: 0,
data: LruCache::unbounded(),
}
}
pub fn get(&mut self, file: &str) -> Result<Bytes, anyhow::Error> {
let data = self.data.get(file);
if data.is_none() {
return Err(anyhow::anyhow!("file not in cache"));
}
Ok(data.unwrap().to_owned())
}
pub fn set(&mut self, file: &str, data: Bytes) -> Result<(), anyhow::Error> {
let data_size = data.len();
if self.cur_size + data_size >= self.max_size {
log::info!(
"[TRACE] File cache is full {}/{}, can't cache {} bytes",
self.cur_size,
self.max_size,
data_size
);
// cache is full, need release some space
let need_release_size = max(CONFIG.memory_cache.release_size, data_size * 100);
let mut release_size = 0;
loop {
let item = self.data.pop_lru();
if item.is_none() {
break;
}
let (_, v) = item.unwrap();
release_size += v.len();
if release_size >= need_release_size {
break;
}
}
self.cur_size -= release_size;
}
self.cur_size += data_size;
self.data.put(file.to_string(), data);
Ok(())
}
pub fn size(&self) -> (usize, usize) {
(self.max_size, self.cur_size)
}
}
#[inline]
pub fn get(file: &str) -> Result<Bytes, anyhow::Error> {
if !CONFIG.memory_cache.enabled {
return Err(anyhow::anyhow!("memory cache is disabled"));
}
let mut files = FILES.write().unwrap();
files.get(file)
}
#[inline]
pub fn exist(file: &str) -> Result<bool, anyhow::Error> {
let mut files = FILES.write().unwrap();
match files.get(file) {
Ok(_) => Ok(true),
Err(_) => Ok(false),
}
}
#[inline]
pub fn set(file: &str, data: Bytes) -> Result<(), anyhow::Error> {
if !CONFIG.memory_cache.enabled {
return Ok(());
}
let mut files = FILES.write().unwrap();
files.set(file, data)
}
#[inline]
pub fn stats() -> (usize, usize) {
let files = FILES.read().unwrap();
files.size()
}
#[inline]
#[tracing::instrument(name = "infra:cache:file_data:download")]
pub async fn download(file: &str) -> Result<Bytes, anyhow::Error> {
let store = &storage::DEFAULT;
let data = store.get(file).await?;
if let Err(e) = set(file, data.clone()) {
return Err(anyhow::anyhow!(
"set file {} to memory cache failed: {}",
file,
e
));
};
Ok(data)
}
src/infra/cache/file_list.rs
Vendored
+524
View File
@@ -0,0 +1,524 @@
use chrono::{Datelike, TimeZone, Timelike, Utc};
use dashmap::DashMap;
use crate::meta::{common::FileMeta, StreamType};
lazy_static! {
pub static ref FILES: DashMap<String, Box<FileList>> = DashMap::new();
}
const FILE_LIST_MEM_SIZE: usize = std::mem::size_of::<Box<FileList>>();
const FILE_META_MEM_SIZE: usize = std::mem::size_of::<Option<FileMeta>>();
#[derive(Debug)]
pub struct FileList {
sub: Option<DashMap<String, Box<FileList>>>,
files: Option<DashMap<String, FileMeta>>,
size: usize,
}
pub fn set_file_to_cache(
key: &str,
val: Option<FileMeta>,
delete: bool,
) -> Result<(), anyhow::Error> {
// eg: files/default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet
let columns = key.split('/').collect::<Vec<&str>>();
if columns.len() < 8 {
return Err(anyhow::anyhow!(
"[TRACE] [set_file_to_cache] Invalid file path: {}",
key
));
}
let _ = columns[0].to_string();
let org_id = columns[1].to_string();
let data_type = columns[2].to_string();
let stream_name = columns[3].to_string();
let year = columns[4].to_string();
let month = columns[5].to_string();
let day = columns[6].to_string();
let hour = columns[7].to_string();
let file_name = columns[8..].join("/");
let mut org_cache = FILES.entry(org_id).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: None,
size: 0,
}));
let cache = org_cache.sub.as_mut().unwrap();
let mut type_cache = cache.entry(data_type).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: None,
size: 0,
}));
let cache = type_cache.sub.as_mut().unwrap();
let mut stream_cache = cache.entry(stream_name).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: None,
size: 0,
}));
let cache = stream_cache.sub.as_mut().unwrap();
let mut year_cache = cache.entry(year).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: None,
size: 0,
}));
let cache = year_cache.sub.as_mut().unwrap();
let mut month_cache = cache.entry(month).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: None,
size: 0,
}));
let cache = month_cache.sub.as_mut().unwrap();
let mut day_cache = cache.entry(day).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: None,
size: 0,
}));
let cache = day_cache.sub.as_mut().unwrap();
let mut hour_cache = cache.entry(hour).or_insert(Box::new(FileList {
sub: Some(DashMap::new()),
files: Some(DashMap::with_capacity(8)),
size: 0,
}));
let cache = hour_cache.files.as_mut().unwrap();
if delete {
cache.remove(&file_name);
hour_cache.size -= key.len() + FILE_META_MEM_SIZE;
} else {
cache.insert(file_name, val.unwrap());
hour_cache.size += key.len() + FILE_META_MEM_SIZE;
}
Ok(())
}
pub fn get_file_from_cache(key: &str) -> Result<FileMeta, anyhow::Error> {
// eg: files/default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet
let columns = key.split('/').collect::<Vec<&str>>();
let _ = columns[0].to_string();
let org_id = columns[1].to_string();
let data_type = columns[2].to_string();
let stream_name = columns[3].to_string();
let year = columns[4].to_string();
if year.len() != 4 {
return Err(anyhow::anyhow!(
"file_list: year format error, value: {}",
year
));
}
let month = columns[5].to_string();
let day = columns[6].to_string();
let hour = columns[7].to_string();
let file_name = columns[8..].join("/");
let org_cache = match FILES.get(&org_id) {
Some(v) => v,
None => return Err(anyhow::anyhow!("file_list: org_id not found, key: {}", key)),
};
let cache = org_cache.sub.as_ref().unwrap();
let type_cache = match cache.get(&data_type) {
Some(v) => v,
None => {
return Err(anyhow::anyhow!(
"file_list: data_type not found, key: {}",
key
))
}
};
let cache = type_cache.sub.as_ref().unwrap();
let stream_cache = match cache.get(&stream_name) {
Some(v) => v,
None => {
return Err(anyhow::anyhow!(
"file_list: stream_name not found, key: {}",
key
))
}
};
let cache = stream_cache.sub.as_ref().unwrap();
let year_cache = match cache.get(&year) {
Some(v) => v,
None => return Err(anyhow::anyhow!("file_list: year not found, key: {}", key)),
};
let cache = year_cache.sub.as_ref().unwrap();
let month_cache = match cache.get(&month) {
Some(v) => v,
None => return Err(anyhow::anyhow!("file_list: month not found, key: {}", key)),
};
let cache = month_cache.sub.as_ref().unwrap();
let day_cache = match cache.get(&day) {
Some(v) => v,
None => return Err(anyhow::anyhow!("file_list: day not found, key: {}", key)),
};
let cache = day_cache.sub.as_ref().unwrap();
let hour_cache = match cache.get(&hour) {
Some(v) => v,
None => return Err(anyhow::anyhow!("file_list: hour not found, key: {}", key)),
};
let cache = hour_cache.files.as_ref().unwrap();
let resp = match cache.get(&file_name) {
Some(v) => v,
None => {
return Err(anyhow::anyhow!(
"file_list: file_name not found, key: {}",
key
))
}
};
// println!("get_file_meta_from_cache: {}", key);
Ok(*resp)
}
async fn scan_prefix(
org_id: &str,
stream_name: &str,
stream_type: StreamType,
prefix: &str,
) -> Result<Vec<String>, anyhow::Error> {
let prefix = if prefix.starts_with("files/") {
prefix.trim_start_matches("files/").trim_end_matches('/')
} else {
prefix.trim_end_matches('/')
};
let stream_type = stream_type.to_string();
let columns = prefix.split('/').collect::<Vec<&str>>();
let column_num = columns.len();
let mut year = "";
let mut month = "";
let mut day = "";
let mut hour = "";
if column_num >= 1 {
year = columns[0];
}
if column_num >= 2 {
month = columns[1];
}
if column_num >= 3 {
day = columns[2];
}
if column_num >= 4 {
hour = columns[3];
}
let mut resp = Vec::new();
let org_cache = match FILES.get(org_id) {
Some(v) => v,
None => return Ok(resp),
};
let cache = org_cache.sub.as_ref().unwrap();
let type_cache = match cache.get(&stream_type) {
Some(v) => v,
None => return Ok(resp),
};
let cache = type_cache.sub.as_ref().unwrap();
let stream_cache = match cache.get(stream_name) {
Some(v) => v,
None => return Ok(resp),
};
if year.is_empty() {
for cache in stream_cache.sub.as_ref().unwrap().iter() {
let year = cache.key();
for cache in cache.sub.as_ref().unwrap().iter() {
let month = cache.key();
for cache in cache.sub.as_ref().unwrap().iter() {
let day = cache.key();
for cache in cache.sub.as_ref().unwrap().iter() {
let hour = cache.key();
for item in cache.files.as_ref().unwrap().iter() {
resp.push(format!(
"files/{}/{}/{}/{}/{}/{}/{}/{}",
org_id,
stream_type,
stream_name,
year,
month,
day,
hour,
item.key()
));
}
}
}
}
}
return Ok(resp);
}
let cache = stream_cache.sub.as_ref().unwrap();
let year_cache = match cache.get(year) {
Some(v) => v,
None => return Ok(resp),
};
if month.is_empty() {
for cache in year_cache.sub.as_ref().unwrap().iter() {
let month = cache.key();
for cache in cache.sub.as_ref().unwrap().iter() {
let day = cache.key();
for cache in cache.sub.as_ref().unwrap().iter() {
let hour = cache.key();
for item in cache.files.as_ref().unwrap().iter() {
resp.push(format!(
"files/{}/{}/{}/{}/{}/{}/{}/{}",
org_id,
stream_type,
stream_name,
year,
month,
day,
hour,
item.key()
));
}
}
}
}
return Ok(resp);
}
let cache = year_cache.sub.as_ref().unwrap();
let month_cache = match cache.get(month) {
Some(v) => v,
None => return Ok(resp),
};
if day.is_empty() {
for cache in month_cache.sub.as_ref().unwrap().iter() {
let day = cache.key();
for cache in cache.sub.as_ref().unwrap().iter() {
let hour = cache.key();
for item in cache.files.as_ref().unwrap().iter() {
resp.push(format!(
"files/{}/{}/{}/{}/{}/{}/{}/{}",
org_id,
stream_type,
stream_name,
year,
month,
day,
hour,
item.key()
));
}
}
}
return Ok(resp);
}
let cache = month_cache.sub.as_ref().unwrap();
let day_cache = match cache.get(day) {
Some(v) => v,
None => return Ok(resp),
};
if hour.is_empty() {
for cache in day_cache.sub.as_ref().unwrap().iter() {
let hour = cache.key();
for item in cache.files.as_ref().unwrap().iter() {
resp.push(format!(
"files/{}/{}/{}/{}/{}/{}/{}/{}",
org_id,
stream_type,
stream_name,
year,
month,
day,
hour,
item.key()
));
}
}
return Ok(resp);
}
let cache = day_cache.sub.as_ref().unwrap();
let hour_cache = match cache.get(hour) {
Some(v) => v,
None => return Ok(resp),
};
for item in hour_cache.files.as_ref().unwrap().iter() {
resp.push(format!(
"files/{}/{}/{}/{}/{}/{}/{}/{}",
org_id,
stream_type,
stream_name,
year,
month,
day,
hour,
item.key()
));
}
Ok(resp)
}
#[tracing::instrument(name = "cache:file_list:get_file_list")]
pub async fn get_file_list(
org_id: &str,
stream_name: &str,
stream_type: StreamType,
time_min: i64,
time_max: i64,
) -> Result<Vec<String>, anyhow::Error> {
let mut files = Vec::new();
let mut keys = Vec::new();
let mut key = "".to_string();
if time_min > 0 && time_max > 0 {
let time_min = Utc.timestamp_nanos(time_min * 1000);
let time_max = Utc.timestamp_nanos(time_max * 1000);
if time_min.year() == time_max.year() {
key.push_str(&time_min.format("%Y").to_string());
key.push('/');
if time_min.month() == time_max.month() {
key.push_str(&time_min.format("%m").to_string());
key.push('/');
if time_min.day() == time_max.day() {
key.push_str(&time_min.format("%d").to_string());
key.push('/');
if time_min.hour() == time_max.hour() {
key.push_str(&time_min.format("%H").to_string());
key.push('/');
keys.push(key);
} else {
for i in time_min.hour()..(time_max.hour() + 1) {
let mut k = key.clone();
if i < 10 {
k.push_str(&format!("0{}", i))
} else {
k.push_str(&format!("{}", i))
}
k.push('/');
keys.push(k);
}
}
} else {
for i in time_min.day()..(time_max.day() + 1) {
let mut k = key.clone();
if i < 10 {
k.push_str(&format!("0{}", i))
} else {
k.push_str(&format!("{}", i))
}
k.push('/');
keys.push(k);
}
}
} else {
for i in time_min.month()..(time_max.month() + 1) {
let mut k = key.clone();
if i < 10 {
k.push_str(&format!("0{}", i))
} else {
k.push_str(&format!("{}", i))
}
k.push('/');
keys.push(k);
}
}
} else {
for i in time_min.year()..(time_max.year() + 1) {
let mut k = key.clone();
if i < 10 {
k.push_str(&format!("0{}", i))
} else {
k.push_str(&format!("{}", i))
}
k.push('/');
keys.push(k);
}
}
} else {
keys.push(key);
}
for key in keys {
let resp = scan_prefix(org_id, stream_name, stream_type, &key)
.await
.unwrap();
files.extend(resp);
}
Ok(files)
}
/// Get file list num and total keys length
pub fn get_file_num() -> Result<(usize, usize, usize), anyhow::Error> {
let mut files_num = 0;
let mut mem_size = 0;
let mut file_list_num = 0;
for cache in FILES.iter() {
let org_cache = cache.sub.as_ref().unwrap();
for cache in org_cache.iter() {
let type_cache = cache.sub.as_ref().unwrap();
for cache in type_cache.iter() {
let stream_cache = cache.sub.as_ref().unwrap();
for cache in stream_cache.iter() {
let year_cache = cache.sub.as_ref().unwrap();
for cache in year_cache.iter() {
let month_cache = cache.sub.as_ref().unwrap();
for cache in month_cache.iter() {
let day_cache = cache.sub.as_ref().unwrap();
for cache in day_cache.iter() {
files_num += cache.files.as_ref().unwrap().len();
mem_size += cache.size;
file_list_num += 1;
}
}
}
}
}
}
}
mem_size += file_list_num * FILE_LIST_MEM_SIZE;
Ok((file_list_num, files_num, mem_size))
}
pub fn get_all_organization() -> Result<Vec<String>, anyhow::Error> {
let mut orgs = Vec::new();
for cache in FILES.iter() {
orgs.push(cache.key().to_string());
}
Ok(orgs)
}
pub fn get_all_stream(org_id: &str, stream_type: StreamType) -> Result<Vec<String>, anyhow::Error> {
let mut streams = Vec::new();
let org_cache = FILES.get(org_id).unwrap();
let org_cache = org_cache.sub.as_ref().unwrap();
let type_cache = match org_cache.get(&stream_type.to_string()) {
Some(cache) => cache,
None => return Ok(streams),
};
let type_cache = type_cache.sub.as_ref().unwrap();
for cache in type_cache.iter() {
streams.push(cache.key().to_string());
}
Ok(streams)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_get_file_from_cache() {
let meta = FileMeta {
min_ts: 100,
max_ts: 200,
records: 10000,
original_size: 1024,
compressed_size: 1,
};
let _ret = set_file_to_cache(
"files/default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet",
Some(meta),
false,
)
.unwrap();
let res = get_file_from_cache(
"files/default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet",
);
assert_eq!(res.unwrap().records, meta.records);
}
}
src/infra/cache/mod.rs
Vendored
+4
View File
@@ -0,0 +1,4 @@
pub mod file_data;
pub mod file_list;
pub mod stats;
pub mod tmpfs;
src/infra/cache/stats.rs
Vendored
+90
View File
@@ -0,0 +1,90 @@
use dashmap::DashMap;
use crate::meta::common::FileMeta;
use crate::meta::stream::Stats;
lazy_static! {
static ref STATS: DashMap<String, Stats> = DashMap::with_capacity(2);
}
const STREAM_STATS_MEM_SIZE: usize = std::mem::size_of::<Stats>();
pub fn get_stats() -> DashMap<String, Stats> {
STATS.clone()
}
pub fn get_stream_stats(org_id: &str, stream_name: &str, stream_type: &str) -> Option<Stats> {
let key = format!("{}/{}/{}", org_id, stream_type, stream_name);
STATS.get(&key).map(|v| *v.value())
}
pub fn set_stream_stats(org_id: &str, stream_name: &str, stream_type: &str, val: Stats) {
let key = format!("{}/{}/{}", org_id, stream_type, stream_name);
STATS.insert(key, val);
}
pub fn incr_stream_stats(key: &str, val: FileMeta) -> Result<(), anyhow::Error> {
// eg: files/default/olympics/2022/10/03/10/6982652937134804993_1.parquet
let columns = key.split('/').collect::<Vec<&str>>();
if columns.len() < 8 {
return Err(anyhow::anyhow!(
"[TRACE] [incr_stream_stats] Invalid file path: {}",
key
));
}
let _ = columns[0].to_string();
let org_id = columns[1].to_string();
let stream_type = columns[2].to_string();
let stream_name = columns[3].to_string();
let key = format!("{}/{}/{}", org_id, stream_type, stream_name);
let mut stats = STATS.entry(key).or_default();
if val.records == 0 {
return Ok(());
}
if stats.doc_time_min > val.min_ts || stats.doc_time_min == 0 {
stats.doc_time_min = val.min_ts;
}
if stats.doc_time_max < val.max_ts {
stats.doc_time_max = val.max_ts;
}
stats.doc_num += val.records;
stats.file_num += 1;
stats.storage_size += val.original_size as f64;
stats.compressed_size += val.compressed_size as f64;
Ok(())
}
pub fn decr_stream_stats(key: &str, val: FileMeta) -> Result<(), anyhow::Error> {
// eg: files/default/olympics/2022/10/03/10/6982652937134804993_1.parquet
let columns = key.split('/').collect::<Vec<&str>>();
if columns.len() < 8 {
return Err(anyhow::anyhow!(
"[TRACE] [decr_stream_stats] Invalid file path: {}",
key
));
}
let _ = columns[0].to_string();
let org_id = columns[1].to_string();
let stream_type = columns[2].to_string();
let stream_name = columns[3].to_string();
let key = format!("{}/{}/{}", org_id, stream_type, stream_name);
let mut stats = STATS.entry(key).or_default();
stats.doc_num -= val.records;
stats.file_num -= 1;
stats.storage_size -= val.original_size as f64;
stats.compressed_size -= val.compressed_size as f64;
Ok(())
}
pub fn get_stream_stats_len() -> usize {
STATS.len()
}
pub fn get_stream_stats_in_memory_size() -> usize {
STATS
.iter()
.map(|v| v.key().len() + STREAM_STATS_MEM_SIZE)
.sum()
}
src/infra/cache/tmpfs.rs
Vendored
+76
View File
@@ -0,0 +1,76 @@
use parking_lot::RwLock;
use rsfs::mem;
use rsfs::*;
use std::io::Error;
use std::io::Read;
use std::path::Path;
lazy_static! {
pub static ref FS: RwLock<mem::FS> = RwLock::new(mem::FS::new());
}
pub fn open_file<P: AsRef<Path>>(path: P) -> Result<mem::File, Error> {
FS.write().open_file(path)
}
pub fn create_file<P: AsRef<Path>>(path: P) -> Result<mem::File, Error> {
FS.write().create_file(path)
}
pub fn read_file<P: AsRef<Path>>(path: P) -> Result<Vec<u8>, Error> {
let mut file = open_file(path)?;
let mut buf = vec![];
file.read_to_end(&mut buf)?;
Ok(buf)
}
pub fn create_dir<P: AsRef<Path>>(path: P) -> Result<(), Error> {
FS.write().create_dir(path)
}
pub fn create_dir_all<P: AsRef<Path>>(path: P) -> Result<(), Error> {
FS.write().create_dir_all(path)
}
pub fn read_dir<P: AsRef<Path>>(path: P) -> Result<mem::ReadDir, Error> {
FS.read().read_dir(path)
}
pub fn remove_dir<P: AsRef<Path>>(path: P) -> Result<(), Error> {
FS.write().remove_dir(path)
}
pub fn remove_dir_all<P: AsRef<Path>>(path: P) -> Result<(), Error> {
FS.write().remove_dir_all(path)
}
pub fn remove_file<P: AsRef<Path>>(path: P) -> Result<(), Error> {
FS.write().remove_file(path)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Read;
use std::io::Write;
#[test]
fn create_read_file() {
let path = "/test.txt";
let mut file = create_file(path).unwrap();
file.write_all(b"hello world").unwrap();
let mut file = open_file(path).unwrap();
let mut buf = vec![];
file.read_to_end(&mut buf).unwrap();
assert_eq!(buf, b"hello world");
remove_file(path).unwrap();
}
#[test]
fn create_read_directory() {
let path = "/test_dir/abc/";
assert_eq!(true, create_dir_all(path).is_ok());
assert_eq!(false, remove_dir("/test_dir/").is_ok());
assert_eq!(true, remove_dir_all("/test_dir/").is_ok());
}
}
src/infra/cluster.rs
+525
View File
@@ -0,0 +1,525 @@
use dashmap::DashMap;
use etcd_client::PutOptions;
use serde::{Deserialize, Serialize};
use std::str::FromStr;
use std::sync::Arc;
use uuid::Uuid;
use super::config::CONFIG;
use super::db::ETCD_CLIENT;
use super::errors::{Error, Result};
use crate::common::json;
use crate::infra::db::{etcd, Event};
use crate::service::db;
pub static mut LOCAL_NODE_ID: i32 = 0;
pub static mut LOCAL_NODE_KEY_LEASE_ID: i64 = 0;
pub static LOCAL_NODE_KEY_TTL: i64 = 10; // node ttl, seconds
pub static mut LOCAL_NODE_STATUS: NodeStatus = NodeStatus::Prepare;
lazy_static! {
pub static ref LOCAL_NODE_UUID: String = load_local_node_uuid();
pub static ref LOCAL_NODE_ROLE: Vec<Role> = load_local_node_role();
pub static ref NODES: DashMap<String, Node> = DashMap::new();
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Node {
pub id: i32,
pub uuid: String,
pub name: String,
pub http_addr: String,
pub grpc_addr: String,
pub role: Vec<Role>,
pub cpu_num: u64,
pub status: NodeStatus,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum NodeStatus {
Prepare,
Online,
Offline,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum Role {
All,
Ingester,
Querier,
Compactor,
Router,
AlertManager,
}
impl FromStr for Role {
type Err = String;
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
let s = s.to_lowercase();
match s.as_str() {
"all" => Ok(Role::All),
"ingester" => Ok(Role::Ingester),
"querier" => Ok(Role::Querier),
"compactor" => Ok(Role::Compactor),
"router" => Ok(Role::Router),
"alertmanager" => Ok(Role::AlertManager),
_ => Err(format!("Invalid cluster role: {}", s)),
}
}
}
/// Register and keepalive the node to cluster
pub async fn register_and_keepalive() -> Result<()> {
if CONFIG.common.local_mode {
let nodes = load_local_node_role();
if !is_single_node(nodes) {
panic!("For local mode only single node deployment is allowed !");
}
return Ok(());
}
if let Err(e) = register().await {
log::error!("[TRACE] Register to cluster failed: {}", e);
return Err(e);
}
// keep alive
tokio::task::spawn(async move {
loop {
if is_offline() {
break;
}
let lease_id = unsafe { LOCAL_NODE_KEY_LEASE_ID };
let ret = etcd::keepalive_lease_id(lease_id, LOCAL_NODE_KEY_TTL, is_offline).await;
if ret.is_ok() {
break;
}
let e = ret.unwrap_err();
let estr = e.to_string();
if is_offline()
|| estr
!= Error::from(etcd_client::Error::LeaseKeepAliveError(
"lease expired or revoked".to_string(),
))
.to_string()
{
break;
}
log::error!("keepalive lease id expired or revoked, set node online again.");
// get new lease id
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let resp = match client.lease_grant(LOCAL_NODE_KEY_TTL, None).await {
Ok(resp) => resp,
Err(e) => {
log::error!("lease grant failed: {}", e);
continue;
}
};
let id = resp.id();
// update local node key lease id
unsafe {
LOCAL_NODE_KEY_LEASE_ID = id;
}
if let Err(e) = set_online().await {
log::error!("set node online failed: {}", e);
continue;
}
}
});
Ok(())
}
/// Register to cluster
pub async fn register() -> Result<()> {
// 1. create a cluster lock for node register
let mut locker = etcd::Locker::new("nodes/register");
locker.lock(0).await?;
// 2. get node list
let node_list = list_nodes().await?;
// 3. calculate node_id
let mut node_id = 1;
let mut node_ids = Vec::new();
for node in node_list {
node_ids.push(node.id);
NODES.insert(node.uuid.clone(), node);
}
node_ids.sort();
for id in &node_ids {
if *id == node_id {
node_id += 1;
} else {
break;
}
}
// update local id
unsafe {
LOCAL_NODE_ID = node_id;
}
// 4. join the cluster
let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
let val = Node {
id: node_id,
uuid: LOCAL_NODE_UUID.clone(),
name: CONFIG.common.instance_name.clone(),
http_addr: format!("http://{}:{}", get_node_ip(), CONFIG.http.port),
grpc_addr: format!("http://{}:{}", get_node_ip(), CONFIG.grpc.port),
role: LOCAL_NODE_ROLE.clone(),
cpu_num: sys_info::cpu_num().unwrap() as u64,
status: NodeStatus::Prepare,
};
// cache local node
NODES.insert(LOCAL_NODE_UUID.clone(), val.clone());
let val = json::to_string(&val).unwrap();
// register node to cluster
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let resp = client.lease_grant(LOCAL_NODE_KEY_TTL, None).await?;
let id = resp.id();
// update local node key lease id
unsafe {
LOCAL_NODE_KEY_LEASE_ID = id;
}
let opt = PutOptions::new().with_lease(id);
let _resp = client.put(key, val, Some(opt)).await?;
// 5. watch node list
tokio::task::spawn(async move { watch_node_list().await });
// 7. register ok, release lock
locker.unlock().await?;
log::info!("[TRACE] Register to cluster ok");
Ok(())
}
/// set online to cluster
pub async fn set_online() -> Result<()> {
if CONFIG.common.local_mode {
return Ok(());
}
// set node status to online
let val = match NODES.get(LOCAL_NODE_UUID.as_str()) {
Some(node) => {
let mut val = node.value().clone();
val.status = NodeStatus::Online;
val
}
None => Node {
id: unsafe { LOCAL_NODE_ID },
uuid: LOCAL_NODE_UUID.clone(),
name: CONFIG.common.instance_name.clone(),
http_addr: format!("http://{}:{}", get_node_ip(), CONFIG.http.port),
grpc_addr: format!("http://{}:{}", get_node_ip(), CONFIG.grpc.port),
role: LOCAL_NODE_ROLE.clone(),
cpu_num: sys_info::cpu_num().unwrap() as u64,
status: NodeStatus::Online,
},
};
unsafe {
LOCAL_NODE_STATUS = NodeStatus::Online;
}
// cache local node
NODES.insert(LOCAL_NODE_UUID.clone(), val.clone());
let val = json::to_string(&val).unwrap();
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
let opt = PutOptions::new().with_lease(unsafe { LOCAL_NODE_KEY_LEASE_ID });
let _resp = client.put(key, val, Some(opt)).await?;
Ok(())
}
/// Leave cluster
pub async fn leave() -> Result<()> {
if CONFIG.common.local_mode {
return Ok(());
}
unsafe {
LOCAL_NODE_STATUS = NodeStatus::Offline;
}
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
let _resp = client.delete(key, None).await?;
Ok(())
}
#[inline(always)]
pub fn get_cached_online_nodes() -> Option<Vec<Node>> {
if NODES.len() == 0 {
return None;
}
let mut node_list = Vec::new();
for node in NODES.iter() {
if node.status.eq(&NodeStatus::Online) {
node_list.push(node.value().clone());
}
}
Some(node_list)
}
#[inline(always)]
pub fn get_cached_online_query_nodes() -> Option<Vec<Node>> {
if NODES.len() == 0 {
return None;
}
let mut node_list = Vec::new();
for node in NODES.iter() {
if node.status.eq(&NodeStatus::Online)
&& (is_querier(&node.value().role) || is_ingester(&node.value().role))
{
node_list.push(node.value().clone());
}
}
Some(node_list)
}
#[inline]
pub fn get_cached_online_ingester_nodes() -> Option<Vec<Node>> {
if NODES.len() == 0 {
return None;
}
let mut node_list = Vec::new();
for node in NODES.iter() {
if node.status.eq(&NodeStatus::Online) && is_ingester(&node.value().role) {
node_list.push(node.value().clone());
}
}
Some(node_list)
}
#[inline]
pub fn get_cached_online_querier_nodes() -> Option<Vec<Node>> {
if NODES.len() == 0 {
return None;
}
let mut node_list = Vec::new();
for node in NODES.iter() {
if node.status.eq(&NodeStatus::Online) && is_querier(&node.value().role) {
node_list.push(node.value().clone());
}
}
Some(node_list)
}
/// List nodes from cluster or local cache
async fn list_nodes() -> Result<Vec<Node>> {
let mut nodes = Vec::new();
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let key = format!("{}nodes/", &CONFIG.etcd.prefix);
let opt = etcd_client::GetOptions::new().with_prefix();
let ret = client.get(key.clone(), Some(opt)).await.map_err(|e| {
log::error!("Error getting node list from etcd {}: {}", key.clone(), e);
e
})?;
for item in ret.kvs() {
// let item_key = item.key_str().unwrap().to_string();
// let short_key = item_key.strip_prefix(&format!("{}nodes/", &CONFIG.etcd.prefix)).unwrap();
// println!("key: {:?}", item_key);
// println!("key: {:?}", short_key);
// println!("val: {:?}", item.value_str());
let node: Node = json::from_slice(item.value())?;
nodes.push(node.to_owned());
}
Ok(nodes)
}
async fn watch_node_list() -> Result<()> {
let db = &super::db::DEFAULT;
let key = "/nodes/";
let mut events = db.watch(key).await?;
let events = Arc::get_mut(&mut events).unwrap();
log::info!("[TRACE] Start watching node_list");
loop {
let ev = match events.recv().await {
Some(ev) => ev,
None => {
log::error!("watch_node_list: event channel closed");
break;
}
};
match ev {
Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
let item_value: Node = json::from_slice(&ev.value.unwrap()).unwrap();
log::info!("[TRACE] cluster->node: join {:?}", item_value.clone());
NODES.insert(item_key.to_string(), item_value.clone());
// need broadcast local file list
if item_value.status.eq(&NodeStatus::Online)
&& !LOCAL_NODE_UUID.eq(item_key)
&& is_ingester(&LOCAL_NODE_ROLE)
{
db::file_list::local::broadcast_cache().await.unwrap();
}
}
Event::Delete(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
let item_value = NODES.get(item_key).unwrap().clone();
log::info!("[TRACE] cluster->node: leave {:?}", item_value.clone());
NODES.remove(item_key);
}
}
}
Ok(())
}
#[inline(always)]
fn load_local_node_uuid() -> String {
Uuid::new_v4().to_string()
}
#[inline(always)]
pub fn load_local_node_role() -> Vec<Role> {
CONFIG
.common
.node_role
.clone()
.split(',')
.map(|s| s.parse().unwrap())
.collect::<Vec<Role>>()
}
#[inline(always)]
pub fn is_ingester(role: &[Role]) -> bool {
if role.contains(&Role::All) {
return true;
}
if role.contains(&Role::Ingester) {
return true;
}
false
}
#[inline(always)]
pub fn is_querier(role: &[Role]) -> bool {
if role.contains(&Role::All) {
return true;
}
if role.contains(&Role::Querier) {
return true;
}
false
}
#[inline(always)]
pub fn is_compactor(role: &[Role]) -> bool {
if role.contains(&Role::All) {
return true;
}
if role.contains(&Role::Compactor) {
return true;
}
false
}
#[inline(always)]
pub fn is_router(role: &[Role]) -> bool {
if role.contains(&Role::Router) {
return true;
}
false
}
#[inline(always)]
pub fn is_alert_manager(role: &[Role]) -> bool {
if role.contains(&Role::All) {
return true;
}
if role.contains(&Role::AlertManager) {
return true;
}
false
}
#[inline(always)]
fn is_single_node(role: Vec<Role>) -> bool {
if role.contains(&Role::All) {
return true;
}
false
}
#[inline(always)]
pub fn is_offline() -> bool {
unsafe {
if LOCAL_NODE_STATUS == NodeStatus::Offline {
return true;
}
}
false
}
#[inline(always)]
pub fn ge_node_by_uuid(uuid: &str) -> Option<Node> {
NODES.get(uuid).map(|node| node.clone())
}
#[inline(always)]
pub fn get_node_ip() -> String {
// Print the ip address of all adapters:
let mut node_ip: String = "".to_string();
for adapter in get_if_addrs::get_if_addrs().unwrap() {
// println!("Ip addresses: {:#?}", adapter.ip());
if adapter.is_loopback() {
continue;
}
node_ip = adapter.ip().to_string();
break;
}
node_ip
}
#[cfg(test)]
mod test_utils {
use super::*;
#[test]
fn test_is_querier() {
let val = is_querier(&[Role::Querier]);
assert_eq!(val, true);
let val = is_querier(&[Role::All]);
assert_eq!(val, true);
let val = is_querier(&[Role::Ingester]);
assert_eq!(val, false);
}
#[test]
fn test_is_ingester() {
let val = is_ingester(&[Role::Ingester]);
assert_eq!(val, true);
let val = is_ingester(&[Role::All]);
assert_eq!(val, true);
let val = is_ingester(&[Role::Querier]);
assert_eq!(val, false);
}
#[test]
fn test_load_local_node_uuid() {
let id = load_local_node_uuid();
assert_ne!(id, "");
}
#[actix_web::test]
#[ignore]
async fn test_list_nodes() {
let res = list_nodes().await;
assert_eq!(res.unwrap().len(), 0);
}
#[test]
fn test_get_cached_online_query_nodes() {
let res = get_cached_online_query_nodes();
assert!(res.is_none());
}
}
src/infra/config.rs
+364
View File
@@ -0,0 +1,364 @@
use dashmap::DashMap;
use datafusion::arrow::datatypes::Schema;
use dotenv::dotenv;
use dotenv_config::EnvConfig;
use sys_info::hostname;
use crate::common::file::get_file_meta;
use crate::meta::alert::{AlertList, Trigger, TriggerTimer};
use crate::meta::prom::ClusterLeader;
use crate::meta::transform::{Transform, TransformList};
use crate::meta::user::User;
lazy_static! {
pub static ref CONFIG: Config = init();
pub static ref LOCKER: DashMap<String, std::sync::Mutex<bool>> = DashMap::new();
}
lazy_static! {
pub static ref STREAM_TRANSFORMS: DashMap<String, TransformList> = DashMap::new();
pub static ref STREAM_SCHEMAS: DashMap<String, Vec<Schema>> = DashMap::new();
pub static ref QUERY_TRANSFORMS: DashMap<String, Transform> = DashMap::new();
pub static ref USERS: DashMap<String, User> = DashMap::new();
pub static ref PASSWORD_HASH: DashMap<String, String> = DashMap::new();
#[derive(Debug)]
pub static ref METRIC_CLUSTER_MAP: DashMap<String, Vec<String>> = DashMap::new();
#[derive(Debug)]
pub static ref METRIC_CLUSTER_LEADER: DashMap<String, ClusterLeader> = DashMap::new();
pub static ref STREAM_ALERTS: DashMap<String, AlertList> = DashMap::new();
pub static ref TRIGGERS: DashMap<String, Trigger> = DashMap::new();
pub static ref TRIGGERS_IN_PROCESS: DashMap<String, TriggerTimer> = DashMap::new();
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Config {
pub auth: Auth,
pub http: Http,
pub grpc: Grpc,
pub route: Route,
pub common: Common,
pub limit: Limit,
pub compact: Compact,
pub memory_cache: MemoryCache,
pub log: Log,
pub etcd: Etcd,
pub sled: Sled,
pub s3: S3,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Auth {
#[env_config(name = "ZIOX_USER_NAME", default = "admin")]
pub username: String,
#[env_config(name = "ZIOX_USER_PASSWORD", default = "Complexpass#123")]
pub password: String,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Http {
#[env_config(name = "ZIOX_HTTP_PORT", default = 5080)]
pub port: u16,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Grpc {
#[env_config(name = "ZIOX_GRPC_PORT", default = 5081)]
pub port: u16,
#[env_config(name = "ZIOX_GRPC_TIMEOUT", default = 600)]
pub timeout: u64,
#[env_config(name = "ZIOX_GRPC_ORG_HEADER_KEY", default = "zinc-org-id")]
pub org_header_key: String,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Route {
#[env_config(name = "ZIOX_ROUTE_TIMEOUT", default = 600)]
pub timeout: u64,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Common {
#[env_config(name = "ZIOX_LOCAL_MODE", default = false)]
pub local_mode: bool,
// ZIOX_LOCAL_MODE_STORAGE is ignored when ZIOX_LOCAL_MODE is set to false
#[env_config(name = "ZIOX_LOCAL_MODE_STORAGE", default = "disk")]
pub local_mode_storage: String,
#[env_config(name = "ZIOX_NODE_ROLE", default = "all")]
pub node_role: String,
#[env_config(name = "ZIOX_INSTANCE_NAME", default = "")]
pub instance_name: String,
#[env_config(name = "ZIOX_DATA_DIR", default = "./data/")]
pub data_dir: String,
#[env_config(name = "ZIOX_DATA_WAL_DIR", default = "./data/wal/")]
pub data_wal_dir: String,
#[env_config(name = "ZIOX_DATA_STREAM_DIR", default = "./data/stream/")]
pub data_stream_dir: String,
#[env_config(name = "ZIOX_FILE_EXT_JSON", default = ".json")]
pub file_ext_json: String,
#[env_config(name = "ZIOX_FILE_EXT_PARQUET", default = ".parquet")]
pub file_ext_parquet: String,
#[env_config(name = "ZIOX_PARQUET_COMPRESSION", default = "zstd")]
pub parquet_compression: String,
#[env_config(name = "ZIOX_TIME_STAMP_COL", default = "_timestamp")]
pub time_stamp_col: String,
#[env_config(name = "ZIOX_WIDENING_SCHEMA_EVOLUTION", default = false)]
pub widening_schema_evoluation: bool,
#[env_config(name = "ZIOX_FEATURE_PER_THREAD_LOCK", default = false)]
pub feature_per_thread_lock: bool,
#[env_config(name = "ZIOX_FEATURE_FULLTEXT_ON_ALL_FIELDS", default = false)]
pub feature_fulltext_on_all_fields: bool,
#[env_config(name = "ZIOX_UI_ENABLED", default = true)]
pub ui_enabled: bool,
#[env_config(name = "ZIOX_METRICS_DEDUP_ENABLED", default = true)]
pub metrics_dedup_enabled: bool,
#[env_config(name = "ZIOX_TRACING_ENABLED", default = false)]
pub tracing_enabled: bool,
#[env_config(
name = "OTEL_OTLP_HTTP_ENDPOINT",
default = "http://127.0.0.1:5080/api/nexus/traces"
)]
pub otel_otlp_url: String,
#[env_config(name = "ZIOX_TRACING_HEADER_KEY", default = "Authorization")]
pub tracing_header_key: String,
#[env_config(
name = "ZIOX_TRACING_HEADER_VALUE",
default = "Basic YWRtaW46Q29tcGxleHBhc3MjMTIz"
)]
pub tracing_header_value: String,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Limit {
#[env_config(name = "ZIOX_JSON_LIMIT", default = 209715200)]
pub req_json_limit: usize,
#[env_config(name = "ZIOX_PAYLOAD_LIMIT", default = 209715200)]
pub req_payload_limit: usize,
#[env_config(name = "ZIOX_MAX_FILE_SIZE_ON_DISK", default = 10)] // MB
pub max_file_size_on_disk: u64,
#[env_config(name = "ZIOX_MAX_FILE_RETENTION_TIME", default = 600)] // seconds
pub max_file_retention_time: u64,
#[env_config(name = "ZIOX_FILE_PUSH_INTERVAL", default = 10)] // seconds
pub file_push_interval: u64,
#[env_config(name = "ZIOX_FILE_MOVE_THREAD_NUM", default = 0)]
pub file_move_thread_num: usize,
#[env_config(name = "ZIOX_QUERY_THREAD_NUM", default = 0)]
pub query_thread_num: usize,
#[env_config(name = "ZIOX_TS_ALLOWED_UPTO", default = 5)] // in hours - in past
pub allowed_upto: i64,
#[env_config(name = "ZIOX_METRICS_LEADER_PUSH_INTERVAL", default = 15)]
pub metrics_leader_push_interval: u64,
#[env_config(name = "ZIOX_METRICS_LEADER_ELECTION_INTERVAL", default = 30)]
pub metrics_leader_election_interval: i64,
// no need set by environment
pub cpu_num: usize,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Compact {
#[env_config(name = "ZIOX_COMPACT_ENABLED", default = true)]
pub enabled: bool,
#[env_config(name = "ZIOX_COMPACT_INTERVAL", default = 600)] // seconds
pub interval: u64,
#[env_config(name = "ZIOX_COMPACT_MAX_FILE_SIZE", default = 256)] // MB
pub max_file_size: u64,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct MemoryCache {
#[env_config(name = "ZIOX_MEMORY_CACHE_ENABLED", default = true)]
pub enabled: bool,
#[env_config(name = "ZIOX_MEMORY_CACHE_CACHE_LATEST_FILES", default = false)]
pub cache_latest_files: bool,
#[env_config(name = "ZIOX_MEMORY_CACHE_MAX_SIZE", default = 0)]
// MB, default is half of system memory
pub max_size: usize,
#[env_config(name = "ZIOX_MEMORY_CACHE_RELEASE_SIZE", default = 0)]
// MB, default is 1% of max_size
pub release_size: usize,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Log {
#[env_config(name = "RUST_LOG", default = "info")]
pub level: String,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Etcd {
#[env_config(name = "ZIOX_ETCD_ADDR", default = "localhost:2379")]
pub addr: String,
#[env_config(name = "ZIOX_ETCD_PREFIX", default = "/zinc/observe/")]
pub prefix: String,
#[env_config(name = "ZIOX_ETCD_CONNECT_TIMEOUT", default = 2)]
pub connect_timeout: u64,
#[env_config(name = "ZIOX_ETCD_COMMAND_TIMEOUT", default = 5)]
pub command_timeout: u64,
#[env_config(name = "ZIOX_ETCD_LOCK_WAIT_TIMEOUT", default = 600)]
pub lock_wait_timeout: u64,
#[env_config(name = "ZIOX_ETCD_USER", default = "")]
pub user: String,
#[env_config(name = "ZIOX_ETCD_PASSWORD", default = "")]
pub password: String,
#[env_config(name = "ZIOX_ETCD_CLIENT_CERT_AUTH", default = false)]
pub cert_auth: bool,
#[env_config(name = "ZIOX_ETCD_TRUSTED_CA_FILE", default = "")]
pub ca_file: String,
#[env_config(name = "ZIOX_ETCD_CERT_FILE", default = "")]
pub cert_file: String,
#[env_config(name = "ZIOX_ETCD_KEY_FILE", default = "")]
pub key_file: String,
#[env_config(name = "ZIOX_ETCD_DOMAIN_NAME", default = "")]
pub domain_name: String,
#[env_config(name = "ZIOX_ETCD_LOAD_PAGE_SIZE", default = 10000)]
pub load_page_size: i64,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct Sled {
#[env_config(name = "ZIOX_SLED_DATA_DIR", default = "./data/db/")]
pub data_dir: String,
#[env_config(name = "ZIOX_SLED_PREFIX", default = "/zinc/observe/")]
pub prefix: String,
}
#[derive(Clone, Debug, EnvConfig)]
pub struct S3 {
#[env_config(name = "ZIOX_S3_SERVER_URL", default = "")]
pub server_url: String,
#[env_config(name = "ZIOX_S3_REGION_NAME", default = "")]
pub region_name: String,
#[env_config(name = "ZIOX_S3_ACCESS_KEY", default = "")]
pub access_key: String,
#[env_config(name = "ZIOX_S3_SECRET_KEY", default = "")]
pub secret_key: String,
#[env_config(name = "ZIOX_S3_BUCKET_NAME", default = "")]
pub bucket_name: String,
}
pub fn init() -> Config {
dotenv().ok();
let mut cfg = Config::init().unwrap();
// set cpu num
let cpu_num = sys_info::cpu_num().unwrap() as usize;
cfg.limit.cpu_num = cpu_num;
// HACK for thread_num equal to CPU core * 4
if cfg.limit.query_thread_num == 0 {
cfg.limit.query_thread_num = cpu_num * 4;
}
// HACK for move_file_thread_num equal to CPU core
if cfg.limit.file_move_thread_num == 0 {
cfg.limit.file_move_thread_num = cpu_num;
}
if cfg.limit.file_push_interval == 0 {
cfg.limit.file_push_interval = 10;
}
// HACK instance_name
if cfg.common.instance_name.is_empty() {
cfg.common.instance_name = hostname().unwrap();
}
// check max_file_size_on_disk to MB
cfg.limit.max_file_size_on_disk *= 1024 * 1024;
// check compact_max_file_size to MB
cfg.compact.max_file_size *= 1024 * 1024;
if cfg.compact.interval == 0 {
cfg.compact.interval = 600;
}
// format local_mode_storage
cfg.common.local_mode_storage = cfg.common.local_mode_storage.to_lowercase();
// check data path config
if let Err(e) = check_path_config(&mut cfg) {
panic!("data path config error: {}", e);
}
// check memeory cache
if let Err(e) = check_memory_cache_config(&mut cfg) {
panic!("memory cache config error: {}", e);
}
// check etcd config
if let Err(e) = check_etcd_config(&mut cfg) {
panic!("etcd config error: {}", e);
}
cfg
}
fn check_path_config(cfg: &mut Config) -> Result<(), anyhow::Error> {
if !cfg.common.data_wal_dir.ends_with('/') {
cfg.common.data_wal_dir = format!("{}/", cfg.common.data_wal_dir);
}
if !cfg.common.data_stream_dir.ends_with('/') {
cfg.common.data_stream_dir = format!("{}/", cfg.common.data_stream_dir);
}
if !cfg.sled.data_dir.ends_with('/') {
cfg.sled.data_dir = format!("{}/", cfg.sled.data_dir);
}
Ok(())
}
fn check_etcd_config(cfg: &mut Config) -> Result<(), anyhow::Error> {
if !cfg.etcd.cert_auth {
return Ok(());
}
if let Err(e) = get_file_meta(&cfg.etcd.ca_file) {
return Err(anyhow::anyhow!(
"ZIOX_ETCD_TRUSTED_CA_FILE check err: {}",
e
));
}
if let Err(e) = get_file_meta(&cfg.etcd.cert_file) {
return Err(anyhow::anyhow!(
"ZIOX_ETCD_TRUSTED_CA_FILE check err: {}",
e
));
}
if let Err(e) = get_file_meta(&cfg.etcd.key_file) {
return Err(anyhow::anyhow!(
"ZIOX_ETCD_TRUSTED_CA_FILE check err: {}",
e
));
}
// check domain name
if cfg.etcd.domain_name.is_empty() {
let mut name = cfg.etcd.addr.clone();
if name.contains("//") {
name = name.split("//").collect::<Vec<&str>>()[1].to_string();
}
if name.contains(':') {
name = name.split(':').collect::<Vec<&str>>()[0].to_string();
}
cfg.etcd.domain_name = name;
}
Ok(())
}
fn check_memory_cache_config(cfg: &mut Config) -> Result<(), anyhow::Error> {
if cfg.memory_cache.max_size == 0 {
// meminfo unit is KB
let meminfo = sys_info::mem_info()?;
cfg.memory_cache.max_size = meminfo.total as usize * 1024 / 2;
} else {
cfg.memory_cache.max_size *= 1024 * 1024;
}
if cfg.memory_cache.release_size == 0 {
// when memory cache is full will release 1% (default)
cfg.memory_cache.release_size = cfg.memory_cache.max_size / 100;
} else {
cfg.memory_cache.release_size *= 1024 * 1024;
}
Ok(())
}
#[inline]
pub fn get_parquet_compression() -> parquet::basic::Compression {
match CONFIG.common.parquet_compression.to_lowercase().as_str() {
"snappy" => parquet::basic::Compression::SNAPPY,
"gzip" => parquet::basic::Compression::GZIP,
"brotli" => parquet::basic::Compression::BROTLI,
"lz4" => parquet::basic::Compression::LZ4_RAW,
"zstd" => parquet::basic::Compression::ZSTD,
_ => parquet::basic::Compression::ZSTD,
}
}
src/infra/db/etcd.rs
+556
View File
@@ -0,0 +1,556 @@
use ahash::HashMap;
use async_once::AsyncOnce;
use async_trait::async_trait;
use bytes::Bytes;
use etcd_client::{
Compare, CompareOp, DeleteOptions, EventType, GetOptions, SortOrder, SortTarget, TxnOp,
};
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tokio::time;
use tonic::transport::{Certificate, ClientTlsConfig, Identity};
use tracing::info_span;
use super::{Event, EventData};
use crate::infra::cluster;
use crate::infra::{config::CONFIG, errors::*};
/// max operations in txn request
pub const MAX_OPS_PER_TXN: usize = 120; // etcd hard coded limit is 128
lazy_static! {
pub static ref ETCD_CLIENT: AsyncOnce<Option<etcd_client::Client>> =
AsyncOnce::new(async { connect_etcd().await });
}
pub struct Etcd {
prefix: String,
}
impl Etcd {
pub fn new(prefix: &str) -> Etcd {
let prefix = prefix.trim_end_matches(|v| v == '/');
Etcd {
prefix: prefix.to_string(),
}
}
}
impl Default for Etcd {
fn default() -> Self {
Self::new(&CONFIG.etcd.prefix)
}
}
#[async_trait]
impl super::Db for Etcd {
async fn get(&self, key: &str) -> Result<Bytes> {
let key = format!("{}{}", self.prefix, key);
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let ret = client.get(key.as_str(), None).await?;
if ret.kvs().is_empty() {
return Err(Error::from(DbError::KeyNotExists(key)));
}
let value: Bytes = Bytes::from(ret.kvs()[0].value().to_vec());
Ok(value)
}
async fn put(&self, key: &str, value: Bytes) -> Result<()> {
let key = format!("{}{}", self.prefix, key);
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let _ = client.put(key, value, None).await?;
Ok(())
}
async fn delete(&self, key: &str, with_prefix: bool) -> Result<()> {
let key = format!("{}{}", self.prefix, key);
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let opt = if with_prefix {
Some(DeleteOptions::new().with_prefix())
} else {
None
};
let result = client.delete(key.as_str(), opt).await?;
if result.deleted() > 0 {
Ok(())
} else {
return Err(Error::from(DbError::KeyNotExists(key)));
}
}
async fn list(&self, prefix: &str) -> Result<HashMap<String, Bytes>> {
let mut result = HashMap::default();
let key = format!("{}{}", self.prefix, prefix);
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
.with_limit(CONFIG.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
if kvs_num < CONFIG.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
let item_key = kv.key_str().unwrap();
if !item_key.starts_with(&key) {
have_next = false;
break;
}
if item_key.eq(last_key.as_str()) {
continue;
}
let item_key = item_key.strip_prefix(&self.prefix).unwrap();
result.insert(item_key.to_string(), Bytes::from(kv.value().to_vec()));
}
tokio::task::yield_now().await; // yield to other tasks
if !have_next {
break;
}
if first_call {
first_call = false;
opt = opt.with_from_key();
}
last_key = resp.kvs().last().unwrap().key_str().unwrap().to_string();
resp = client.get(last_key.clone(), Some(opt.clone())).await?;
}
Ok(result)
}
async fn list_use_channel(&self, prefix: &str) -> Result<Arc<mpsc::Receiver<(String, Bytes)>>> {
let (tx, rx) = mpsc::channel(CONFIG.etcd.load_page_size as usize);
let key = format!("{}{}", self.prefix, prefix);
let prefix_key = self.prefix.to_string();
let _task: JoinHandle<Result<()>> = tokio::task::spawn(async move {
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
.with_limit(CONFIG.etcd.load_page_size);
let mut resp = match client.get(key.clone(), Some(opt.clone())).await {
Ok(resp) => resp,
Err(err) => {
log::error!("get prefix error: {}", err);
return Err(Error::from(err));
}
};
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
if kvs_num < CONFIG.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
let item_key = kv.key_str().unwrap();
if !item_key.starts_with(&key) {
have_next = false;
break;
}
if item_key.eq(last_key.as_str()) {
continue;
}
let item_key = item_key.strip_prefix(&prefix_key).unwrap();
tx.send((item_key.to_string(), Bytes::from(kv.value().to_vec())))
.await
.unwrap();
}
tokio::task::yield_now().await; // yield to other tasks
if !have_next {
break;
}
if first_call {
first_call = false;
opt = opt.with_from_key();
}
last_key = resp.kvs().last().unwrap().key_str().unwrap().to_string();
resp = client.get(last_key.clone(), Some(opt.clone())).await?;
}
Ok(())
});
Ok(Arc::new(rx))
}
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>> {
let mut result = Vec::new();
let key = format!("{}{}", self.prefix, prefix);
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
.with_limit(CONFIG.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
if kvs_num < CONFIG.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
let item_key = kv.key_str().unwrap();
if !item_key.starts_with(&key) {
have_next = false;
break;
}
if item_key.eq(last_key.as_str()) {
continue;
}
let item_key = item_key.strip_prefix(&self.prefix).unwrap();
result.push(item_key.to_string());
}
tokio::task::yield_now().await; // yield to other tasks
if !have_next {
break;
}
if first_call {
first_call = false;
opt = opt.with_from_key();
}
last_key = resp.kvs().last().unwrap().key_str().unwrap().to_string();
resp = client.get(last_key.clone(), Some(opt.clone())).await?;
}
Ok(result)
}
async fn list_values(&self, prefix: &str) -> Result<Vec<Bytes>> {
let mut result = Vec::new();
let key = format!("{}{}", self.prefix, prefix);
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
.with_limit(CONFIG.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
if kvs_num < CONFIG.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
let item_key = kv.key_str().unwrap();
if !item_key.starts_with(&key) {
have_next = false;
break;
}
if item_key.eq(last_key.as_str()) {
continue;
}
result.push(Bytes::from(kv.value().to_vec()));
}
tokio::task::yield_now().await; // yield to other tasks
if !have_next {
break;
}
if first_call {
first_call = false;
opt = opt.with_from_key();
}
last_key = resp.kvs().last().unwrap().key_str().unwrap().to_string();
resp = client.get(last_key.clone(), Some(opt.clone())).await?;
}
Ok(result)
}
async fn watch(&self, prefix: &str) -> Result<Arc<mpsc::Receiver<Event>>> {
let (tx, rx) = mpsc::channel(1024);
let key = format!("{}{}", &self.prefix, prefix);
let prefix_key = self.prefix.to_string();
let _task: JoinHandle<Result<()>> = tokio::task::spawn(async move {
loop {
if cluster::is_offline() {
break;
}
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let opt = etcd_client::WatchOptions::new().with_prefix();
let (mut _watcher, mut stream) =
match client.watch(key.clone(), Some(opt.clone())).await {
Ok((watcher, stream)) => (watcher, stream),
Err(e) => {
log::error!("watching prefix: {}, error: {}", key, e);
time::sleep(time::Duration::from_secs(1)).await;
continue;
}
};
loop {
let resp = match stream.message().await {
Ok(resp) => resp,
Err(e) => {
log::error!("watching prefix: {}, get message error: {}", key, e);
break;
}
};
if let Some(ev) = resp {
for ev in ev.events() {
let kv = ev.kv().unwrap();
let item_key = kv.key_str().unwrap();
let item_key = item_key.strip_prefix(&prefix_key).unwrap();
match ev.event_type() {
EventType::Put => tx
.send(Event::Put(EventData {
key: item_key.to_string(),
value: Some(Bytes::from(kv.value().to_vec())),
}))
.await
.unwrap(),
EventType::Delete => tx
.send(Event::Delete(EventData {
key: item_key.to_string(),
value: None,
}))
.await
.unwrap(),
}
}
}
}
}
Ok(())
});
Ok(Arc::new(rx))
}
async fn transaction(
&self,
check_key: &str,
and_ops: Vec<Event>,
else_ops: Vec<Event>,
) -> Result<()> {
let db_span = info_span!("infra:db:etcd:transaction");
let _guard = db_span.enter();
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let mut txn = etcd_client::Txn::new();
let compares = vec![Compare::value(
check_key.to_string(),
CompareOp::NotEqual,
"",
)];
txn = txn.when(compares);
let mut txn_and_ops = Vec::new();
for op in and_ops {
match op {
Event::Put(data) => {
let key = format!("{}{}", self.prefix, data.key);
txn_and_ops.push(TxnOp::put(key, data.value.unwrap(), None));
}
Event::Delete(data) => {
let key = format!("{}{}", self.prefix, data.key);
txn_and_ops.push(TxnOp::delete(key, None));
}
}
}
let mut txn_else_ops = Vec::new();
for op in else_ops {
match op {
Event::Put(data) => {
let key = format!("{}{}", self.prefix, data.key);
txn_else_ops.push(TxnOp::put(key, data.value.unwrap(), None));
}
Event::Delete(data) => {
let key = format!("{}{}", self.prefix, data.key);
txn_else_ops.push(TxnOp::delete(key, None));
}
}
}
txn = txn.and_then(txn_and_ops);
txn = txn.or_else(txn_else_ops);
let _ = client.txn(txn).await?;
Ok(())
}
}
pub async fn connect_etcd() -> Option<etcd_client::Client> {
if CONFIG.common.local_mode {
return None;
}
let mut opts = etcd_client::ConnectOptions::new()
.with_timeout(core::time::Duration::from_secs(CONFIG.etcd.command_timeout))
.with_connect_timeout(core::time::Duration::from_secs(CONFIG.etcd.connect_timeout));
if !&CONFIG.etcd.user.is_empty() {
opts = opts.with_user(&CONFIG.etcd.user, &CONFIG.etcd.password);
}
if CONFIG.etcd.cert_auth {
let server_root_ca_cert = tokio::fs::read(&CONFIG.etcd.ca_file).await.unwrap();
let server_root_ca_cert = Certificate::from_pem(server_root_ca_cert);
let client_cert = tokio::fs::read(&CONFIG.etcd.cert_file).await.unwrap();
let client_key = tokio::fs::read(&CONFIG.etcd.key_file).await.unwrap();
let client_identity = Identity::from_pem(client_cert, client_key);
let tls = ClientTlsConfig::new()
.domain_name(&CONFIG.etcd.domain_name)
.ca_certificate(server_root_ca_cert)
.identity(client_identity);
opts = opts.with_tls(tls);
}
let client = etcd_client::Client::connect([&CONFIG.etcd.addr], Some(opts))
.await
.unwrap();
// enable keep alive for auth token
tokio::task::spawn(async move { keepalive_connection().await });
Some(client)
}
pub async fn keepalive_connection() -> Result<()> {
loop {
if cluster::is_offline() {
break;
}
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let key = format!("{}healthz", &CONFIG.etcd.prefix);
let key = key.as_str();
client.put(key, "OK", None).await?;
let mut interval = time::interval(time::Duration::from_secs(60));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
match client.get(key, None).await {
Ok(ret) => for _item in ret.kvs() {},
Err(e) => {
log::error!("keep alive connection error: {:?}", e);
break;
}
};
}
}
Ok(())
}
pub async fn keepalive_lease_id<F>(id: i64, ttl: i64, stopper: F) -> Result<()>
where
F: Fn() -> bool,
{
let mut ttl_keep_alive = (ttl / 2) as u64;
loop {
if stopper() {
break;
}
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let (mut keeper, mut stream) = match client.lease_keep_alive(id).await {
Ok((keeper, stream)) => (keeper, stream),
Err(e) => {
log::error!("lease {:?} keep alive error: {:?}", id, e);
time::sleep(time::Duration::from_secs(1)).await;
continue;
}
};
loop {
if stopper() {
break;
}
time::sleep(time::Duration::from_secs(ttl_keep_alive)).await;
match keeper.keep_alive().await {
Ok(_) => {}
Err(e) => {
log::error!("lease {:?} keep alive do keeper error: {:?}", id, e);
ttl_keep_alive = 1;
break;
}
}
match stream.message().await {
Ok(v) => {
if v.unwrap().ttl() == 0 {
log::error!("lease {:?} keep alive ttl is 0", id);
return Err(Error::from(etcd_client::Error::LeaseKeepAliveError(
"lease expired or revoked".to_string(),
)));
}
ttl_keep_alive = (ttl / 2) as u64;
}
Err(e) => {
log::error!("lease {:?} keep alive receive message: {:?}", id, e);
ttl_keep_alive = 1;
break;
}
};
}
}
Ok(())
}
pub struct Locker {
key: String,
lock_id: String,
state: Arc<AtomicU8>, // 0: init, 1: locking, 2: release
}
impl Locker {
pub fn new(key: &str) -> Self {
let key = format!("{}lock/{}", &CONFIG.etcd.prefix, key);
Self {
key,
lock_id: "".to_string(),
state: Arc::new(AtomicU8::new(0)),
}
}
/// lock with timeout, 0 means use default timeout, unit: second
pub async fn lock(&mut self, timeout: u64) -> Result<()> {
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
let mut last_err = None;
let timeout = if timeout == 0 {
CONFIG.etcd.lock_wait_timeout
} else {
timeout
};
let mut n = timeout / CONFIG.etcd.command_timeout;
if n < 1 {
n = 1;
}
for _ in 0..n {
match client.lock(self.key.as_str(), None).await {
Ok(resp) => {
self.lock_id = String::from_utf8_lossy(resp.key()).to_string();
self.state.store(1, Ordering::SeqCst);
last_err = None;
break;
}
Err(err) => {
last_err = Some(err.to_string());
// log::error!("get lock error: {}, key: {}", err, self.key);
if !err.to_string().contains("Timeout expired") {
break;
}
}
};
}
if let Some(err) = last_err {
return Err(Error::Message(format!("get lock error: {}", err)));
}
Ok(())
}
pub async fn unlock(&mut self) -> Result<()> {
if self.state.load(Ordering::SeqCst) != 1 {
return Ok(());
}
let mut client = ETCD_CLIENT.get().await.clone().unwrap();
match client.unlock(self.lock_id.as_str()).await {
Ok(_) => {}
Err(err) => {
log::error!("unlock error: {}, key: {}", err, self.key);
return Err(Error::Message("unlock error".to_string()));
}
};
self.state.store(2, Ordering::SeqCst);
Ok(())
}
}
src/infra/db/mod.rs
+99
View File
@@ -0,0 +1,99 @@
use ahash::HashMap;
use async_trait::async_trait;
use bytes::Bytes;
use std::sync::Arc;
use tokio::sync::mpsc;
use super::config::CONFIG;
use super::errors::Result;
pub mod etcd;
pub mod sled;
pub use self::etcd::ETCD_CLIENT;
pub use self::sled::SLED_CLIENT;
lazy_static! {
pub static ref DEFAULT: Box<dyn Db> = default();
}
pub fn default() -> Box<dyn Db> {
match CONFIG.common.local_mode {
true => Box::new(sled::Sled::new(&CONFIG.sled.prefix)),
false => Box::new(etcd::Etcd::new(&CONFIG.etcd.prefix)),
}
}
#[derive(Clone, Debug)]
pub enum Event {
Put(EventData),
Delete(EventData),
}
#[derive(Clone, Debug)]
pub struct EventData {
pub key: String,
pub value: Option<Bytes>,
}
#[async_trait]
pub trait Db: Sync + 'static {
async fn get(&self, key: &str) -> Result<Bytes>;
async fn put(&self, key: &str, value: Bytes) -> Result<()>;
async fn delete(&self, key: &str, with_prefix: bool) -> Result<()>;
async fn list(&self, prefix: &str) -> Result<HashMap<String, Bytes>>;
async fn list_use_channel(&self, prefix: &str) -> Result<Arc<mpsc::Receiver<(String, Bytes)>>>;
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>>;
async fn list_values(&self, prefix: &str) -> Result<Vec<Bytes>>;
async fn watch(&self, prefix: &str) -> Result<Arc<mpsc::Receiver<Event>>>;
async fn transaction(
&self,
check_key: &str, // check the key exists
and_ops: Vec<Event>,
else_ops: Vec<Event>,
) -> Result<()>;
}
#[cfg(test)]
mod tests {
use super::default;
use bytes::Bytes;
#[tokio::test]
#[ignore]
async fn test_get() {
let db = default();
db.put("/foo/bar", Bytes::from("hello")).await.unwrap();
let value = db.get("/foo/bar").await.unwrap();
assert_eq!(value, Bytes::from("hello"));
}
#[tokio::test]
#[ignore]
async fn test_put() {
let db = default();
assert_eq!(true, db.put("/foo/bar", Bytes::from("hello")).await.is_ok());
}
#[tokio::test]
async fn test_delete() {
let db = default();
assert_eq!(
true,
db.put("/foo/bar1", Bytes::from("hello")).await.is_ok()
);
assert_eq!(
true,
db.put("/foo/bar2", Bytes::from("hello")).await.is_ok()
);
assert_eq!(
true,
db.put("/foo/bar3", Bytes::from("hello")).await.is_ok()
);
assert_eq!(true, db.delete("/foo/bar1", false).await.is_ok());
assert_eq!(true, db.delete("/foo/bar4", false).await.is_err());
assert_eq!(true, db.delete("/foo/", true).await.is_ok());
let value = db.list_keys("/foo/").await.unwrap();
assert_eq!(value.len(), 0);
}
}
src/infra/db/sled.rs
+288
View File
@@ -0,0 +1,288 @@
use ahash::HashMap;
use async_trait::async_trait;
use bytes::Bytes;
use sled::transaction::ConflictableTransactionError;
use std::sync::Arc;
use tokio::{sync::mpsc, task::JoinHandle};
use tracing::info_span;
use super::{Event, EventData};
use crate::infra::cluster;
use crate::infra::{config::CONFIG, errors::*};
lazy_static! {
pub static ref SLED_CLIENT: Option<::sled::Db> = connect_sled();
}
pub struct Sled {
prefix: String,
}
impl Sled {
pub fn new(prefix: &str) -> Sled {
let prefix = prefix.trim_end_matches(|v| v == '/');
Sled {
prefix: prefix.to_string(),
}
}
}
impl Default for Sled {
fn default() -> Self {
Self::new(&CONFIG.sled.prefix)
}
}
#[async_trait]
impl super::Db for Sled {
async fn get(&self, key: &str) -> Result<Bytes> {
let key = format!("{}{}", self.prefix, key);
let client = SLED_CLIENT.clone().unwrap();
let result = client.get(&key);
if result.is_err() {
return Err(Error::from(DbError::KeyNotExists(key)));
}
let result = result.unwrap();
if result.is_none() {
return Err(Error::from(DbError::KeyNotExists(key)));
}
let value: Bytes = Bytes::from(result.unwrap().as_ref().to_vec());
Ok(value)
}
async fn put(&self, key: &str, value: Bytes) -> Result<()> {
let key = format!("{}{}", self.prefix, key);
let client = SLED_CLIENT.clone().unwrap();
client.insert(key.as_str(), value.to_vec())?;
Ok(())
}
async fn delete(&self, key: &str, with_prefix: bool) -> Result<()> {
let key = format!("{}{}", self.prefix, key);
let client = SLED_CLIENT.clone().unwrap();
if !with_prefix {
let result = client.remove(&key)?;
if result.is_none() {
return Err(Error::from(DbError::KeyNotExists(key)));
}
return Ok(());
}
// prefix mod
let mut resp = client.scan_prefix(key);
loop {
let item = resp.next();
if item.is_none() {
break;
}
let item = item.unwrap();
let (k, _v) = item.unwrap();
client.remove(&k)?;
}
Ok(())
}
async fn list(&self, prefix: &str) -> Result<HashMap<String, Bytes>> {
let mut result = HashMap::default();
let key = format!("{}{}", self.prefix, prefix);
let client = SLED_CLIENT.clone().unwrap();
let mut resp = client.scan_prefix(key);
loop {
let item = resp.next();
if item.is_none() {
break;
}
let item = item.unwrap();
let (k, v) = item.unwrap();
let item_key = std::str::from_utf8(k.as_ref()).unwrap();
let item_key = item_key.strip_prefix(&self.prefix).unwrap();
result.insert(item_key.to_string(), Bytes::from(v.as_ref().to_vec()));
}
Ok(result)
}
async fn list_use_channel(&self, prefix: &str) -> Result<Arc<mpsc::Receiver<(String, Bytes)>>> {
let (tx, rx) = mpsc::channel(CONFIG.etcd.load_page_size as usize);
let key = format!("{}{}", self.prefix, prefix);
let prefix_key = self.prefix.to_string();
let _task: JoinHandle<Result<()>> = tokio::task::spawn(async move {
let client = SLED_CLIENT.clone().unwrap();
let mut resp = client.scan_prefix(key);
let mut load_num = 0;
loop {
let item = resp.next();
if item.is_none() {
break;
}
let item = item.unwrap();
let (k, v) = item.unwrap();
let item_key = std::str::from_utf8(k.as_ref()).unwrap();
let item_key = item_key.strip_prefix(&prefix_key).unwrap();
tx.send((item_key.to_string(), Bytes::from(v.as_ref().to_vec())))
.await
.unwrap();
load_num += 1;
if load_num % CONFIG.etcd.load_page_size == 0 {
tokio::task::yield_now().await; // yield to other tasks
}
}
Ok(())
});
Ok(Arc::new(rx))
}
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>> {
let mut result = Vec::new();
let key = format!("{}{}", self.prefix, prefix);
let client = SLED_CLIENT.clone().unwrap();
let mut resp = client.scan_prefix(key);
loop {
let item = resp.next();
if item.is_none() {
break;
}
let item = item.unwrap();
let (k, _v) = item.unwrap();
let item_key = std::str::from_utf8(k.as_ref()).unwrap();
let item_key = item_key.strip_prefix(&self.prefix).unwrap();
result.push(item_key.to_string());
}
Ok(result)
}
async fn list_values(&self, prefix: &str) -> Result<Vec<Bytes>> {
let mut result = Vec::new();
let key = format!("{}{}", self.prefix, prefix);
let client = SLED_CLIENT.clone().unwrap();
let mut resp = client.scan_prefix(key);
loop {
let item = resp.next();
if item.is_none() {
break;
}
let item = item.unwrap();
let (_k, v) = item.unwrap();
result.push(Bytes::from(v.as_ref().to_vec()));
}
Ok(result)
}
async fn watch(&self, prefix: &str) -> Result<Arc<mpsc::Receiver<Event>>> {
let (tx, rx) = mpsc::channel(1024);
let key = format!("{}{}", &self.prefix, prefix);
let prefix_key = self.prefix.to_string();
let _task: JoinHandle<Result<()>> = tokio::task::spawn(async move {
loop {
if cluster::is_offline() {
break;
}
let client = SLED_CLIENT.clone().unwrap();
let mut subscriber = client.watch_prefix(key.clone());
while let Some(event) = (&mut subscriber).await {
match event {
::sled::Event::Insert { key, value } => {
let item_key = std::str::from_utf8(key.as_ref()).unwrap();
let item_key = item_key.strip_prefix(&prefix_key.clone()).unwrap();
tx.send(Event::Put(EventData {
key: item_key.to_string(),
value: Some(Bytes::from(value.as_ref().to_vec())),
}))
.await
.unwrap();
}
::sled::Event::Remove { key } => {
let item_key = std::str::from_utf8(key.as_ref()).unwrap();
let item_key = item_key.strip_prefix(&prefix_key.clone()).unwrap();
tx.send(Event::Delete(EventData {
key: item_key.to_string(),
value: None,
}))
.await
.unwrap();
}
};
}
}
Ok(())
});
Ok(Arc::new(rx))
}
async fn transaction(
&self,
check_key: &str,
and_ops: Vec<Event>,
else_ops: Vec<Event>,
) -> Result<()> {
let db_span = info_span!("infra:db:sled:transaction");
let _guard = db_span.enter();
let client = SLED_CLIENT.clone().unwrap();
let mut check_result = false;
let key = format!("{}{}", self.prefix, check_key);
let ret = client.get(key);
if ret.is_err() {
return Err(Error::from(ret.err().unwrap()));
}
let ret = ret.unwrap();
if ret.is_some() {
check_result = true;
}
match client.transaction(|db| {
match check_result {
true => {
for op in &and_ops {
match op {
Event::Put(data) => {
let key = format!("{}{}", self.prefix, data.key);
let value = data.value.clone().unwrap();
if let Err(e) = db.insert(key.as_str(), value.to_vec()) {
return Err(ConflictableTransactionError::Abort(e));
}
}
Event::Delete(data) => {
let key = format!("{}{}", self.prefix, data.key);
if let Err(e) = db.remove(key.as_str()) {
return Err(ConflictableTransactionError::Abort(e));
}
}
}
}
}
false => {
for op in &else_ops {
match op {
Event::Put(data) => {
let key = format!("{}{}", self.prefix, data.key);
let value = data.value.clone().unwrap();
if let Err(e) = db.insert(key.as_str(), value.to_vec()) {
return Err(ConflictableTransactionError::Abort(e));
}
}
Event::Delete(data) => {
let key = format!("{}{}", self.prefix, data.key);
if let Err(e) = db.remove(key.as_str()) {
return Err(ConflictableTransactionError::Abort(e));
}
}
}
}
}
}
Ok(())
}) {
Ok(_) => Ok(()),
Err(e) => Err(Error::Message(e.to_string())),
}
}
}
pub fn connect_sled() -> Option<::sled::Db> {
if !CONFIG.common.local_mode {
return None;
}
let db = ::sled::open(&CONFIG.sled.data_dir)
.unwrap_or_else(|e| panic!("Error creating local sled db dir: {}", e));
Some(db)
}
src/infra/errors.rs
+31
View File
@@ -0,0 +1,31 @@
use thiserror::Error as ThisError;
pub type Result<T> = std::result::Result<T, Error>;
#[derive(ThisError, Debug)]
pub enum Error {
#[error("DbError# {0}")]
DbError(#[from] DbError),
#[error("EtcdError# {0}")]
EtcdError(#[from] etcd_client::Error),
#[error("SledError# {0}")]
SledError(#[from] sled::Error),
#[error("SerdeJsonError# {0}")]
SerdeJsonError(#[from] serde_json::Error),
#[error("SimdJsonError# {0}")]
SimdJsonError(#[from] simd_json::Error),
#[error("Error# watcher is exists {0}")]
WatcherExists(String),
#[error("Error# {0}")]
Message(String),
#[error("Not implemented")]
NotImplemented,
#[error("Unknown error")]
Unknown,
}
#[derive(ThisError, Debug)]
pub enum DbError {
#[error("key:{0} not exists")]
KeyNotExists(String),
}
src/infra/file_lock.rs
+274
View File
@@ -0,0 +1,274 @@
use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::sync::{Arc, RwLock};
use crate::infra::config::CONFIG;
use crate::infra::ider;
use crate::meta::StreamType;
lazy_static! {
pub static ref LOCKER: Locker = Locker::new();
}
pub struct Locker {
data: Arc<Vec<RwData>>,
}
type RwData = RwLock<HashMap<String, Arc<RwFile>>>;
#[derive(Debug)]
pub struct RwFile {
file: RwLock<File>,
dir: String,
name: String,
expired: i64,
}
#[inline(always)]
pub fn get_or_create(
thread_id: usize,
org_id: &str,
stream_name: &str,
stream_type: StreamType,
key: &str,
) -> Arc<RwFile> {
LOCKER.get_or_create(thread_id, org_id, stream_name, stream_type, key)
}
#[inline(always)]
pub fn check_in_use(
org_id: &str,
stream_name: &str,
stream_type: StreamType,
file_name: &str,
) -> bool {
LOCKER.check_in_use(org_id, stream_name, stream_type, file_name)
}
impl Default for Locker {
fn default() -> Self {
Self::new()
}
}
impl Locker {
pub fn new() -> Locker {
let size = CONFIG.limit.cpu_num;
let mut data = Vec::with_capacity(size);
for _i in 0..size {
data.push(RwLock::new(HashMap::<String, Arc<RwFile>>::new()));
}
Locker {
data: Arc::new(data),
}
}
pub fn get(
&self,
thread_id: usize,
org_id: &str,
stream_name: &str,
stream_type: StreamType,
key: &str,
) -> Option<Arc<RwFile>> {
let full_key = format!("{}_{}_{}_{}", org_id, stream_name, stream_type, key);
let file = match self
.data
.get(thread_id)
.unwrap()
.read()
.unwrap()
.get(&full_key)
{
Some(file) => file.clone(),
None => {
return None;
}
};
// check size & ttl
if file.size() >= (CONFIG.limit.max_file_size_on_disk as i64)
|| file.expired() <= chrono::Utc::now().timestamp()
{
file.sync();
self.data
.get(thread_id)
.unwrap()
.write()
.unwrap()
.remove(&full_key);
return None;
}
Some(file)
}
pub fn create(
&self,
thread_id: usize,
org_id: &str,
stream_name: &str,
stream_type: StreamType,
key: &str,
) -> Arc<RwFile> {
let full_key = format!("{}_{}_{}_{}", org_id, stream_name, stream_type, key);
let file = Arc::new(RwFile::new(
thread_id,
org_id,
stream_name,
stream_type,
key,
));
let mut data = self.data.get(thread_id).unwrap().write().unwrap();
data.insert(full_key, file.clone());
file
}
pub fn get_or_create(
&self,
thread_id: usize,
org_id: &str,
stream_name: &str,
stream_type: StreamType,
key: &str,
) -> Arc<RwFile> {
if let Some(file) = self.get(thread_id, org_id, stream_name, stream_type, key) {
file
} else {
self.create(thread_id, org_id, stream_name, stream_type, key)
}
}
pub fn check_in_use(
&self,
org_id: &str,
stream_name: &str,
stream_type: StreamType,
file_name: &str,
) -> bool {
let columns = file_name.split('_').collect::<Vec<&str>>();
let thread_id: usize = columns[0].parse().unwrap();
let key = columns[1..columns.len() - 1].join("_");
if let Some(file) = self.get(thread_id, org_id, stream_name, stream_type, &key) {
if file.name() == file_name {
return true;
}
}
false
}
}
impl RwFile {
fn new(
thread_id: usize,
org_id: &str,
stream_name: &str,
stream_type: StreamType,
key: &str,
) -> RwFile {
let mut dir_path = format!(
"{}files/{}/{}/{}/",
&CONFIG.common.data_wal_dir, org_id, stream_type, stream_name
);
// Hack for file_list
let file_list_prefix = "/files//file_list//";
if dir_path.contains(file_list_prefix) {
dir_path = dir_path.replace(file_list_prefix, "/file_list/");
}
let id = ider::generate();
let file_name = format!(
"{}_{}_{}{}",
thread_id, key, id, &CONFIG.common.file_ext_json
);
let file_path = format!("{}{}", dir_path, file_name);
std::fs::create_dir_all(&dir_path).unwrap();
let f = OpenOptions::new()
.write(true)
.create(true)
.append(true)
.open(file_path)
.unwrap();
RwFile {
file: RwLock::new(f),
dir: dir_path,
name: file_name,
expired: chrono::Utc::now().timestamp() + CONFIG.limit.max_file_retention_time as i64,
}
}
pub fn write(&self, data: &[u8]) {
let mut f = self.file.write().unwrap();
f.write_all(data).unwrap();
}
pub fn sync(&self) {
self.file.write().unwrap().sync_all().unwrap();
}
pub fn size(&self) -> i64 {
self.file.read().unwrap().metadata().unwrap().len() as i64
}
pub fn name(&self) -> &str {
&self.name
}
pub fn full_name(&self) -> String {
format!("{}{}", self.dir, self.name)
}
pub fn expired(&self) -> i64 {
self.expired
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_file_loc_with_stream() {
let locker = Locker::new();
let thread_id = 1;
let org_id = "org1";
let stream_name = "stream1";
let stream_type = StreamType::Logs;
let key = "2022_10_17_10";
let file = locker.get_or_create(thread_id, org_id, stream_name, stream_type, key);
file.write(b"hello world");
assert_eq!(file.size(), 11);
assert_eq!(file.name().contains(key), true);
assert_eq!(file.expired() > 0, true);
assert_eq!(
locker
.get(1, org_id, stream_name, stream_type, key)
.unwrap()
.name(),
file.name()
);
}
#[test]
fn test_file_loc_without_stream() {
let locker = Locker::new();
let thread_id = 1;
let org_id = "";
let stream_name = "";
let stream_type = StreamType::Filelist;
let key = "2022_10_17_10";
let file = locker.get_or_create(thread_id, org_id, stream_name, stream_type, key);
file.write(b"hello world");
assert_eq!(file.size(), 11);
assert_eq!(file.name().contains(key), true);
assert_eq!(file.expired() > 0, true);
assert_eq!(
locker
.get(1, org_id, stream_name, stream_type, key)
.unwrap()
.name(),
file.name()
);
}
}
src/infra/ider.rs
+21
View File
@@ -0,0 +1,21 @@
use super::cluster;
use once_cell::sync::Lazy;
use snowflake::SnowflakeIdGenerator;
static mut IDER: Lazy<SnowflakeIdGenerator> =
Lazy::new(|| unsafe { SnowflakeIdGenerator::new(1, cluster::LOCAL_NODE_ID) });
pub fn generate() -> String {
let id = unsafe { IDER.real_time_generate() };
id.to_string()
}
#[cfg(test)]
mod test_utils {
use super::*;
#[test]
fn test_generate_id() {
let id = generate();
assert_ne!(id, "");
}
}
src/infra/mod.rs
+8
View File
@@ -0,0 +1,8 @@
pub mod cache;
pub mod cluster;
pub mod config;
pub mod db;
pub mod errors;
pub mod file_lock;
pub mod ider;
pub mod storage;
src/infra/storage/local.rs
+56
View File
@@ -0,0 +1,56 @@
use async_trait::async_trait;
use std::{fs, io::Read, path::Path};
use super::FileStorage;
use crate::common::file::put_file_contents;
use crate::infra::config::CONFIG;
pub struct Local {}
#[async_trait]
impl FileStorage for Local {
async fn list(&self, prefix: &str) -> Result<Vec<String>, anyhow::Error> {
let prefix = format!("{}{}", CONFIG.common.data_stream_dir, prefix);
let root_path = Path::new(&CONFIG.common.data_stream_dir).to_str().unwrap();
let mut files = Vec::new();
for entry in walkdir::WalkDir::new(prefix)
.follow_links(true)
.into_iter()
.filter_map(|e| e.ok())
{
if entry.file_type().is_dir() {
continue;
}
let f_path = entry.path().to_str().unwrap();
let f_path = f_path[root_path.len()..].to_string();
files.push(f_path);
}
Ok(files)
}
async fn get(&self, file: &str) -> Result<bytes::Bytes, anyhow::Error> {
let file = format!("{}{}", CONFIG.common.data_stream_dir, file);
let mut file = fs::File::open(file)?;
let mut data = Vec::new();
file.read_to_end(&mut data)?;
Ok(bytes::Bytes::from(data))
}
async fn put(&self, file: &str, data: bytes::Bytes) -> Result<(), anyhow::Error> {
let file = format!("{}{}", CONFIG.common.data_stream_dir, file);
let file_path = Path::new(&file);
fs::create_dir_all(file_path.parent().unwrap()).unwrap();
match put_file_contents(&file, &data) {
Ok(_) => Ok(()),
Err(e) => Err(anyhow::anyhow!(e)),
}
}
async fn del(&self, file: &str) -> Result<(), anyhow::Error> {
let file = format!("{}{}", CONFIG.common.data_stream_dir, file);
match fs::remove_file(file) {
Ok(_) => Ok(()),
Err(e) => Err(anyhow::anyhow!(e)),
}
}
}
src/infra/storage/mod.rs
+66
View File
@@ -0,0 +1,66 @@
use async_trait::async_trait;
use bytes::Bytes;
use chrono::{TimeZone, Utc};
use crate::{common::utils::is_local_disk_storage, infra::ider, meta::StreamType};
pub mod local;
pub mod s3;
#[async_trait]
pub trait FileStorage: Sync + 'static {
async fn list(&self, prefix: &str) -> Result<Vec<String>, anyhow::Error>;
async fn get(&self, file: &str) -> Result<Bytes, anyhow::Error>;
async fn put(&self, file: &str, data: Bytes) -> Result<(), anyhow::Error>;
async fn del(&self, file: &str) -> Result<(), anyhow::Error>;
}
lazy_static! {
pub static ref DEFAULT: Box<dyn FileStorage> = default();
}
pub fn default() -> Box<dyn FileStorage> {
match is_local_disk_storage() {
true => Box::new(local::Local {}),
false => Box::new(s3::S3 {}),
}
}
pub fn generate_partioned_file_key(
org_id: &str,
stream_name: &str,
stream_type: StreamType,
min_ts: i64,
extn: &str,
) -> (std::string::String, std::string::String) {
let id = ider::generate();
let time = Utc.timestamp_nanos(min_ts * 1000);
let prefix = if stream_type.eq(&StreamType::Metadata) {
time.format("%Y/00/00/00").to_string()
} else {
time.format("%Y/%m/%d/%H").to_string()
};
(
format!("{}/{}/{}/{}/", org_id, stream_type, stream_name, prefix),
format!("{}{}", id, extn),
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::infra::config::CONFIG;
#[test]
fn test_generate_partioned_file_key() {
let file_key = generate_partioned_file_key(
"org",
"stream_name",
StreamType::Logs,
1665580243211047,
&CONFIG.common.file_ext_parquet,
);
assert_eq!(file_key.0.as_str(), "org/logs/stream_name/2022/10/12/13/");
assert!(file_key.1.as_str().contains(".parquet"));
}
}
src/infra/storage/s3.rs
+151
View File
@@ -0,0 +1,151 @@
use async_once::AsyncOnce;
use async_trait::async_trait;
use aws_config::{default_provider::credentials::DefaultCredentialsChain, SdkConfig};
use aws_sdk_s3::{Client, Credentials, Endpoint, Region};
use std::{sync::Arc, time::Duration};
use super::FileStorage;
use crate::common::utils::is_local_disk_storage;
use crate::infra::config::CONFIG;
lazy_static! {
static ref S3CONFIG: AsyncOnce<Option<Arc<SdkConfig>>> =
AsyncOnce::new(async { init_s3config().await });
}
pub struct S3 {}
#[async_trait]
impl FileStorage for S3 {
async fn list(&self, prefix: &str) -> Result<Vec<String>, anyhow::Error> {
let mut files = Vec::new();
let s3config = S3CONFIG.get().await.clone().unwrap().clone();
let client = Client::new(&s3config);
let mut start_after: String = "".to_string();
loop {
let objects = client
.list_objects_v2()
.bucket(&CONFIG.s3.bucket_name)
.prefix(prefix)
.start_after(&start_after)
.send()
.await;
if objects.is_err() {
return Err(anyhow::anyhow!("{}", objects.err().unwrap()));
}
let objects = objects.unwrap();
for obj in objects.contents().unwrap_or_default() {
start_after = obj.key().unwrap().to_string();
files.push(start_after.clone());
}
if objects.key_count < 1000 {
break;
}
}
Ok(files)
}
async fn get(&self, file: &str) -> Result<bytes::Bytes, anyhow::Error> {
let s3config = S3CONFIG.get().await.clone().unwrap();
let client = Client::new(&s3config);
let object = match client
.get_object()
.bucket(&CONFIG.s3.bucket_name)
.key(file)
.send()
.await
{
Ok(object) => object,
Err(e) => {
log::error!("s3 get object {} error: {:?}", file, e);
return Err(anyhow::anyhow!("s3 get object {} error: {:?}", file, e));
}
};
Ok(object.body.collect().await.unwrap().into_bytes())
}
async fn put(&self, file: &str, data: bytes::Bytes) -> Result<(), anyhow::Error> {
let s3config = S3CONFIG.get().await.clone().unwrap();
let client = Client::new(&s3config);
let result = client
.put_object()
.bucket(&CONFIG.s3.bucket_name)
.key(file)
.body(data.into())
.send()
.await;
match result {
Ok(_output) => {
log::info!("s3 File upload success: {}", file);
Ok(())
}
Err(err) => {
log::error!("s3 File upload error: {:?}", err);
Err(anyhow::anyhow!(err))
}
}
}
async fn del(&self, file: &str) -> Result<(), anyhow::Error> {
let s3config = S3CONFIG.get().await.clone().unwrap();
let client = Client::new(&s3config);
let result = client
.delete_object()
.bucket(&CONFIG.s3.bucket_name)
.key(file)
.send()
.await;
match result {
Ok(_output) => {
log::info!("s3 File delete success: {}", file);
Ok(())
}
Err(err) => {
log::error!("s3 File delete error: {:?}", err);
Err(anyhow::anyhow!(err))
}
}
}
}
async fn init_s3config() -> Option<Arc<SdkConfig>> {
if is_local_disk_storage() {
return None;
}
let mut s3config = aws_config::from_env();
if !CONFIG.s3.server_url.is_empty() {
s3config = s3config.endpoint_resolver(Endpoint::immutable(
CONFIG
.s3
.server_url
.parse()
.expect("invalid ZIOX_S3_SERVER_URL"),
));
}
if !CONFIG.s3.region_name.is_empty() {
s3config = s3config.region(Region::new(&CONFIG.s3.region_name));
}
if !CONFIG.s3.access_key.is_empty() {
let creds = Credentials::new(
&CONFIG.s3.access_key,
&CONFIG.s3.secret_key,
None,
None,
"s3",
);
s3config = s3config.credentials_provider(creds);
} else {
let creds = DefaultCredentialsChain::builder()
.load_timeout(Duration::from_secs(10))
.build()
.await;
s3config = s3config.credentials_provider(creds);
}
let s3config = s3config.load().await;
Some(Arc::new(s3config))
}
src/job/alert_manager.rs
+20
View File
@@ -0,0 +1,20 @@
use tokio::time;
use crate::infra::cluster::is_alert_manager;
use crate::service;
pub async fn run() -> Result<(), anyhow::Error> {
if !is_alert_manager(&super::cluster::LOCAL_NODE_ROLE) {
return Ok(());
}
// should run it every 10 seconds
let mut interval = time::interval(time::Duration::from_secs(10));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
let ret = service::alert_manager::run().await;
if ret.is_err() {
log::error!("[ALERT MANAGER] run error: {}", ret.err().unwrap());
}
}
}

Some files were not shown because too many files have changed in this diff Show More