rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-29 15:06:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: security fix, sqlx, hashbrown, idna and CI updates (#5330)

Browse Source 
* fix: security fix, sqlx, hashbrown, idna

* ci: optimize ci cache generation

* feat: update pprof
This commit is contained in:
Ning Sun
2025-01-14 08:19:33 +00:00
committed by GitHub
parent 87bd12d6df
commit f5829364a2
12 changed files with 597 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/develop.yml
+9 -36
View File
@@ -1,4 +1,6 @@
on:
schedule:
- cron: "0 15 * * 1-5"
merge_group:
pull_request:
types: [ opened, synchronize, reopened, ready_for_review ]
@@ -51,12 +53,6 @@ jobs:
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# # Shares with `Clippy` job
# shared-key: "check-lint"
- name: Run cargo check
run: cargo check --locked --workspace --all-targets
@@ -67,11 +63,6 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "check-toml"
- name: Install taplo
run: cargo +stable install taplo-cli --version ^0.9 --locked --force
- name: Run taplo
@@ -94,6 +85,8 @@ jobs:
with:
# Shares across multiple jobs
shared-key: "build-binaries"
cache-all-crates: "true"
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install cargo-gc-bin
shell: bash
run: cargo install cargo-gc-bin --force
@@ -142,11 +135,6 @@ jobs:
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "fuzz-test-targets"
- name: Set Rust Fuzz
shell: bash
run: |
@@ -200,11 +188,6 @@ jobs:
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "fuzz-test-targets"
- name: Set Rust Fuzz
shell: bash
run: |
@@ -255,6 +238,8 @@ jobs:
with:
# Shares across multiple jobs
shared-key: "build-greptime-ci"
cache-all-crates: "true"
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install cargo-gc-bin
shell: bash
run: cargo install cargo-gc-bin --force
@@ -317,11 +302,6 @@ jobs:
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "fuzz-test-targets"
- name: Set Rust Fuzz
shell: bash
run: |
@@ -466,11 +446,6 @@ jobs:
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "fuzz-test-targets"
- name: Set Rust Fuzz
shell: bash
run: |
@@ -612,11 +587,6 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "check-rust-fmt"
- name: Check format
run: make fmt-check
@@ -638,6 +608,8 @@ jobs:
# Shares across multiple jobs
# Shares with `Check` job
shared-key: "check-lint"
cache-all-crates: "true"
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Run cargo clippy
run: make clippy
@@ -670,6 +642,7 @@ jobs:
with:
# Shares cross multiple jobs
shared-key: "coverage-test"
cache-all-crates: "true"
save-if: ${{ github.ref == 'refs/heads/main' }}
# Disabled temporarily to see performance
# - name: Docker Cache
.github/workflows/nightly-ci.yml
-45
View File
@@ -108,51 +108,6 @@ jobs:
GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
UNITTEST_LOG_DIR: "__unittest_logs"
## this is designed for generating cache that usable for pull requests
test-on-linux:
name: Run tests on Linux
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
runs-on: ubuntu-20.04-8-cores
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: arduino/setup-protoc@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: rui314/setup-mold@v1
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Rust Cache
uses: Swatinem/rust-cache@v2
with:
# Shares cross multiple jobs
shared-key: "coverage-test"
- name: Install Cargo Nextest
uses: taiki-e/install-action@nextest
- name: Setup external services
working-directory: tests-integration/fixtures
run: docker compose up -d --wait
- name: Running tests
run: cargo nextest run -F dashboard -F pg_kvbackend
env:
CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
RUST_BACKTRACE: 1
CARGO_INCREMENTAL: 0
GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
GT_MINIO_BUCKET: greptime
GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
GT_MINIO_ACCESS_KEY: superpower_password
GT_MINIO_REGION: us-west-2
GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
UNITTEST_LOG_DIR: "__unittest_logs"
cleanbuild-linux-nix:
name: Run clean build on Linux
runs-on: ubuntu-latest-8-cores
Cargo.lock
Generated
+549 -287
View File
File diff suppressed because it is too large Load Diff
src/common/pprof/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ snafu.workspace = true
tokio.workspace = true
[target.'cfg(unix)'.dependencies]
pprof = { version = "0.13", features = [
pprof = { version = "0.14", features = [
"flamegraph",
"prost-codec",
"protobuf",
tests-fuzz/Cargo.toml
+1 -1
View File
@@ -53,7 +53,7 @@ serde_yaml = "0.9"
snafu = { workspace = true }
sql = { workspace = true }
sqlparser.workspace = true
sqlx = { version = "0.6", features = [
sqlx = { version = "0.8", features = [
"runtime-tokio-rustls",
"mysql",
"postgres",
tests-fuzz/src/utils.rs
+5 -2
View File
@@ -59,8 +59,11 @@ pub async fn init_greptime_connections_via_env() -> Connections {
/// Connects to GreptimeDB.
pub async fn init_greptime_connections(mysql: Option<String>) -> Connections {
let mysql = if let Some(addr) = mysql {
let mut opts: MySqlConnectOptions = format!("mysql://{addr}/public").parse().unwrap();
opts.log_statements(LevelFilter::Off);
let opts = format!("mysql://{addr}/public")
.parse::<MySqlConnectOptions>()
.unwrap()
.log_statements(LevelFilter::Off);
Some(MySqlPoolOptions::new().connect_with(opts).await.unwrap())
} else {
None
tests-fuzz/src/utils/cluster_info.rs
+5 -15
View File
@@ -17,8 +17,7 @@ use std::time::Duration;
use common_telemetry::info;
use humantime::parse_duration;
use snafu::ResultExt;
use sqlx::database::HasArguments;
use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, MySql, Pool, Type};
use sqlx::MySqlPool;
use super::wait::wait_condition_fn;
use crate::error::{self, Result};
@@ -34,19 +33,10 @@ pub struct NodeInfo {
}
/// Returns all [NodeInfo] in the cluster.
pub async fn fetch_nodes<'a, DB, E>(e: E) -> Result<Vec<NodeInfo>>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> i64: Decode<'c, DB> + Type<DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> String: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
let sql = "select * from information_schema.cluster_info;";
pub async fn fetch_nodes(db: &MySqlPool) -> Result<Vec<NodeInfo>> {
let sql = "select * from information_schema.cluster_info";
sqlx::query_as::<_, NodeInfo>(sql)
.fetch_all(e)
.fetch_all(db)
.await
.context(error::ExecuteQuerySnafu { sql })
}
@@ -55,7 +45,7 @@ where
///
/// This function repeatedly checks the status of all datanodes and waits until all of them are online
/// or the timeout period elapses. A datanode is considered online if its `active_time` is less than 3 seconds.
pub async fn wait_for_all_datanode_online(greptime: Pool<MySql>, timeout: Duration) {
pub async fn wait_for_all_datanode_online(greptime: MySqlPool, timeout: Duration) {
wait_condition_fn(
timeout,
|| {
tests-fuzz/src/utils/partition.rs
+10 -39
View File
@@ -17,8 +17,7 @@ use std::time::Duration;
use common_telemetry::info;
use snafu::ResultExt;
use sqlx::database::HasArguments;
use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, MySql, Pool, Type};
use sqlx::MySqlPool;
use store_api::storage::RegionId;
use super::wait::wait_condition_fn;
@@ -36,61 +35,33 @@ pub struct PartitionCount {
pub count: i64,
}
pub async fn count_partitions<'a, DB, E>(e: E, datanode_id: u64) -> Result<PartitionCount>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> i64: Decode<'c, DB> + Type<DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> u64: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
pub async fn count_partitions(db: &MySqlPool, datanode_id: u64) -> Result<PartitionCount> {
let sql = "select count(1) as count from information_schema.region_peers where peer_id == ?";
Ok(sqlx::query_as::<_, PartitionCount>(sql)
sqlx::query_as::<_, PartitionCount>(sql)
.bind(datanode_id)
.fetch_all(e)
.fetch_one(db)
.await
.context(error::ExecuteQuerySnafu { sql })?
.remove(0))
.context(error::ExecuteQuerySnafu { sql })
}
/// Returns the [Partition] of the specific `region_id`
pub async fn fetch_partition<'a, DB, E>(e: E, region_id: u64) -> Result<Partition>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> u64: Decode<'c, DB> + Type<DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> u64: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
pub async fn fetch_partition(db: &MySqlPool, region_id: u64) -> Result<Partition> {
let sql = "select region_id, peer_id as datanode_id from information_schema.region_peers where region_id = ?;";
sqlx::query_as::<_, Partition>(sql)
.bind(region_id)
.fetch_one(e)
.fetch_one(db)
.await
.context(error::ExecuteQuerySnafu { sql })
}
/// Returns all [Partition] of the specific `table`
pub async fn fetch_partitions<'a, DB, E>(e: E, table_name: Ident) -> Result<Vec<Partition>>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> u64: Decode<'c, DB> + Type<DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> String: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
pub async fn fetch_partitions(db: &MySqlPool, table_name: Ident) -> Result<Vec<Partition>> {
let sql = "select b.peer_id as datanode_id, a.greptime_partition_id as region_id
from information_schema.partitions a left join information_schema.region_peers b
on a.greptime_partition_id = b.region_id where a.table_name= ? order by datanode_id asc;";
sqlx::query_as::<_, Partition>(sql)
.bind(table_name.value.to_string())
.fetch_all(e)
.fetch_all(db)
.await
.context(error::ExecuteQuerySnafu { sql })
}
@@ -124,7 +95,7 @@ pub fn pretty_print_region_distribution(distribution: &BTreeMap<u64, Vec<RegionI
/// This function repeatedly checks the number of partitions on the specified datanode and waits until
/// the count reaches zero or the timeout period elapses. It logs the number of partitions on each check.
pub async fn wait_for_all_regions_evicted(
greptime: Pool<MySql>,
greptime: MySqlPool,
selected_datanode: u64,
timeout: Duration,
) {
tests-fuzz/src/validator/column.rs
+5 -14
View File
@@ -15,8 +15,7 @@
use common_telemetry::debug;
use datatypes::data_type::DataType;
use snafu::{ensure, ResultExt};
use sqlx::database::HasArguments;
use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, Type};
use sqlx::MySqlPool;
use crate::error::{self, Result};
use crate::ir::create_expr::ColumnOption;
@@ -198,24 +197,16 @@ pub fn assert_eq(fetched_columns: &[ColumnEntry], columns: &[Column]) -> Result<
}
/// Returns all [ColumnEntry] of the `table_name` from `information_schema`.
pub async fn fetch_columns<'a, DB, E>(
e: E,
pub async fn fetch_columns(
db: &MySqlPool,
schema_name: Ident,
table_name: Ident,
) -> Result<Vec<ColumnEntry>>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> String: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
) -> Result<Vec<ColumnEntry>> {
let sql = "SELECT table_schema, table_name, column_name, greptime_data_type as data_type, semantic_type, column_default, is_nullable FROM information_schema.columns WHERE table_schema = ? AND table_name = ?";
sqlx::query_as::<_, ColumnEntry>(sql)
.bind(schema_name.value.to_string())
.bind(table_name.value.to_string())
.fetch_all(e)
.fetch_all(db)
.await
.context(error::ExecuteQuerySnafu { sql })
}
tests-fuzz/src/validator/row.rs
+8 -27
View File
@@ -17,11 +17,8 @@ use common_time::date::Date;
use common_time::{DateTime, Timestamp};
use datatypes::value::Value;
use snafu::{ensure, ResultExt};
use sqlx::database::HasArguments;
use sqlx::{
Column, ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, Row, Type, TypeInfo,
ValueRef,
};
use sqlx::mysql::MySqlRow;
use sqlx::{Column, ColumnIndex, Database, MySqlPool, Row, TypeInfo, ValueRef};
use crate::error::{self, Result};
use crate::ir::insert_expr::{RowValue, RowValues};
@@ -151,33 +148,17 @@ pub struct ValueCount {
pub count: i64,
}
pub async fn count_values<'a, DB, E>(e: E, sql: &'a str) -> Result<ValueCount>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> i64: Decode<'c, DB> + Type<DB>,
for<'c> String: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
Ok(sqlx::query_as::<_, ValueCount>(sql)
.fetch_all(e)
pub async fn count_values(db: &MySqlPool, sql: &str) -> Result<ValueCount> {
sqlx::query_as::<_, ValueCount>(sql)
.fetch_one(db)
.await
.context(error::ExecuteQuerySnafu { sql })?
.remove(0))
.context(error::ExecuteQuerySnafu { sql })
}
/// Returns all [RowEntry] of the `table_name`.
pub async fn fetch_values<'a, DB, E>(e: E, sql: &'a str) -> Result<Vec<<DB as Database>::Row>>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> String: Encode<'c, DB> + Type<DB>,
{
pub async fn fetch_values(db: &MySqlPool, sql: &str) -> Result<Vec<MySqlRow>> {
sqlx::query(sql)
.fetch_all(e)
.fetch_all(db)
.await
.context(error::ExecuteQuerySnafu { sql })
}
tests-fuzz/src/validator/table.rs
+3 -12
View File
@@ -13,8 +13,7 @@
// limitations under the License.
use snafu::{ensure, ResultExt};
use sqlx::database::HasArguments;
use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, Row, Type};
use sqlx::{MySqlPool, Row};
use crate::error::{self, Result, UnexpectedSnafu};
use crate::ir::alter_expr::AlterTableOption;
@@ -47,17 +46,9 @@ fn parse_show_create(show_create: &str) -> Result<Vec<AlterTableOption>> {
}
/// Fetches table options from the context
pub async fn fetch_table_options<'a, DB, E>(e: E, sql: &'a str) -> Result<Vec<AlterTableOption>>
where
DB: Database,
<DB as HasArguments<'a>>::Arguments: IntoArguments<'a, DB>,
for<'c> E: 'a + Executor<'c, Database = DB>,
for<'c> String: Decode<'c, DB> + Type<DB>,
for<'c> String: Encode<'c, DB> + Type<DB>,
usize: ColumnIndex<<DB as Database>::Row>,
{
pub async fn fetch_table_options(db: &MySqlPool, sql: &str) -> Result<Vec<AlterTableOption>> {
let fetched_rows = sqlx::query(sql)
.fetch_all(e)
.fetch_all(db)
.await
.context(error::ExecuteQuerySnafu { sql })?;
ensure!(
tests-integration/Cargo.toml
+1 -1
View File
@@ -64,7 +64,7 @@ session.workspace = true
similar-asserts.workspace = true
snafu.workspace = true
sql.workspace = true
sqlx = { version = "0.6", features = [
sqlx = { version = "0.8", features = [
"runtime-tokio-rustls",
"mysql",
"postgres",