rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-06 05:12:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(metric-engine)!: Replace mur3 with fxhash for faster TSID generation (#7316)

Browse Source 
* feat/change-tsid-gen:
 perf(metric-engine): replace mur3 with fxhash for faster TSID generation

 - Switches from mur3::Hasher128 to fxhash::FxHasher for TSID hashing
 - Pre-computes label-name hash when no nulls are present, avoiding redundant work
 - Adds fast-path for rows without nulls; falls back to slow path otherwise
 - Updates Cargo.toml and lockfile to reflect dependency change

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>

* feat/change-tsid-gen:
 fix: only check primary-key labels for null when re-using cached hash

 - Rename has_null() → has_null_labels() and restrict the check to the
   primary-key columns so that non-label NULLs do not force a full
   TSID re-computation.
 - Update expected hashes in tests to match the new logic.

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>

* feat/change-tsid-gen:
 test: add comprehensive TSID generation tests for label ordering and null handling

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>

* feat/change-tsid-gen:
 bench: add criterion benchmark for TSID generator

 - Compare original mur3 vs current fxhash fast/slow paths
 - Test 2, 5, 10 label sets plus null-value slow path
 - Add mur3 & criterion dev-deps; register bench target

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>

* feat/change-tsid-gen:
 test: stabilize metric-engine tests by fixing non-deterministic row order

 - Add ORDER BY to SELECTs in TTL tests to ensure consistent output
 - Update expected __tsid values after hash function change
 - Swap expected OTLP metric rows to match new ordering

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>

* feat/change-tsid-gen:
 refactor: simplify Default impls and remove redundant code

 - Replace manual Default for TsidGenerator with derive
 - Remove unnecessary into_iter() call
 - Simplify Option::unwrap_or_else to unwrap_or

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>

---------

Signed-off-by: Lei, HUANG <mrsatangel@gmail.com>
This commit is contained in:
Lei, HUANG
2025-12-02 00:38:29 -08:00
committed by GitHub
parent 69f0249039
commit 931556dbd3
12 changed files with 658 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/cases/standalone/common/insert/logical_metric_table.result
View File

@@ -37,8 +37,8 @@ SELECT * from t2;
+------+-------------------------+-----+
| job | ts | val |
+------+-------------------------+-----+
| job2 | 1970-01-01T00:00:00.001 | 1.0 |
| job1 | 1970-01-01T00:00:00 | 0.0 |
| job2 | 1970-01-01T00:00:00.001 | 1.0 |
+------+-------------------------+-----+
DROP TABLE t1;
@@ -67,10 +67,10 @@ SELECT ts, val, __tsid, host, job FROM phy;
+-------------------------+-----+----------------------+-------+------+
| ts | val | __tsid | host | job |
+-------------------------+-----+----------------------+-------+------+
| 1970-01-01T00:00:00.001 | 1.0 | 1128149335081630826 | host2 | |
| 1970-01-01T00:00:00 | 0.0 | 18067404594631612786 | host1 | |
| 1970-01-01T00:00:00.001 | 1.0 | 2176048834144407834 | | job2 |
| 1970-01-01T00:00:00 | 0.0 | 15980333303142110493 | | job1 |
| 1970-01-01T00:00:00.001 | 1.0 | 7947983149541006936 | host2 | |
| 1970-01-01T00:00:00 | 0.0 | 13882403126406556045 | host1 | |
| 1970-01-01T00:00:00 | 0.0 | 6248409809737953425 | | job1 |
| 1970-01-01T00:00:00.001 | 1.0 | 12867770218286207316 | | job2 |
+-------------------------+-----+----------------------+-------+------+
DROP TABLE phy;
@@ -123,8 +123,8 @@ SELECT * from t2;
+------+-------------------------+-----+
| job | ts | val |
+------+-------------------------+-----+
| job2 | 1970-01-01T00:00:00.001 | 1.0 |
| job1 | 1970-01-01T00:00:00 | 0.0 |
| job2 | 1970-01-01T00:00:00.001 | 1.0 |
+------+-------------------------+-----+
ADMIN flush_table('phy');
@@ -154,10 +154,10 @@ SELECT * from t2;
+------+-------------------------+-----+
| job | ts | val |
+------+-------------------------+-----+
| job2 | 1970-01-01T00:00:00.001 | 1.0 |
| job3 | 1970-01-01T00:00:00 | 0.0 |
| job4 | 1970-01-01T00:00:00.001 | 1.0 |
| job1 | 1970-01-01T00:00:00 | 0.0 |
| job4 | 1970-01-01T00:00:00.001 | 1.0 |
| job2 | 1970-01-01T00:00:00.001 | 1.0 |
+------+-------------------------+-----+
DROP TABLE t1;

12
tests/cases/standalone/common/ttl/database_ttl_with_metric_engine.result
View File

@@ -22,14 +22,14 @@ INSERT INTO test_ttl(ts, val, host) VALUES
Affected Rows: 3
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
+-----+-------+
| val | host |
+-----+-------+
| 1.0 | host1 |
| 2.0 | host2 |
| 3.0 | host3 |
| 1.0 | host1 |
+-----+-------+
-- SQLNESS SLEEP 2s
@@ -83,26 +83,26 @@ ADMIN compact_table('phy');
+----------------------------+
--- should not be expired --
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
+-----+-------+
| val | host |
+-----+-------+
| 1.0 | host1 |
| 2.0 | host2 |
| 3.0 | host3 |
| 1.0 | host1 |
+-----+-------+
-- restart the db, ensure everything is ok
-- SQLNESS ARG restart=true
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
+-----+-------+
| val | host |
+-----+-------+
| 1.0 | host1 |
| 2.0 | host2 |
| 3.0 | host3 |
| 1.0 | host1 |
+-----+-------+
DROP TABLE test_ttl;

6
tests/cases/standalone/common/ttl/database_ttl_with_metric_engine.sql
View File

@@ -13,7 +13,7 @@ INSERT INTO test_ttl(ts, val, host) VALUES
(now(), 2, 'host2'),
(now(), 3, 'host3');
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
-- SQLNESS SLEEP 2s
ADMIN flush_table('phy');
@@ -35,11 +35,11 @@ ADMIN flush_table('phy');
ADMIN compact_table('phy');
--- should not be expired --
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
-- restart the db, ensure everything is ok
-- SQLNESS ARG restart=true
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
DROP TABLE test_ttl;

12
tests/cases/standalone/common/ttl/metric_engine_ttl.result
View File

@@ -13,14 +13,14 @@ INSERT INTO test_ttl(ts, val, host) VALUES
Affected Rows: 3
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
+-----+-------+
| val | host |
+-----+-------+
| 1.0 | host1 |
| 2.0 | host2 |
| 3.0 | host3 |
| 1.0 | host1 |
+-----+-------+
-- SQLNESS SLEEP 2s
@@ -74,26 +74,26 @@ ADMIN compact_table('phy');
+----------------------------+
--- should not be expired --
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
+-----+-------+
| val | host |
+-----+-------+
| 1.0 | host1 |
| 2.0 | host2 |
| 3.0 | host3 |
| 1.0 | host1 |
+-----+-------+
-- restart the db, ensure everything is ok
-- SQLNESS ARG restart=true
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
+-----+-------+
| val | host |
+-----+-------+
| 1.0 | host1 |
| 2.0 | host2 |
| 3.0 | host3 |
| 1.0 | host1 |
+-----+-------+
DROP TABLE test_ttl;

6
tests/cases/standalone/common/ttl/metric_engine_ttl.sql
View File

@@ -7,7 +7,7 @@ INSERT INTO test_ttl(ts, val, host) VALUES
(now(), 2, 'host2'),
(now(), 3, 'host3');
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
-- SQLNESS SLEEP 2s
ADMIN flush_table('phy');
@@ -29,11 +29,11 @@ ADMIN flush_table('phy');
ADMIN compact_table('phy');
--- should not be expired --
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
-- restart the db, ensure everything is ok
-- SQLNESS ARG restart=true
SELECT val, host FROM test_ttl;
SELECT val, host FROM test_ttl ORDER BY host;
DROP TABLE test_ttl;