insert some rows and query them across DDL to test the integrity of data

fix: fix bug
fix: test program
2026-01-03 20:02:54 +00:00 · 2024-05-14 20:41:47 +08:00 · 2024-05-13 08:25:18 +00:00 · 2024-05-13 07:46:30 +00:00 · 2024-05-13 02:44:34 +00:00 · 2024-04-19 14:14:34 +00:00
59 changed files with 2868 additions and 1208 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -957,15 +957,6 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"

-[[package]]
-name = "bitpacking"
-version = "0.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92"
-dependencies = [
- "crunchy",
-]
-
 [[package]]
 name = "bitvec"
 version = "1.0.1"
@@ -1083,6 +1074,12 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "bufstream"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
+
 [[package]]
 name = "build-data"
 version = "0.1.5"
@@ -1312,12 +1309,6 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "060303ef31ef4a522737e1b1ab68c67916f2a787bb2f4f54f383279adba962b5"

-[[package]]
-name = "census"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
-
 [[package]]
 name = "cesu8"
 version = "1.1.0"
@@ -2550,6 +2541,7 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
 [[package]]
 name = "datafusion"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "ahash 0.8.6",
 "arrow",
@@ -2596,6 +2588,7 @@ dependencies = [
 [[package]]
 name = "datafusion-common"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "ahash 0.8.6",
 "arrow",
@@ -2613,6 +2606,7 @@ dependencies = [
 [[package]]
 name = "datafusion-execution"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "arrow",
 "chrono",
@@ -2632,6 +2626,7 @@ dependencies = [
 [[package]]
 name = "datafusion-expr"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "ahash 0.8.6",
 "arrow",
@@ -2645,6 +2640,7 @@ dependencies = [
 [[package]]
 name = "datafusion-optimizer"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "arrow",
 "async-trait",
@@ -2661,6 +2657,7 @@ dependencies = [
 [[package]]
 name = "datafusion-physical-expr"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "ahash 0.8.6",
 "arrow",
@@ -2693,6 +2690,7 @@ dependencies = [
 [[package]]
 name = "datafusion-physical-plan"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "ahash 0.8.6",
 "arrow",
@@ -2722,6 +2720,7 @@ dependencies = [
 [[package]]
 name = "datafusion-sql"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "arrow",
 "arrow-schema",
@@ -2734,6 +2733,7 @@ dependencies = [
 [[package]]
 name = "datafusion-substrait"
 version = "32.0.0"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9"
 dependencies = [
 "async-recursion",
 "chrono",
@@ -2963,6 +2963,17 @@ dependencies = [
 "syn 1.0.109",
 ]

+[[package]]
+name = "derive_utils"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61bb5a1014ce6dfc2a378578509abe775a5aa06bff584a547555d9efdb81b926"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.43",
+]
+
 [[package]]
 name = "diff"
 version = "0.1.13"
@@ -3094,12 +3105,6 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"

-[[package]]
-name = "downcast-rs"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
-
 [[package]]
 name = "dyn-clone"
 version = "1.0.16"
@@ -3269,12 +3274,6 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"

-[[package]]
-name = "fastdivide"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b"
-
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -3443,6 +3442,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -3584,16 +3598,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "fs4"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21dabded2e32cd57ded879041205c60a4a4c4bab47bd0fd2fa8b01f30849f02b"
-dependencies = [
- "rustix 0.38.28",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "fsevent-sys"
 version = "4.1.0"
@@ -3721,19 +3725,6 @@ dependencies = [
 "slab",
 ]

-[[package]]
-name = "generator"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e"
-dependencies = [
- "cc",
- "libc",
- "log",
- "rustversion",
- "windows 0.48.0",
-]
-
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -4020,12 +4011,6 @@ dependencies = [
 "utf8-width",
 ]

-[[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
 [[package]]
 name = "http"
 version = "0.2.11"
@@ -4326,7 +4311,6 @@ dependencies = [
 "regex",
 "regex-automata 0.4.3",
 "snafu",
- "tantivy",
 "tempfile",
 "tokio",
 "tokio-util",
@@ -4462,6 +4446,15 @@ version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8573b2b1fb643a372c73b23f4da5f888677feef3305146d68a539250a9bccc7"

+[[package]]
+name = "io-enum"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53b53d712d99a73eec59ee5e4fe6057f8052142d38eeafbbffcb06b36d738a6e"
+dependencies = [
+ "derive_utils",
+]
+
 [[package]]
 name = "io-lifetimes"
 version = "1.0.11"
@@ -4706,12 +4699,6 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"

-[[package]]
-name = "levenshtein_automata"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
-
 [[package]]
 name = "lexical-core"
 version = "0.8.5"
@@ -4924,20 +4911,6 @@ dependencies = [
 "uuid",
 ]

-[[package]]
-name = "loom"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
-dependencies = [
- "cfg-if 1.0.0",
- "generator",
- "pin-utils",
- "scoped-tls",
- "tracing",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "lrlex"
 version = "0.12.0"
@@ -5029,12 +5002,6 @@ dependencies = [
 "twox-hash",
 ]

-[[package]]
-name = "lz4_flex"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
-
 [[package]]
 name = "lzma-sys"
 version = "0.1.20"
@@ -5133,16 +5100,6 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"

-[[package]]
-name = "measure_time"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56220900f1a0923789ecd6bf25fbae8af3b2f1ff3e9e297fc9b6b8674dd4d852"
-dependencies = [
- "instant",
- "log",
-]
-
 [[package]]
 name = "memchr"
 version = "2.6.4"
@@ -5512,10 +5469,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97af489e1e21b68de4c390ecca6703318bc1aa16e9733bcb62c089b73c6fbb1b"

 [[package]]
-name = "murmurhash32"
-version = "0.3.1"
+name = "mysql"
+version = "25.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+checksum = "a4cc09a8118051e4617886c9c6e693c61444c2eeb5f9a792dc5d631501706565"
+dependencies = [
+ "bufstream",
+ "bytes",
+ "crossbeam",
+ "flate2",
+ "io-enum",
+ "libc",
+ "lru",
+ "mysql_common 0.32.0",
+ "named_pipe",
+ "native-tls",
+ "once_cell",
+ "pem",
+ "percent-encoding",
+ "serde",
+ "serde_json",
+ "socket2 0.5.5",
+ "twox-hash",
+ "url",
+]

 [[package]]
 name = "mysql-common-derive"
@@ -5698,6 +5675,33 @@ dependencies = [
 "syn 1.0.109",
 ]

+[[package]]
+name = "named_pipe"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "ndk-context"
 version = "0.1.1"
@@ -6038,15 +6042,6 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"

-[[package]]
-name = "oneshot"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f6640c6bda7731b1fdbab747981a0f896dd1fedaf9f4a53fa237a04a84431f4"
-dependencies = [
- "loom",
-]
-
 [[package]]
 name = "oorandom"
 version = "11.1.3"
@@ -6111,12 +6106,50 @@ dependencies = [
 "tokio-rustls 0.25.0",
 ]

+[[package]]
+name = "openssl"
+version = "0.10.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
+dependencies = [
+ "bitflags 2.4.1",
+ "cfg-if 1.0.0",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.43",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"

+[[package]]
+name = "openssl-sys"
+version = "0.9.102"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.21.0"
@@ -6392,15 +6425,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"

-[[package]]
-name = "ownedbytes"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
-dependencies = [
- "stable_deref_trait",
-]
-
 [[package]]
 name = "packedvec"
 version = "1.2.4"
@@ -8210,16 +8234,6 @@ dependencies = [
 "tree-sitter-cli",
 ]

-[[package]]
-name = "rust-stemmers"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
-dependencies = [
- "serde",
- "serde_derive",
-]
-
 [[package]]
 name = "rust_decimal"
 version = "1.33.1"
@@ -8450,7 +8464,7 @@ dependencies = [
 "bitflags 1.3.2",
 "bstr",
 "itertools 0.10.5",
- "lz4_flex 0.9.5",
+ "lz4_flex",
 "num-bigint",
 "num-complex",
 ]
@@ -8841,12 +8855,6 @@ dependencies = [
 "syn 1.0.109",
 ]

-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -9383,15 +9391,6 @@ dependencies = [
 "walkdir",
 ]

-[[package]]
-name = "sketches-ddsketch"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -9925,7 +9924,6 @@ dependencies = [
 "bytes",
 "catalog",
 "common-error",
- "common-function",
 "common-macro",
 "datafusion",
 "datafusion-common",
@@ -9934,7 +9932,6 @@ dependencies = [
 "datatypes",
 "promql",
 "prost 0.12.3",
- "session",
 "snafu",
 "substrait 0.17.1",
 "tokio",
@@ -10127,148 +10124,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"

-[[package]]
-name = "tantivy"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856"
-dependencies = [
- "aho-corasick",
- "arc-swap",
- "base64 0.22.0",
- "bitpacking",
- "byteorder",
- "census",
- "crc32fast",
- "crossbeam-channel",
- "downcast-rs",
- "fastdivide",
- "fnv",
- "fs4",
- "htmlescape",
- "itertools 0.12.0",
- "levenshtein_automata",
- "log",
- "lru",
- "lz4_flex 0.11.3",
- "measure_time",
- "memmap2 0.9.3",
- "num_cpus",
- "once_cell",
- "oneshot",
- "rayon",
- "regex",
- "rust-stemmers",
- "rustc-hash",
- "serde",
- "serde_json",
- "sketches-ddsketch",
- "smallvec",
- "tantivy-bitpacker",
- "tantivy-columnar",
- "tantivy-common",
- "tantivy-fst",
- "tantivy-query-grammar",
- "tantivy-stacker",
- "tantivy-tokenizer-api",
- "tempfile",
- "thiserror",
- "time",
- "uuid",
- "winapi",
- "zstd 0.13.0",
-]
-
-[[package]]
-name = "tantivy-bitpacker"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df"
-dependencies = [
- "bitpacking",
-]
-
-[[package]]
-name = "tantivy-columnar"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
-dependencies = [
- "downcast-rs",
- "fastdivide",
- "itertools 0.12.0",
- "serde",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-sstable",
- "tantivy-stacker",
-]
-
-[[package]]
-name = "tantivy-common"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
-dependencies = [
- "async-trait",
- "byteorder",
- "ownedbytes",
- "serde",
- "time",
-]
-
-[[package]]
-name = "tantivy-fst"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
-dependencies = [
- "byteorder",
- "regex-syntax 0.8.2",
- "utf8-ranges",
-]
-
-[[package]]
-name = "tantivy-query-grammar"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82"
-dependencies = [
- "nom",
-]
-
-[[package]]
-name = "tantivy-sstable"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
-dependencies = [
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-fst",
- "zstd 0.13.0",
-]
-
-[[package]]
-name = "tantivy-stacker"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
-dependencies = [
- "murmurhash32",
- "rand_distr",
- "tantivy-common",
-]
-
-[[package]]
-name = "tantivy-tokenizer-api"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -10338,6 +10193,32 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"

+[[package]]
+name = "tests-chaos"
+version = "0.7.2"
+dependencies = [
+ "axum",
+ "axum-macros",
+ "common-error",
+ "common-macro",
+ "common-telemetry",
+ "common-time",
+ "lazy_static",
+ "mysql",
+ "nix 0.26.4",
+ "prometheus",
+ "rand",
+ "rand_chacha",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "snafu",
+ "sqlx",
+ "tests-fuzz",
+ "tinytemplate",
+ "tokio",
+]
+
 [[package]]
 name = "tests-fuzz"
 version = "0.7.2"
@@ -10355,6 +10236,7 @@ dependencies = [
 "dotenv",
 "lazy_static",
 "libfuzzer-sys",
+ "mysql",
 "partition",
 "rand",
 "rand_chacha",
@@ -11588,12 +11470,6 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"

-[[package]]
-name = "utf8-ranges"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
-
 [[package]]
 name = "utf8-width"
 version = "0.1.7"
@@ -11949,15 +11825,6 @@ dependencies = [
 "windows_x86_64_msvc 0.39.0",
 ]

-[[package]]
-name = "windows"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
-dependencies = [
- "windows-targets 0.48.5",
-]
-
 [[package]]
 name = "windows"
 version = "0.52.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,6 +55,7 @@ members = [
    "src/store-api",
    "src/table",
    "src/index",
+    "tests-chaos",
    "tests-fuzz",
    "tests-integration",
    "tests/runner",
@@ -91,20 +92,13 @@ bytes = { version = "1.5", features = ["serde"] }
 chrono = { version = "0.4", features = ["serde"] }
 clap = { version = "4.4", features = ["derive"] }
 dashmap = "5.4"
-# datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-# datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-# datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-# datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-# datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-# datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-# datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
-datafusion = { path = "../arrow-datafusion/datafusion/core" }
-datafusion-common = { path = "../arrow-datafusion/datafusion/common" }
-datafusion-expr = { path = "../arrow-datafusion/datafusion/expr" }
-datafusion-optimizer = { path = "../arrow-datafusion/datafusion/optimizer" }
-datafusion-physical-expr = { path = "../arrow-datafusion/datafusion/physical-expr" }
-datafusion-sql = { path = "../arrow-datafusion/datafusion/sql" }
-datafusion-substrait = { path = "../arrow-datafusion/datafusion/substrait" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 derive_builder = "0.12"
 dotenv = "0.15"
 etcd-client = "0.12"
@@ -219,6 +213,7 @@ sql = { path = "src/sql" }
 store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
+tests-fuzz = { path = "tests-fuzz" }

 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
--- a/feed.py
+++ b/feed.py
@@ -1,54 +0,0 @@
-# read line from log-1000.txt and POST it to http://localhost:4000/v1/influxdb/write?db=public&precision=ms
-# POST data format: "many_logs,host=1 log=<FILE CONTENT> <INCREMENT ID>"
-
-import requests
-from tqdm import tqdm
-from concurrent.futures import ThreadPoolExecutor
-
-batch_size = 3000
-worker = 8
-
-# Define the URL
-url = "http://localhost:4000/v1/influxdb/write?db=public&precision=ms"
-
-
-def send_data(start, data):
-    # Send the POST request
-    response = requests.post(url, data=data)
-    # Check the response
-    if response.status_code >= 300:
-        print(
-            f"Failed to send log line {start}: {response.status_code} {response.text}"
-        )
-
-
-# Open the file
-with open("target/log-1000.txt", "r") as file:
-    lines = file.readlines()
-
-# Create a progress bar
-with tqdm(
-    total=len(lines),
-    desc="Processing lines",
-    bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt}",
-) as pbar:
-    data = ""
-    with ThreadPoolExecutor(max_workers=worker) as executor:
-        for i, line in enumerate(lines):
-            # Prepare the POST data
-            content = line.strip()
-            content = content.replace('"', " ")
-            content = content.replace("'", " ")
-            content = content.replace("=", " ")
-            content = content.replace(".", " ")
-
-            data = data + f'many_logs,host=1 log="{content}" {i}\n'
-
-            if i % batch_size == 0:
-                executor.submit(send_data, i, data)
-                data = ""
-                # Update the progress bar
-                pbar.update(batch_size)
-
-        # close the executor
-        executor.shutdown(wait=True)
--- a/src/cmd/src/cli/bench/metadata.rs
+++ b/src/cmd/src/cli/bench/metadata.rs
@@ -107,14 +107,11 @@ impl TableMetadataBencher {
                    .unwrap();
                let start = Instant::now();
                let table_info = table_info.unwrap();
+                let table_route = table_route.unwrap();
                let table_id = table_info.table_info.ident.table_id;
                let _ = self
                    .table_metadata_manager
-                    .delete_table_metadata(
-                        table_id,
-                        &table_info.table_name(),
-                        table_route.unwrap().region_routes().unwrap(),
-                    )
+                    .delete_table_metadata(table_id, &table_info.table_name(), &table_route)
                    .await;
                start.elapsed()
            },
--- a/src/common/function/src/scalars.rs
+++ b/src/common/function/src/scalars.rs
@@ -15,7 +15,6 @@
 pub mod aggregate;
 pub(crate) mod date;
 pub mod expression;
-pub mod matches;
 pub mod math;
 pub mod numpy;
 #[cfg(test)]
--- a/src/common/function/src/scalars/matches.rs
+++ b/src/common/function/src/scalars/matches.rs
@@ -1,61 +0,0 @@
-// Copyright 2024 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fmt;
-use std::fmt::Display;
-use std::sync::Arc;
-
-use common_query::error::Result;
-use common_query::prelude::{Signature, Volatility};
-use datatypes::prelude::ConcreteDataType;
-use datatypes::vectors::{BooleanVector, VectorRef};
-
-use crate::function::{Function, FunctionContext};
-
-const NAME: &str = "matches";
-
-/// The function to find remainders
-#[derive(Clone, Debug, Default)]
-pub struct MatchesFunction;
-
-impl Display for MatchesFunction {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", NAME.to_ascii_uppercase())
-    }
-}
-
-impl Function for MatchesFunction {
-    fn name(&self) -> &str {
-        NAME
-    }
-
-    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::boolean_datatype())
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::exact(
-            vec![
-                ConcreteDataType::string_datatype(),
-                ConcreteDataType::string_datatype(),
-            ],
-            Volatility::Immutable,
-        )
-    }
-
-    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        let num_rows = columns[1].len();
-        Ok(Arc::new(BooleanVector::from(vec![true; num_rows])))
-    }
-}
--- a/src/common/function/src/scalars/math.rs
+++ b/src/common/function/src/scalars/math.rs
@@ -31,7 +31,6 @@ pub use pow::PowFunction;
 pub use rate::RateFunction;
 use snafu::ResultExt;

-use super::matches::MatchesFunction;
 use crate::function::{Function, FunctionContext};
 use crate::function_registry::FunctionRegistry;
 use crate::scalars::math::modulo::ModuloFunction;
@@ -45,7 +44,6 @@ impl MathFunction {
        registry.register(Arc::new(RateFunction));
        registry.register(Arc::new(RangeFunction));
        registry.register(Arc::new(ClampFunction));
-        registry.register(Arc::new(MatchesFunction));
    }
 }

--- a/src/common/meta/src/ddl/drop_database/cursor.rs
+++ b/src/common/meta/src/ddl/drop_database/cursor.rs
@@ -76,6 +76,7 @@ impl DropDatabaseCursor {
                    .await?;
                Ok((
                    Box::new(DropDatabaseExecutor::new(
+                        table_id,
                        table_id,
                        TableName::new(&ctx.catalog, &ctx.schema, &table_name),
                        table_route.region_routes,
@@ -86,6 +87,7 @@ impl DropDatabaseCursor {
            }
            (DropTableTarget::Physical, TableRouteValue::Physical(table_route)) => Ok((
                Box::new(DropDatabaseExecutor::new(
+                    table_id,
                    table_id,
                    TableName::new(&ctx.catalog, &ctx.schema, &table_name),
                    table_route.region_routes,
@@ -220,7 +222,7 @@ mod tests {
            .get_physical_table_route(physical_table_id)
            .await
            .unwrap();
-        assert_eq!(table_route.region_routes, executor.region_routes);
+        assert_eq!(table_route.region_routes, executor.physical_region_routes);
        assert_eq!(executor.target, DropTableTarget::Logical);
    }

--- a/src/common/meta/src/ddl/drop_database/executor.rs
+++ b/src/common/meta/src/ddl/drop_database/executor.rs
@@ -26,6 +26,7 @@ use crate::ddl::drop_database::State;
 use crate::ddl::drop_table::executor::DropTableExecutor;
 use crate::ddl::DdlContext;
 use crate::error::{self, Result};
+use crate::key::table_route::TableRouteValue;
 use crate::region_keeper::OperatingRegionGuard;
 use crate::rpc::router::{operating_leader_regions, RegionRoute};
 use crate::table_name::TableName;
@@ -33,8 +34,10 @@ use crate::table_name::TableName;
 #[derive(Debug, Serialize, Deserialize)]
 pub(crate) struct DropDatabaseExecutor {
    table_id: TableId,
+    physical_table_id: TableId,
    table_name: TableName,
-    pub(crate) region_routes: Vec<RegionRoute>,
+    /// The physical table region routes.
+    pub(crate) physical_region_routes: Vec<RegionRoute>,
    pub(crate) target: DropTableTarget,
    #[serde(skip)]
    dropping_regions: Vec<OperatingRegionGuard>,
@@ -44,14 +47,16 @@ impl DropDatabaseExecutor {
    /// Returns a new [DropDatabaseExecutor].
    pub fn new(
        table_id: TableId,
+        physical_table_id: TableId,
        table_name: TableName,
-        region_routes: Vec<RegionRoute>,
+        physical_region_routes: Vec<RegionRoute>,
        target: DropTableTarget,
    ) -> Self {
        Self {
-            table_name,
            table_id,
-            region_routes,
+            physical_table_id,
+            table_name,
+            physical_region_routes,
            target,
            dropping_regions: vec![],
        }
@@ -60,7 +65,7 @@ impl DropDatabaseExecutor {

 impl DropDatabaseExecutor {
    fn register_dropping_regions(&mut self, ddl_ctx: &DdlContext) -> Result<()> {
-        let dropping_regions = operating_leader_regions(&self.region_routes);
+        let dropping_regions = operating_leader_regions(&self.physical_region_routes);
        let mut dropping_region_guards = Vec::with_capacity(dropping_regions.len());
        for (region_id, datanode_id) in dropping_regions {
            let guard = ddl_ctx
@@ -87,12 +92,18 @@ impl State for DropDatabaseExecutor {
    ) -> Result<(Box<dyn State>, Status)> {
        self.register_dropping_regions(ddl_ctx)?;
        let executor = DropTableExecutor::new(self.table_name.clone(), self.table_id, true);
+        // Deletes metadata for table permanently.
+        let table_route_value = TableRouteValue::new(
+            self.table_id,
+            self.physical_table_id,
+            self.physical_region_routes.clone(),
+        );
        executor
-            .on_remove_metadata(ddl_ctx, &self.region_routes)
+            .on_destroy_metadata(ddl_ctx, &table_route_value)
            .await?;
        executor.invalidate_table_cache(ddl_ctx).await?;
        executor
-            .on_drop_regions(ddl_ctx, &self.region_routes)
+            .on_drop_regions(ddl_ctx, &self.physical_region_routes)
            .await?;
        info!("Table: {}({}) is dropped", self.table_name, self.table_id);

@@ -122,7 +133,9 @@ mod tests {
    use crate::ddl::drop_database::{DropDatabaseContext, DropTableTarget, State};
    use crate::ddl::test_util::{create_logical_table, create_physical_table};
    use crate::error::{self, Error, Result};
+    use crate::key::datanode_table::DatanodeTableKey;
    use crate::peer::Peer;
+    use crate::rpc::router::region_distribution;
    use crate::table_name::TableName;
    use crate::test_util::{new_ddl_context, MockDatanodeHandler, MockDatanodeManager};

@@ -157,6 +170,7 @@ mod tests {
            .unwrap();
        {
            let mut state = DropDatabaseExecutor::new(
+                physical_table_id,
                physical_table_id,
                TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "phy"),
                table_route.region_routes.clone(),
@@ -181,9 +195,10 @@ mod tests {
            tables: None,
        };
        let mut state = DropDatabaseExecutor::new(
+            physical_table_id,
            physical_table_id,
            TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "phy"),
-            table_route.region_routes,
+            table_route.region_routes.clone(),
            DropTableTarget::Physical,
        );
        let (state, status) = state.next(&ddl_context, &mut ctx).await.unwrap();
@@ -207,6 +222,7 @@ mod tests {
            .unwrap();
        {
            let mut state = DropDatabaseExecutor::new(
+                logical_table_id,
                physical_table_id,
                TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "metric"),
                table_route.region_routes.clone(),
@@ -231,8 +247,9 @@ mod tests {
            tables: None,
        };
        let mut state = DropDatabaseExecutor::new(
+            logical_table_id,
            physical_table_id,
-            TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "phy"),
+            TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "metric"),
            table_route.region_routes,
            DropTableTarget::Logical,
        );
@@ -240,6 +257,33 @@ mod tests {
        assert!(!status.need_persist());
        let cursor = state.as_any().downcast_ref::<DropDatabaseCursor>().unwrap();
        assert_eq!(cursor.target, DropTableTarget::Logical);
+        // Checks table info
+        ddl_context
+            .table_metadata_manager
+            .table_info_manager()
+            .get(physical_table_id)
+            .await
+            .unwrap()
+            .unwrap();
+        // Checks table route
+        let table_route = ddl_context
+            .table_metadata_manager
+            .table_route_manager()
+            .table_route_storage()
+            .get(physical_table_id)
+            .await
+            .unwrap()
+            .unwrap();
+        let region_routes = table_route.region_routes().unwrap();
+        for datanode_id in region_distribution(region_routes).into_keys() {
+            ddl_context
+                .table_metadata_manager
+                .datanode_table_manager()
+                .get(&DatanodeTableKey::new(datanode_id, physical_table_id))
+                .await
+                .unwrap()
+                .unwrap();
+        }
    }

    #[derive(Clone)]
@@ -279,6 +323,7 @@ mod tests {
            .await
            .unwrap();
        let mut state = DropDatabaseExecutor::new(
+            physical_table_id,
            physical_table_id,
            TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "phy"),
            table_route.region_routes,
--- a/src/common/meta/src/ddl/drop_table.rs
+++ b/src/common/meta/src/ddl/drop_table.rs
@@ -18,9 +18,11 @@ mod metadata;
 use async_trait::async_trait;
 use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
 use common_procedure::{
-    Context as ProcedureContext, LockKey, Procedure, Result as ProcedureResult, Status,
+    Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure,
+    Result as ProcedureResult, Status,
 };
 use common_telemetry::info;
+use common_telemetry::tracing::warn;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 use strum::AsRefStr;
@@ -31,9 +33,7 @@ use self::executor::DropTableExecutor;
 use crate::ddl::utils::handle_retry_error;
 use crate::ddl::DdlContext;
 use crate::error::{self, Result};
-use crate::key::table_info::TableInfoValue;
 use crate::key::table_route::TableRouteValue;
-use crate::key::DeserializedValueWithBytes;
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
 use crate::metrics;
 use crate::region_keeper::OperatingRegionGuard;
@@ -47,46 +47,50 @@ pub struct DropTableProcedure {
    pub data: DropTableData,
    /// The guards of opening regions.
    pub dropping_regions: Vec<OperatingRegionGuard>,
+    /// The drop table executor.
+    executor: DropTableExecutor,
 }

 impl DropTableProcedure {
    pub const TYPE_NAME: &'static str = "metasrv-procedure::DropTable";

    pub fn new(cluster_id: u64, task: DropTableTask, context: DdlContext) -> Self {
+        let data = DropTableData::new(cluster_id, task);
+        let executor = data.build_executor();
        Self {
            context,
-            data: DropTableData::new(cluster_id, task),
+            data,
            dropping_regions: vec![],
+            executor,
        }
    }

    pub fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
-        let data = serde_json::from_str(json).context(FromJsonSnafu)?;
+        let data: DropTableData = serde_json::from_str(json).context(FromJsonSnafu)?;
+        let executor = data.build_executor();
        Ok(Self {
            context,
            data,
            dropping_regions: vec![],
+            executor,
        })
    }

-    pub(crate) async fn on_prepare<'a>(&mut self, executor: &DropTableExecutor) -> Result<Status> {
-        if executor.on_prepare(&self.context).await?.stop() {
+    pub(crate) async fn on_prepare<'a>(&mut self) -> Result<Status> {
+        if self.executor.on_prepare(&self.context).await?.stop() {
            return Ok(Status::done());
        }
        self.fill_table_metadata().await?;
-        self.data.state = DropTableState::RemoveMetadata;
+        self.data.state = DropTableState::DeleteMetadata;

        Ok(Status::executing(true))
    }

    /// Register dropping regions if doesn't exist.
    fn register_dropping_regions(&mut self) -> Result<()> {
-        // Safety: filled in `on_prepare`.
-        let region_routes = self.data.region_routes().unwrap()?;
+        let dropping_regions = operating_leader_regions(&self.data.physical_region_routes);

-        let dropping_regions = operating_leader_regions(region_routes);
-
-        if self.dropping_regions.len() == dropping_regions.len() {
+        if !self.dropping_regions.is_empty() {
            return Ok(());
        }

@@ -109,7 +113,7 @@ impl DropTableProcedure {
    }

    /// Removes the table metadata.
-    async fn on_remove_metadata(&mut self, executor: &DropTableExecutor) -> Result<Status> {
+    pub(crate) async fn on_delete_metadata(&mut self) -> Result<Status> {
        self.register_dropping_regions()?;
        // NOTES: If the meta server is crashed after the `RemoveMetadata`,
        // Corresponding regions of this table on the Datanode will be closed automatically.
@@ -117,12 +121,15 @@ impl DropTableProcedure {

        // TODO(weny): Considers introducing a RegionStatus to indicate the region is dropping.
        let table_id = self.data.table_id();
-        executor
-            .on_remove_metadata(
-                &self.context,
-                // Safety: filled in `on_prepare`.
-                self.data.region_routes().unwrap()?,
-            )
+        let table_route_value = &TableRouteValue::new(
+            self.data.task.table_id,
+            // Safety: checked
+            self.data.physical_table_id.unwrap(),
+            self.data.physical_region_routes.clone(),
+        );
+        // Deletes table metadata logically.
+        self.executor
+            .on_delete_metadata(&self.context, table_route_value)
            .await?;
        info!("Deleted table metadata for table {table_id}");
        self.data.state = DropTableState::InvalidateTableCache;
@@ -130,30 +137,33 @@ impl DropTableProcedure {
    }

    /// Broadcasts invalidate table cache instruction.
-    async fn on_broadcast(&mut self, executor: &DropTableExecutor) -> Result<Status> {
-        executor.invalidate_table_cache(&self.context).await?;
+    async fn on_broadcast(&mut self) -> Result<Status> {
+        self.executor.invalidate_table_cache(&self.context).await?;
        self.data.state = DropTableState::DatanodeDropRegions;

        Ok(Status::executing(true))
    }

-    pub async fn on_datanode_drop_regions(&self, executor: &DropTableExecutor) -> Result<Status> {
-        executor
-            .on_drop_regions(
-                &self.context,
-                // Safety: filled in `on_prepare`.
-                self.data.region_routes().unwrap()?,
-            )
+    pub async fn on_datanode_drop_regions(&mut self) -> Result<Status> {
+        self.executor
+            .on_drop_regions(&self.context, &self.data.physical_region_routes)
            .await?;
-        Ok(Status::done())
+        self.data.state = DropTableState::DeleteTombstone;
+        Ok(Status::executing(true))
    }

-    pub(crate) fn executor(&self) -> DropTableExecutor {
-        DropTableExecutor::new(
-            self.data.task.table_name(),
-            self.data.table_id(),
-            self.data.task.drop_if_exists,
-        )
+    /// Deletes metadata tombstone.
+    async fn on_delete_metadata_tombstone(&self) -> Result<Status> {
+        let table_route_value = &TableRouteValue::new(
+            self.data.task.table_id,
+            // Safety: checked
+            self.data.physical_table_id.unwrap(),
+            self.data.physical_region_routes.clone(),
+        );
+        self.executor
+            .on_delete_metadata_tombstone(&self.context, table_route_value)
+            .await?;
+        Ok(Status::done())
    }
 }

@@ -164,17 +174,17 @@ impl Procedure for DropTableProcedure {
    }

    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
-        let executor = self.executor();
        let state = &self.data.state;
        let _timer = metrics::METRIC_META_PROCEDURE_DROP_TABLE
            .with_label_values(&[state.as_ref()])
            .start_timer();

        match self.data.state {
-            DropTableState::Prepare => self.on_prepare(&executor).await,
-            DropTableState::RemoveMetadata => self.on_remove_metadata(&executor).await,
-            DropTableState::InvalidateTableCache => self.on_broadcast(&executor).await,
-            DropTableState::DatanodeDropRegions => self.on_datanode_drop_regions(&executor).await,
+            DropTableState::Prepare => self.on_prepare().await,
+            DropTableState::DeleteMetadata => self.on_delete_metadata().await,
+            DropTableState::InvalidateTableCache => self.on_broadcast().await,
+            DropTableState::DatanodeDropRegions => self.on_datanode_drop_regions().await,
+            DropTableState::DeleteTombstone => self.on_delete_metadata_tombstone().await,
        }
        .map_err(handle_retry_error)
    }
@@ -194,6 +204,28 @@ impl Procedure for DropTableProcedure {

        LockKey::new(lock_key)
    }
+
+    fn rollback_supported(&self) -> bool {
+        !matches!(self.data.state, DropTableState::Prepare)
+    }
+
+    async fn rollback(&mut self, _: &ProcedureContext) -> ProcedureResult<()> {
+        warn!(
+            "Rolling back the drop table procedure, table: {}",
+            self.data.table_id()
+        );
+
+        let table_route_value = &TableRouteValue::new(
+            self.data.task.table_id,
+            // Safety: checked
+            self.data.physical_table_id.unwrap(),
+            self.data.physical_region_routes.clone(),
+        );
+        self.executor
+            .on_restore_metadata(&self.context, table_route_value)
+            .await
+            .map_err(ProcedureError::external)
+    }
 }

 #[derive(Debug, Serialize, Deserialize)]
@@ -201,8 +233,8 @@ pub struct DropTableData {
    pub state: DropTableState,
    pub cluster_id: u64,
    pub task: DropTableTask,
-    pub table_route_value: Option<DeserializedValueWithBytes<TableRouteValue>>,
-    pub table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
+    pub physical_region_routes: Vec<RegionRoute>,
+    pub physical_table_id: Option<TableId>,
 }

 impl DropTableData {
@@ -211,8 +243,8 @@ impl DropTableData {
            state: DropTableState::Prepare,
            cluster_id,
            task,
-            table_route_value: None,
-            table_info_value: None,
+            physical_region_routes: vec![],
+            physical_table_id: None,
        }
    }

@@ -220,13 +252,17 @@ impl DropTableData {
        self.task.table_ref()
    }

-    fn region_routes(&self) -> Option<Result<&Vec<RegionRoute>>> {
-        self.table_route_value.as_ref().map(|v| v.region_routes())
-    }
-
    fn table_id(&self) -> TableId {
        self.task.table_id
    }
+
+    fn build_executor(&self) -> DropTableExecutor {
+        DropTableExecutor::new(
+            self.task.table_name(),
+            self.task.table_id,
+            self.task.drop_if_exists,
+        )
+    }
 }

 /// The state of drop table.
@@ -234,10 +270,12 @@ impl DropTableData {
 pub enum DropTableState {
    /// Prepares to drop the table
    Prepare,
-    /// Removes metadata
-    RemoveMetadata,
+    /// Deletes metadata logically
+    DeleteMetadata,
    /// Invalidates Table Cache
    InvalidateTableCache,
    /// Drops regions on Datanode
    DatanodeDropRegions,
+    /// Deletes metadata tombstone permanently
+    DeleteTombstone,
 }
--- a/src/common/meta/src/ddl/drop_table/executor.rs
+++ b/src/common/meta/src/ddl/drop_table/executor.rs
@@ -30,6 +30,7 @@ use crate::ddl::DdlContext;
 use crate::error::{self, Result};
 use crate::instruction::CacheIdent;
 use crate::key::table_name::TableNameKey;
+use crate::key::table_route::TableRouteValue;
 use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
 use crate::table_name::TableName;

@@ -99,14 +100,73 @@ impl DropTableExecutor {
        Ok(Control::Continue(()))
    }

-    /// Removes the table metadata.
-    pub async fn on_remove_metadata(
+    /// Deletes the table metadata **logically**.
+    pub async fn on_delete_metadata(
        &self,
        ctx: &DdlContext,
-        region_routes: &[RegionRoute],
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        let table_name_key = TableNameKey::new(
+            &self.table.catalog_name,
+            &self.table.schema_name,
+            &self.table.table_name,
+        );
+        if !ctx
+            .table_metadata_manager
+            .table_name_manager()
+            .exists(table_name_key)
+            .await?
+        {
+            return Ok(());
+        }
+        ctx.table_metadata_manager
+            .delete_table_metadata(self.table_id, &self.table, table_route_value)
+            .await
+    }
+
+    /// Deletes the table metadata tombstone **permanently**.
+    pub async fn on_delete_metadata_tombstone(
+        &self,
+        ctx: &DdlContext,
+        table_route_value: &TableRouteValue,
    ) -> Result<()> {
        ctx.table_metadata_manager
-            .delete_table_metadata(self.table_id, &self.table, region_routes)
+            .delete_table_metadata_tombstone(self.table_id, &self.table, table_route_value)
+            .await
+    }
+
+    /// Deletes metadata for table **permanently**.
+    pub async fn on_destroy_metadata(
+        &self,
+        ctx: &DdlContext,
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        ctx.table_metadata_manager
+            .destroy_table_metadata(self.table_id, &self.table, table_route_value)
+            .await
+    }
+
+    /// Restores the table metadata.
+    pub async fn on_restore_metadata(
+        &self,
+        ctx: &DdlContext,
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        let table_name_key = TableNameKey::new(
+            &self.table.catalog_name,
+            &self.table.schema_name,
+            &self.table.table_name,
+        );
+        if ctx
+            .table_metadata_manager
+            .table_name_manager()
+            .exists(table_name_key)
+            .await?
+        {
+            return Ok(());
+        }
+        ctx.table_metadata_manager
+            .restore_table_metadata(self.table_id, &self.table, table_route_value)
            .await
    }

--- a/src/common/meta/src/ddl/drop_table/metadata.rs
+++ b/src/common/meta/src/ddl/drop_table/metadata.rs
@@ -12,35 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use common_catalog::format_full_table_name;
-use snafu::OptionExt;
-
 use crate::ddl::drop_table::DropTableProcedure;
-use crate::error::{self, Result};
+use crate::error::Result;

 impl DropTableProcedure {
-    /// Fetches the table info and table route.
+    /// Fetches the table info and physical table route.
    pub(crate) async fn fill_table_metadata(&mut self) -> Result<()> {
        let task = &self.data.task;
-        let table_info_value = self
-            .context
-            .table_metadata_manager
-            .table_info_manager()
-            .get(task.table_id)
-            .await?
-            .with_context(|| error::TableInfoNotFoundSnafu {
-                table: format_full_table_name(&task.catalog, &task.schema, &task.table),
-            })?;
-        let (_, table_route_value) = self
+        let (physical_table_id, physical_table_route_value) = self
            .context
            .table_metadata_manager
            .table_route_manager()
-            .table_route_storage()
-            .get_raw_physical_table_route(task.table_id)
+            .get_physical_table_route(task.table_id)
            .await?;

-        self.data.table_info_value = Some(table_info_value);
-        self.data.table_route_value = Some(table_route_value);
+        self.data.physical_region_routes = physical_table_route_value.region_routes;
+        self.data.physical_table_id = Some(physical_table_id);
+
        Ok(())
    }
 }
--- a/src/common/meta/src/ddl/tests/drop_table.rs
+++ b/src/common/meta/src/ddl/tests/drop_table.rs
@@ -19,17 +19,25 @@ use api::v1::region::{region_request, RegionRequest};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId};
+use common_procedure_test::MockContextProvider;
 use store_api::storage::RegionId;
 use tokio::sync::mpsc;

+use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
 use crate::ddl::drop_table::DropTableProcedure;
 use crate::ddl::test_util::create_table::test_create_table_task;
-use crate::ddl::test_util::datanode_handler::DatanodeWatcher;
+use crate::ddl::test_util::datanode_handler::{DatanodeWatcher, NaiveDatanodeHandler};
+use crate::ddl::test_util::{
+    create_physical_table_metadata, test_create_logical_table_task, test_create_physical_table_task,
+};
+use crate::ddl::{TableMetadata, TableMetadataAllocatorContext};
 use crate::key::table_route::TableRouteValue;
+use crate::kv_backend::memory::MemoryKvBackend;
 use crate::peer::Peer;
 use crate::rpc::ddl::DropTableTask;
 use crate::rpc::router::{Region, RegionRoute};
-use crate::test_util::{new_ddl_context, MockDatanodeManager};
+use crate::test_util::{new_ddl_context, new_ddl_context_with_kv_backend, MockDatanodeManager};

 #[tokio::test]
 async fn test_on_prepare_table_not_exists_err() {
@@ -59,8 +67,7 @@ async fn test_on_prepare_table_not_exists_err() {
    };

    let mut procedure = DropTableProcedure::new(cluster_id, task, ddl_context);
-    let executor = procedure.executor();
-    let err = procedure.on_prepare(&executor).await.unwrap_err();
+    let err = procedure.on_prepare().await.unwrap_err();
    assert_eq!(err.status_code(), StatusCode::TableNotFound);
 }

@@ -93,8 +100,7 @@ async fn test_on_prepare_table() {

    // Drop if exists
    let mut procedure = DropTableProcedure::new(cluster_id, task, ddl_context.clone());
-    let executor = procedure.executor();
-    procedure.on_prepare(&executor).await.unwrap();
+    procedure.on_prepare().await.unwrap();

    let task = DropTableTask {
        catalog: DEFAULT_CATALOG_NAME.to_string(),
@@ -106,8 +112,7 @@ async fn test_on_prepare_table() {

    // Drop table
    let mut procedure = DropTableProcedure::new(cluster_id, task, ddl_context);
-    let executor = procedure.executor();
-    procedure.on_prepare(&executor).await.unwrap();
+    procedure.on_prepare().await.unwrap();
 }

 #[tokio::test]
@@ -162,9 +167,8 @@ async fn test_on_datanode_drop_regions() {
    };
    // Drop table
    let mut procedure = DropTableProcedure::new(cluster_id, task, ddl_context);
-    let executor = procedure.executor();
-    procedure.on_prepare(&executor).await.unwrap();
-    procedure.on_datanode_drop_regions(&executor).await.unwrap();
+    procedure.on_prepare().await.unwrap();
+    procedure.on_datanode_drop_regions().await.unwrap();

    let check = |peer: Peer,
                 request: RegionRequest,
@@ -191,3 +195,97 @@ async fn test_on_datanode_drop_regions() {
    let (peer, request) = results.remove(0);
    check(peer, request, 3, RegionId::new(table_id, 3));
 }
+
+#[tokio::test]
+async fn test_on_rollback() {
+    let datanode_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
+    let kv_backend = Arc::new(MemoryKvBackend::new());
+    let ddl_context = new_ddl_context_with_kv_backend(datanode_manager, kv_backend.clone());
+    let cluster_id = 1;
+    // Prepares physical table metadata.
+    let mut create_physical_table_task = test_create_physical_table_task("phy_table");
+    let TableMetadata {
+        table_id,
+        table_route,
+        ..
+    } = ddl_context
+        .table_metadata_allocator
+        .create(
+            &TableMetadataAllocatorContext { cluster_id },
+            &create_physical_table_task,
+        )
+        .await
+        .unwrap();
+    create_physical_table_task.set_table_id(table_id);
+    create_physical_table_metadata(
+        &ddl_context,
+        create_physical_table_task.table_info.clone(),
+        TableRouteValue::Physical(table_route),
+    )
+    .await;
+    // The create logical table procedure.
+    let physical_table_id = table_id;
+    // Creates the logical table metadata.
+    let task = test_create_logical_table_task("foo");
+    let mut procedure = CreateLogicalTablesProcedure::new(
+        cluster_id,
+        vec![task],
+        physical_table_id,
+        ddl_context.clone(),
+    );
+    procedure.on_prepare().await.unwrap();
+    let ctx = ProcedureContext {
+        procedure_id: ProcedureId::random(),
+        provider: Arc::new(MockContextProvider::default()),
+    };
+    procedure.execute(&ctx).await.unwrap();
+    // Triggers procedure to create table metadata
+    let status = procedure.execute(&ctx).await.unwrap();
+    let table_ids = status.downcast_output_ref::<Vec<u32>>().unwrap();
+    assert_eq!(*table_ids, vec![1025]);
+
+    let expected_kvs = kv_backend.dump();
+    // Drops the physical table
+    {
+        let task = DropTableTask {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "phy_table".to_string(),
+            table_id: physical_table_id,
+            drop_if_exists: false,
+        };
+        let mut procedure = DropTableProcedure::new(cluster_id, task, ddl_context.clone());
+        procedure.on_prepare().await.unwrap();
+        procedure.on_delete_metadata().await.unwrap();
+        let ctx = ProcedureContext {
+            procedure_id: ProcedureId::random(),
+            provider: Arc::new(MockContextProvider::default()),
+        };
+        procedure.rollback(&ctx).await.unwrap();
+        // Rollback again
+        procedure.rollback(&ctx).await.unwrap();
+        let kvs = kv_backend.dump();
+        assert_eq!(kvs, expected_kvs);
+    }
+
+    // Drops the logical table
+    let task = DropTableTask {
+        catalog: DEFAULT_CATALOG_NAME.to_string(),
+        schema: DEFAULT_SCHEMA_NAME.to_string(),
+        table: "foo".to_string(),
+        table_id: table_ids[0],
+        drop_if_exists: false,
+    };
+    let mut procedure = DropTableProcedure::new(cluster_id, task, ddl_context.clone());
+    procedure.on_prepare().await.unwrap();
+    procedure.on_delete_metadata().await.unwrap();
+    let ctx = ProcedureContext {
+        procedure_id: ProcedureId::random(),
+        provider: Arc::new(MockContextProvider::default()),
+    };
+    procedure.rollback(&ctx).await.unwrap();
+    // Rollback again
+    procedure.rollback(&ctx).await.unwrap();
+    let kvs = kv_backend.dump();
+    assert_eq!(kvs, expected_kvs);
+}
--- a/src/common/meta/src/error.rs
+++ b/src/common/meta/src/error.rs
@@ -421,6 +421,9 @@ pub enum Error {
    #[snafu(display("Invalid role: {}", role))]
    InvalidRole { role: i32, location: Location },

+    #[snafu(display("Atomic key changed: {err_msg}"))]
+    CasKeyChanged { err_msg: String, location: Location },
+
    #[snafu(display("Failed to parse {} from utf8", name))]
    FromUtf8 {
        name: String,
@@ -440,7 +443,8 @@ impl ErrorExt for Error {
            | EtcdTxnOpResponse { .. }
            | EtcdFailed { .. }
            | EtcdTxnFailed { .. }
-            | ConnectEtcd { .. } => StatusCode::Internal,
+            | ConnectEtcd { .. }
+            | CasKeyChanged { .. } => StatusCode::Internal,

            SerdeJson { .. }
            | ParseOption { .. }
--- a/src/common/meta/src/key.rs
+++ b/src/common/meta/src/key.rs
@@ -56,9 +56,12 @@ pub mod table_region;
 pub mod table_route;
 #[cfg(any(test, feature = "testing"))]
 pub mod test_utils;
+// TODO(weny): remove it.
+#[allow(dead_code)]
+mod tombstone;
 mod txn_helper;

-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt::Debug;
 use std::ops::Deref;
 use std::sync::Arc;
@@ -83,9 +86,13 @@ use self::catalog_name::{CatalogManager, CatalogNameKey, CatalogNameValue};
 use self::datanode_table::RegionInfo;
 use self::schema_name::{SchemaManager, SchemaNameKey, SchemaNameValue};
 use self::table_route::{TableRouteManager, TableRouteValue};
+use self::tombstone::TombstoneManager;
 use crate::ddl::utils::region_storage_path;
-use crate::error::{self, Result, SerdeJsonSnafu};
-use crate::kv_backend::txn::{Txn, TxnOpResponse};
+use crate::error::{self, Result, SerdeJsonSnafu, UnexpectedSnafu};
+use crate::key::table_route::TableRouteKey;
+use crate::key::tombstone::Key;
+use crate::key::txn_helper::TxnOpGetResponseSet;
+use crate::kv_backend::txn::{Txn, TxnOp, TxnOpResponse};
 use crate::kv_backend::KvBackendRef;
 use crate::rpc::router::{region_distribution, RegionRoute, RegionStatus};
 use crate::table_name::TableName;
@@ -97,7 +104,6 @@ pub const MAINTENANCE_KEY: &str = "maintenance";
 const DATANODE_TABLE_KEY_PREFIX: &str = "__dn_table";
 const TABLE_REGION_KEY_PREFIX: &str = "__table_region";

-pub const REMOVED_PREFIX: &str = "__removed";
 pub const TABLE_INFO_KEY_PREFIX: &str = "__table_info";
 pub const TABLE_NAME_KEY_PREFIX: &str = "__table_name";
 pub const CATALOG_NAME_KEY_PREFIX: &str = "__catalog_name";
@@ -145,6 +151,33 @@ pub trait TableMetaKey {
    fn as_raw_key(&self) -> Vec<u8>;
 }

+pub(crate) trait TableMetaKeyGetTxnOp {
+    fn build_get_op(
+        &self,
+    ) -> (
+        TxnOp,
+        impl for<'a> FnMut(&'a mut TxnOpGetResponseSet) -> Option<Vec<u8>>,
+    );
+}
+
+impl TableMetaKey for String {
+    fn as_raw_key(&self) -> Vec<u8> {
+        self.as_bytes().to_vec()
+    }
+}
+
+impl TableMetaKeyGetTxnOp for String {
+    fn build_get_op(
+        &self,
+    ) -> (
+        TxnOp,
+        impl for<'a> FnMut(&'a mut TxnOpGetResponseSet) -> Option<Vec<u8>>,
+    ) {
+        let key = self.as_raw_key();
+        (TxnOp::Get(key.clone()), TxnOpGetResponseSet::filter(key))
+    }
+}
+
 pub trait TableMetaValue {
    fn try_from_raw_value(raw_value: &[u8]) -> Result<Self>
    where
@@ -162,6 +195,7 @@ pub struct TableMetadataManager {
    catalog_manager: CatalogManager,
    schema_manager: SchemaManager,
    table_route_manager: TableRouteManager,
+    tombstone_manager: TombstoneManager,
    kv_backend: KvBackendRef,
 }

@@ -303,6 +337,7 @@ impl TableMetadataManager {
            catalog_manager: CatalogManager::new(kv_backend.clone()),
            schema_manager: SchemaManager::new(kv_backend.clone()),
            table_route_manager: TableRouteManager::new(kv_backend.clone()),
+            tombstone_manager: TombstoneManager::new(kv_backend.clone()),
            kv_backend,
        }
    }
@@ -363,19 +398,16 @@ impl TableMetadataManager {
        Option<DeserializedValueWithBytes<TableInfoValue>>,
        Option<DeserializedValueWithBytes<TableRouteValue>>,
    )> {
-        let (get_table_route_txn, table_route_decoder) = self
-            .table_route_manager
-            .table_route_storage()
-            .build_get_txn(table_id);
-        let (get_table_info_txn, table_info_decoder) =
-            self.table_info_manager.build_get_txn(table_id);
-
-        let txn = Txn::merge_all(vec![get_table_route_txn, get_table_info_txn]);
-        let res = self.kv_backend.txn(txn).await?;
-
-        let table_info_value = table_info_decoder(&res.responses)?;
-        let table_route_value = table_route_decoder(&res.responses)?;
+        let table_info_key = TableInfoKey::new(table_id);
+        let table_route_key = TableRouteKey::new(table_id);
+        let (table_info_txn, table_info_filter) = table_info_key.build_get_op();
+        let (table_route_txn, table_route_filter) = table_route_key.build_get_op();

+        let txn = Txn::new().and_then(vec![table_info_txn, table_route_txn]);
+        let mut res = self.kv_backend.txn(txn).await?;
+        let mut set = TxnOpGetResponseSet::from(&mut res.responses);
+        let table_info_value = TxnOpGetResponseSet::decode_with(table_info_filter)(&mut set)?;
+        let table_route_value = TxnOpGetResponseSet::decode_with(table_route_filter)(&mut set)?;
        Ok((table_info_value, table_route_value))
    }

@@ -545,47 +577,106 @@ impl TableMetadataManager {
        Ok(())
    }

-    /// Deletes metadata for table.
-    /// The caller MUST ensure it has the exclusive access to `TableNameKey`.
-    pub async fn delete_table_metadata(
+    fn table_metadata_keys(
        &self,
        table_id: TableId,
        table_name: &TableName,
-        region_routes: &[RegionRoute],
-    ) -> Result<()> {
-        // Deletes table name.
+        table_route_value: &TableRouteValue,
+    ) -> Result<Vec<Key>> {
+        // Builds keys
+        let datanode_ids = if table_route_value.is_physical() {
+            region_distribution(table_route_value.region_routes()?)
+                .into_keys()
+                .collect()
+        } else {
+            vec![]
+        };
+        let mut keys = Vec::with_capacity(3 + datanode_ids.len());
        let table_name = TableNameKey::new(
            &table_name.catalog_name,
            &table_name.schema_name,
            &table_name.table_name,
        );
+        let table_info_key = TableInfoKey::new(table_id);
+        let table_route_key = TableRouteKey::new(table_id);
+        let datanode_table_keys = datanode_ids
+            .into_iter()
+            .map(|datanode_id| DatanodeTableKey::new(datanode_id, table_id))
+            .collect::<HashSet<_>>();

-        let delete_table_name_txn = self.table_name_manager().build_delete_txn(&table_name)?;
+        keys.push(Key::compare_and_swap(table_name.as_raw_key()));
+        keys.push(Key::new(table_info_key.as_raw_key()));
+        keys.push(Key::new(table_route_key.as_raw_key()));
+        for key in &datanode_table_keys {
+            keys.push(Key::new(key.as_raw_key()));
+        }
+        Ok(keys)
+    }

-        // Deletes table info.
-        let delete_table_info_txn = self.table_info_manager().build_delete_txn(table_id)?;
+    /// Deletes metadata for table **logically**.
+    /// The caller MUST ensure it has the exclusive access to `TableNameKey`.
+    pub async fn delete_table_metadata(
+        &self,
+        table_id: TableId,
+        table_name: &TableName,
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        let keys = self.table_metadata_keys(table_id, table_name, table_route_value)?;
+        self.tombstone_manager.create(keys).await?;
+        Ok(())
+    }

-        // Deletes datanode table key value pairs.
-        let distribution = region_distribution(region_routes);
-        let delete_datanode_txn = self
-            .datanode_table_manager()
-            .build_delete_txn(table_id, distribution)?;
+    /// Deletes metadata tombstone for table **permanently**.
+    /// The caller MUST ensure it has the exclusive access to `TableNameKey`.
+    pub async fn delete_table_metadata_tombstone(
+        &self,
+        table_id: TableId,
+        table_name: &TableName,
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        let keys = self
+            .table_metadata_keys(table_id, table_name, table_route_value)?
+            .into_iter()
+            .map(|key| key.into_bytes())
+            .collect::<Vec<_>>();
+        self.tombstone_manager.delete(keys).await
+    }

-        // Deletes table route.
-        let delete_table_route_txn = self
-            .table_route_manager()
-            .table_route_storage()
-            .build_delete_txn(table_id)?;
+    /// Restores metadata for table.
+    /// The caller MUST ensure it has the exclusive access to `TableNameKey`.
+    pub async fn restore_table_metadata(
+        &self,
+        table_id: TableId,
+        table_name: &TableName,
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        let keys = self.table_metadata_keys(table_id, table_name, table_route_value)?;
+        self.tombstone_manager.restore(keys).await?;
+        Ok(())
+    }

-        let txn = Txn::merge_all(vec![
-            delete_table_name_txn,
-            delete_table_info_txn,
-            delete_datanode_txn,
-            delete_table_route_txn,
-        ]);
+    /// Deletes metadata for table **permanently**.
+    /// The caller MUST ensure it has the exclusive access to `TableNameKey`.
+    pub async fn destroy_table_metadata(
+        &self,
+        table_id: TableId,
+        table_name: &TableName,
+        table_route_value: &TableRouteValue,
+    ) -> Result<()> {
+        let operations = self
+            .table_metadata_keys(table_id, table_name, table_route_value)?
+            .into_iter()
+            .map(|key| TxnOp::Delete(key.into_bytes()))
+            .collect::<Vec<_>>();

-        // It's always successes.
-        let _ = self.kv_backend.txn(txn).await?;
+        let txn = Txn::new().and_then(operations);
+        let resp = self.kv_backend.txn(txn).await?;
+        ensure!(
+            resp.succeeded,
+            UnexpectedSnafu {
+                err_msg: format!("Failed to destroy table metadata: {table_id}")
+            }
+        );

        Ok(())
    }
@@ -873,6 +964,38 @@ macro_rules! impl_table_meta_value {
    }
 }

+macro_rules! impl_table_meta_key_get_txn_op {
+    ($($key: ty), *) => {
+        $(
+            impl $crate::key::TableMetaKeyGetTxnOp for $key {
+                /// Returns a [TxnOp] to retrieve the corresponding value
+                /// and a filter to retrieve the value from the [TxnOpGetResponseSet]
+                fn build_get_op(
+                    &self,
+                ) -> (
+                    TxnOp,
+                    impl for<'a> FnMut(
+                        &'a mut TxnOpGetResponseSet,
+                    ) -> Option<Vec<u8>>,
+                ) {
+                    let raw_key = self.as_raw_key();
+                    (
+                        TxnOp::Get(raw_key.clone()),
+                        TxnOpGetResponseSet::filter(raw_key),
+                    )
+                }
+            }
+        )*
+    }
+}
+
+impl_table_meta_key_get_txn_op! {
+    TableNameKey<'_>,
+    TableInfoKey,
+    TableRouteKey,
+    DatanodeTableKey
+}
+
 #[macro_export]
 macro_rules! impl_optional_meta_value {
    ($($val_ty: ty), *) => {
@@ -907,6 +1030,7 @@ mod tests {
    use std::sync::Arc;

    use bytes::Bytes;
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
    use common_time::util::current_time_millis;
    use futures::TryStreamExt;
    use store_api::storage::RegionId;
@@ -914,6 +1038,7 @@ mod tests {

    use super::datanode_table::DatanodeTableKey;
    use super::test_utils;
+    use crate::ddl::test_util::create_table::test_create_table_task;
    use crate::ddl::utils::region_storage_path;
    use crate::error::Result;
    use crate::key::datanode_table::RegionInfo;
@@ -1155,15 +1280,10 @@ mod tests {
            table_info.schema_name,
            table_info.name,
        );
+        let table_route_value = &TableRouteValue::physical(region_routes.clone());
        // deletes metadata.
        table_metadata_manager
-            .delete_table_metadata(table_id, &table_name, region_routes)
-            .await
-            .unwrap();
-
-        // if metadata was already deleted, it should be ok.
-        table_metadata_manager
-            .delete_table_metadata(table_id, &table_name, region_routes)
+            .delete_table_metadata(table_id, &table_name, table_route_value)
            .await
            .unwrap();

@@ -1559,4 +1679,118 @@ mod tests {
            .await
            .is_err());
    }
+
+    #[tokio::test]
+    async fn test_destroy_table_metadata() {
+        let mem_kv = Arc::new(MemoryKvBackend::default());
+        let table_metadata_manager = TableMetadataManager::new(mem_kv.clone());
+        let table_id = 1025;
+        let table_name = "foo";
+        let task = test_create_table_task(table_name, table_id);
+        let options = [(0, "test".to_string())].into();
+        table_metadata_manager
+            .create_table_metadata(
+                task.table_info,
+                TableRouteValue::physical(vec![
+                    RegionRoute {
+                        region: Region::new_test(RegionId::new(table_id, 1)),
+                        leader_peer: Some(Peer::empty(1)),
+                        follower_peers: vec![Peer::empty(5)],
+                        leader_status: None,
+                        leader_down_since: None,
+                    },
+                    RegionRoute {
+                        region: Region::new_test(RegionId::new(table_id, 2)),
+                        leader_peer: Some(Peer::empty(2)),
+                        follower_peers: vec![Peer::empty(4)],
+                        leader_status: None,
+                        leader_down_since: None,
+                    },
+                    RegionRoute {
+                        region: Region::new_test(RegionId::new(table_id, 3)),
+                        leader_peer: Some(Peer::empty(3)),
+                        follower_peers: vec![],
+                        leader_status: None,
+                        leader_down_since: None,
+                    },
+                ]),
+                options,
+            )
+            .await
+            .unwrap();
+        let table_name = TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name);
+        let table_route_value = table_metadata_manager
+            .table_route_manager
+            .table_route_storage()
+            .get_raw(table_id)
+            .await
+            .unwrap()
+            .unwrap();
+        table_metadata_manager
+            .destroy_table_metadata(table_id, &table_name, &table_route_value)
+            .await
+            .unwrap();
+        assert!(mem_kv.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_restore_table_metadata() {
+        let mem_kv = Arc::new(MemoryKvBackend::default());
+        let table_metadata_manager = TableMetadataManager::new(mem_kv.clone());
+        let table_id = 1025;
+        let table_name = "foo";
+        let task = test_create_table_task(table_name, table_id);
+        let options = [(0, "test".to_string())].into();
+        table_metadata_manager
+            .create_table_metadata(
+                task.table_info,
+                TableRouteValue::physical(vec![
+                    RegionRoute {
+                        region: Region::new_test(RegionId::new(table_id, 1)),
+                        leader_peer: Some(Peer::empty(1)),
+                        follower_peers: vec![Peer::empty(5)],
+                        leader_status: None,
+                        leader_down_since: None,
+                    },
+                    RegionRoute {
+                        region: Region::new_test(RegionId::new(table_id, 2)),
+                        leader_peer: Some(Peer::empty(2)),
+                        follower_peers: vec![Peer::empty(4)],
+                        leader_status: None,
+                        leader_down_since: None,
+                    },
+                    RegionRoute {
+                        region: Region::new_test(RegionId::new(table_id, 3)),
+                        leader_peer: Some(Peer::empty(3)),
+                        follower_peers: vec![],
+                        leader_status: None,
+                        leader_down_since: None,
+                    },
+                ]),
+                options,
+            )
+            .await
+            .unwrap();
+        let expected_result = mem_kv.dump();
+        let table_route_value = table_metadata_manager
+            .table_route_manager
+            .table_route_storage()
+            .get_raw(table_id)
+            .await
+            .unwrap()
+            .unwrap();
+        let region_routes = table_route_value.region_routes().unwrap();
+        let table_name = TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name);
+        let table_route_value = TableRouteValue::physical(region_routes.clone());
+        table_metadata_manager
+            .delete_table_metadata(table_id, &table_name, &table_route_value)
+            .await
+            .unwrap();
+        table_metadata_manager
+            .restore_table_metadata(table_id, &table_name, &table_route_value)
+            .await
+            .unwrap();
+        let kvs = mem_kv.dump();
+        assert_eq!(kvs, expected_result);
+    }
 }
--- a/src/common/meta/src/key/datanode_table.rs
+++ b/src/common/meta/src/key/datanode_table.rs
@@ -55,6 +55,7 @@ pub struct RegionInfo {
    pub region_wal_options: HashMap<RegionNumber, String>,
 }

+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub struct DatanodeTableKey {
    pub datanode_id: DatanodeId,
    pub table_id: TableId,
--- a/src/common/meta/src/key/table_info.rs
+++ b/src/common/meta/src/key/table_info.rs
@@ -18,10 +18,11 @@ use serde::{Deserialize, Serialize};
 use table::metadata::{RawTableInfo, TableId};
 use table::table_reference::TableReference;

-use super::{txn_helper, DeserializedValueWithBytes, TableMetaValue, TABLE_INFO_KEY_PREFIX};
 use crate::error::Result;
-use crate::key::TableMetaKey;
-use crate::kv_backend::txn::{Txn, TxnOp, TxnOpResponse};
+use crate::key::{
+    txn_helper, DeserializedValueWithBytes, TableMetaKey, TableMetaValue, TABLE_INFO_KEY_PREFIX,
+};
+use crate::kv_backend::txn::{Txn, TxnOpResponse};
 use crate::kv_backend::KvBackendRef;
 use crate::rpc::store::BatchGetRequest;
 use crate::table_name::TableName;
@@ -101,20 +102,6 @@ impl TableInfoManager {
        Self { kv_backend }
    }

-    pub(crate) fn build_get_txn(
-        &self,
-        table_id: TableId,
-    ) -> (
-        Txn,
-        impl FnOnce(&Vec<TxnOpResponse>) -> Result<Option<DeserializedValueWithBytes<TableInfoValue>>>,
-    ) {
-        let key = TableInfoKey::new(table_id);
-        let raw_key = key.as_raw_key();
-        let txn = Txn::new().and_then(vec![TxnOp::Get(raw_key.clone())]);
-
-        (txn, txn_helper::build_txn_response_decoder_fn(raw_key))
-    }
-
    /// Builds a create table info transaction, it expected the `__table_info/{table_id}` wasn't occupied.
    pub(crate) fn build_create_txn(
        &self,
@@ -156,16 +143,6 @@ impl TableInfoManager {
        Ok((txn, txn_helper::build_txn_response_decoder_fn(raw_key)))
    }

-    /// Builds a delete table info transaction.
-    pub(crate) fn build_delete_txn(&self, table_id: TableId) -> Result<Txn> {
-        let key = TableInfoKey::new(table_id);
-        let raw_key = key.as_raw_key();
-
-        let txn = Txn::new().and_then(vec![TxnOp::Delete(raw_key)]);
-
-        Ok(txn)
-    }
-
    pub async fn get(
        &self,
        table_id: TableId,
--- a/src/common/meta/src/key/table_name.rs
+++ b/src/common/meta/src/key/table_name.rs
@@ -194,14 +194,6 @@ impl TableNameManager {
        Ok(txn)
    }

-    /// Builds a delete table name transaction. It only executes while the primary keys comparing successes.
-    pub(crate) fn build_delete_txn(&self, key: &TableNameKey<'_>) -> Result<Txn> {
-        let raw_key = key.as_raw_key();
-        let txn = Txn::new().and_then(vec![TxnOp::Delete(raw_key)]);
-
-        Ok(txn)
-    }
-
    pub async fn get(&self, key: TableNameKey<'_>) -> Result<Option<TableNameValue>> {
        let raw_key = key.as_raw_key();
        self.kv_backend
--- a/src/common/meta/src/key/table_route.rs
+++ b/src/common/meta/src/key/table_route.rs
@@ -26,7 +26,7 @@ use crate::error::{
    UnexpectedLogicalRouteTableSnafu,
 };
 use crate::key::{RegionDistribution, TableMetaKey, TABLE_ROUTE_PREFIX};
-use crate::kv_backend::txn::{Txn, TxnOp, TxnOpResponse};
+use crate::kv_backend::txn::{Txn, TxnOpResponse};
 use crate::kv_backend::KvBackendRef;
 use crate::rpc::router::{region_distribution, RegionRoute};
 use crate::rpc::store::BatchGetRequest;
@@ -61,6 +61,27 @@ pub struct LogicalTableRouteValue {
 }

 impl TableRouteValue {
+    /// Returns a [TableRouteValue::Physical] if `table_id` equals `physical_table_id`.
+    /// Otherwise returns a [TableRouteValue::Logical].
+    pub(crate) fn new(
+        table_id: TableId,
+        physical_table_id: TableId,
+        region_routes: Vec<RegionRoute>,
+    ) -> Self {
+        if table_id == physical_table_id {
+            TableRouteValue::physical(region_routes)
+        } else {
+            let region_routes = region_routes
+                .into_iter()
+                .map(|region| {
+                    debug_assert_eq!(region.region.id.table_id(), physical_table_id);
+                    RegionId::new(table_id, region.region.id.region_number())
+                })
+                .collect::<Vec<_>>();
+            TableRouteValue::logical(physical_table_id, region_routes)
+        }
+    }
+
    pub fn physical(region_routes: Vec<RegionRoute>) -> Self {
        Self::Physical(PhysicalTableRouteValue::new(region_routes))
    }
@@ -425,21 +446,6 @@ impl TableRouteStorage {
        Self { kv_backend }
    }

-    /// Builds a get table route transaction(readonly).
-    pub(crate) fn build_get_txn(
-        &self,
-        table_id: TableId,
-    ) -> (
-        Txn,
-        impl FnOnce(&Vec<TxnOpResponse>) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>>,
-    ) {
-        let key = TableRouteKey::new(table_id);
-        let raw_key = key.as_raw_key();
-        let txn = Txn::new().and_then(vec![TxnOp::Get(raw_key.clone())]);
-
-        (txn, txn_helper::build_txn_response_decoder_fn(raw_key))
-    }
-
    /// Builds a create table route transaction,
    /// it expected the `__table_route/{table_id}` wasn't occupied.
    pub fn build_create_txn(
@@ -483,17 +489,6 @@ impl TableRouteStorage {
        Ok((txn, txn_helper::build_txn_response_decoder_fn(raw_key)))
    }

-    /// Builds a delete table route transaction,
-    /// it expected the remote value equals the `table_route_value`.
-    pub(crate) fn build_delete_txn(&self, table_id: TableId) -> Result<Txn> {
-        let key = TableRouteKey::new(table_id);
-        let raw_key = key.as_raw_key();
-
-        let txn = Txn::new().and_then(vec![TxnOp::Delete(raw_key)]);
-
-        Ok(txn)
-    }
-
    /// Returns the [`TableRouteValue`].
    pub async fn get(&self, table_id: TableId) -> Result<Option<TableRouteValue>> {
        let key = TableRouteKey::new(table_id);
--- a/src/common/meta/src/key/tombstone.rs
+++ b/src/common/meta/src/key/tombstone.rs
@@ -0,0 +1,544 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use snafu::{ensure, OptionExt};
+
+use super::TableMetaKeyGetTxnOp;
+use crate::error::{self, Result};
+use crate::key::txn_helper::TxnOpGetResponseSet;
+use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
+use crate::kv_backend::KvBackendRef;
+
+/// [TombstoneManager] provides the ability to:
+/// - logically delete values
+/// - restore the deleted values
+pub(crate) struct TombstoneManager {
+    kv_backend: KvBackendRef,
+}
+
+const TOMBSTONE_PREFIX: &str = "__tombstone/";
+
+pub(crate) struct TombstoneKey<T>(T);
+
+fn to_tombstone(key: &[u8]) -> Vec<u8> {
+    [TOMBSTONE_PREFIX.as_bytes(), key].concat()
+}
+
+impl TombstoneKey<&Vec<u8>> {
+    /// Returns the origin key and tombstone key.
+    fn to_keys(&self) -> (Vec<u8>, Vec<u8>) {
+        let key = self.0;
+        let tombstone_key = to_tombstone(key);
+        (key.clone(), tombstone_key)
+    }
+
+    /// Returns the origin key and tombstone key.
+    fn into_keys(self) -> (Vec<u8>, Vec<u8>) {
+        self.to_keys()
+    }
+
+    /// Returns the tombstone key.
+    fn to_tombstone_key(&self) -> Vec<u8> {
+        let key = self.0;
+        to_tombstone(key)
+    }
+}
+
+impl TableMetaKeyGetTxnOp for TombstoneKey<&Vec<u8>> {
+    fn build_get_op(
+        &self,
+    ) -> (
+        TxnOp,
+        impl FnMut(&'_ mut TxnOpGetResponseSet) -> Option<Vec<u8>>,
+    ) {
+        TxnOpGetResponseSet::build_get_op(to_tombstone(self.0))
+    }
+}
+
+/// The key used in the [TombstoneManager].
+pub(crate) struct Key {
+    bytes: Vec<u8>,
+    // Atomic Key:
+    // The value corresponding to the key remains consistent between two transactions.
+    atomic: bool,
+}
+
+impl Key {
+    /// Returns a new atomic key.
+    pub(crate) fn compare_and_swap<T: Into<Vec<u8>>>(key: T) -> Self {
+        Self {
+            bytes: key.into(),
+            atomic: true,
+        }
+    }
+
+    /// Returns a new normal key.
+    pub(crate) fn new<T: Into<Vec<u8>>>(key: T) -> Self {
+        Self {
+            bytes: key.into(),
+            atomic: false,
+        }
+    }
+
+    /// Into bytes
+    pub(crate) fn into_bytes(self) -> Vec<u8> {
+        self.bytes
+    }
+
+    fn get_inner(&self) -> &Vec<u8> {
+        &self.bytes
+    }
+
+    fn is_atomic(&self) -> bool {
+        self.atomic
+    }
+}
+
+impl TableMetaKeyGetTxnOp for Key {
+    fn build_get_op(
+        &self,
+    ) -> (
+        TxnOp,
+        impl FnMut(&'_ mut TxnOpGetResponseSet) -> Option<Vec<u8>>,
+    ) {
+        let key = self.get_inner().clone();
+        (TxnOp::Get(key.clone()), TxnOpGetResponseSet::filter(key))
+    }
+}
+
+fn format_on_failure_error_message<F: FnMut(&mut TxnOpGetResponseSet) -> Option<Vec<u8>>>(
+    mut set: TxnOpGetResponseSet,
+    on_failure_kv_and_filters: Vec<(Vec<u8>, Vec<u8>, F)>,
+) -> String {
+    on_failure_kv_and_filters
+        .into_iter()
+        .flat_map(|(key, value, mut filter)| {
+            let got = filter(&mut set);
+            let Some(got) = got else {
+                return Some(format!(
+                    "For key: {} was expected: {}, but value does not exists",
+                    String::from_utf8_lossy(&key),
+                    String::from_utf8_lossy(&value),
+                ));
+            };
+
+            if got != value {
+                Some(format!(
+                    "For key: {} was expected: {}, but got: {}",
+                    String::from_utf8_lossy(&key),
+                    String::from_utf8_lossy(&value),
+                    String::from_utf8_lossy(&got),
+                ))
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>()
+        .join("; ")
+}
+
+fn format_keys(keys: &[Key]) -> String {
+    keys.iter()
+        .map(|key| String::from_utf8_lossy(&key.bytes))
+        .collect::<Vec<_>>()
+        .join(", ")
+}
+
+impl TombstoneManager {
+    /// Returns [TombstoneManager].
+    pub fn new(kv_backend: KvBackendRef) -> Self {
+        Self { kv_backend }
+    }
+
+    /// Creates tombstones for keys.
+    ///
+    /// Preforms to:
+    /// - retrieve all values corresponding `keys`.
+    /// - stores tombstone values.
+    pub(crate) async fn create(&self, keys: Vec<Key>) -> Result<()> {
+        // Builds transaction to retrieve all values
+        let (operations, mut filters): (Vec<_>, Vec<_>) =
+            keys.iter().map(|key| key.build_get_op()).unzip();
+
+        let txn = Txn::new().and_then(operations);
+        let mut resp = self.kv_backend.txn(txn).await?;
+        ensure!(
+            resp.succeeded,
+            error::UnexpectedSnafu {
+                err_msg: format!(
+                    "Failed to retrieves the metadata, keys: {}",
+                    format_keys(&keys)
+                ),
+            }
+        );
+
+        let mut set = TxnOpGetResponseSet::from(&mut resp.responses);
+        // Builds the create tombstone transaction.
+        let mut tombstone_operations = Vec::with_capacity(keys.len() * 2);
+        let mut tombstone_comparison = vec![];
+        let mut on_failure_operations = vec![];
+        let mut on_failure_kv_and_filters = vec![];
+        for (idx, key) in keys.iter().enumerate() {
+            let filter = &mut filters[idx];
+            let value = filter(&mut set).with_context(|| error::UnexpectedSnafu {
+                err_msg: format!(
+                    "Missing value, key: {}",
+                    String::from_utf8_lossy(key.get_inner())
+                ),
+            })?;
+            let (origin_key, tombstone_key) = TombstoneKey(key.get_inner()).into_keys();
+            // Compares the atomic key.
+            if key.is_atomic() {
+                tombstone_comparison.push(Compare::with_not_exist_value(
+                    tombstone_key.clone(),
+                    CompareOp::Equal,
+                ));
+                tombstone_comparison.push(Compare::with_value(
+                    origin_key.clone(),
+                    CompareOp::Equal,
+                    value.clone(),
+                ));
+                let (op, filter) = TxnOpGetResponseSet::build_get_op(origin_key.clone());
+                on_failure_operations.push(op);
+                on_failure_kv_and_filters.push((origin_key.clone(), value.clone(), filter));
+            }
+            tombstone_operations.push(TxnOp::Delete(origin_key));
+            tombstone_operations.push(TxnOp::Put(tombstone_key, value));
+        }
+
+        let txn = if !tombstone_comparison.is_empty() {
+            Txn::new().when(tombstone_comparison)
+        } else {
+            Txn::new()
+        }
+        .and_then(tombstone_operations);
+
+        let txn = if !on_failure_operations.is_empty() {
+            txn.or_else(on_failure_operations)
+        } else {
+            txn
+        };
+
+        let mut resp = self.kv_backend.txn(txn).await?;
+        // TODO(weny): add tests for atomic key changed.
+        if !resp.succeeded {
+            let set = TxnOpGetResponseSet::from(&mut resp.responses);
+            let err_msg = format_on_failure_error_message(set, on_failure_kv_and_filters);
+            return error::CasKeyChangedSnafu { err_msg }.fail();
+        }
+        Ok(())
+    }
+
+    /// Restores tombstones for keys.
+    ///
+    /// Preforms to:
+    /// - retrieve all tombstone values corresponding `keys`.
+    /// - stores tombstone values.
+    pub(crate) async fn restore(&self, keys: Vec<Key>) -> Result<()> {
+        // Builds transaction to retrieve all tombstone values
+        let tombstone_keys = keys
+            .iter()
+            .map(|key| TombstoneKey(key.get_inner()))
+            .collect::<Vec<_>>();
+        let (operations, mut filters): (Vec<_>, Vec<_>) =
+            tombstone_keys.iter().map(|key| key.build_get_op()).unzip();
+
+        let txn = Txn::new().and_then(operations);
+        let mut resp = self.kv_backend.txn(txn).await?;
+        ensure!(
+            resp.succeeded,
+            error::UnexpectedSnafu {
+                err_msg: format!(
+                    "Failed to retrieves the metadata, keys: {}",
+                    format_keys(&keys)
+                ),
+            }
+        );
+
+        let mut set = TxnOpGetResponseSet::from(&mut resp.responses);
+
+        // Builds the restore tombstone transaction.
+        let mut tombstone_operations = Vec::with_capacity(keys.len() * 2);
+        let mut tombstone_comparison = vec![];
+        let mut on_failure_operations = vec![];
+        let mut on_failure_kv_and_filters = vec![];
+        for (idx, key) in keys.iter().enumerate() {
+            let filter = &mut filters[idx];
+            let value = filter(&mut set).with_context(|| error::UnexpectedSnafu {
+                err_msg: format!(
+                    "Missing value, key: {}",
+                    String::from_utf8_lossy(key.get_inner())
+                ),
+            })?;
+            let (origin_key, tombstone_key) = tombstone_keys[idx].to_keys();
+            // Compares the atomic key.
+            if key.is_atomic() {
+                tombstone_comparison.push(Compare::with_not_exist_value(
+                    origin_key.clone(),
+                    CompareOp::Equal,
+                ));
+                tombstone_comparison.push(Compare::with_value(
+                    tombstone_key.clone(),
+                    CompareOp::Equal,
+                    value.clone(),
+                ));
+                let (op, filter) = tombstone_keys[idx].build_get_op();
+                on_failure_operations.push(op);
+                on_failure_kv_and_filters.push((tombstone_key.clone(), value.clone(), filter));
+            }
+            tombstone_operations.push(TxnOp::Delete(tombstone_key));
+            tombstone_operations.push(TxnOp::Put(origin_key, value));
+        }
+
+        let txn = if !tombstone_comparison.is_empty() {
+            Txn::new().when(tombstone_comparison)
+        } else {
+            Txn::new()
+        }
+        .and_then(tombstone_operations);
+
+        let txn = if !on_failure_operations.is_empty() {
+            txn.or_else(on_failure_operations)
+        } else {
+            txn
+        };
+
+        let mut resp = self.kv_backend.txn(txn).await?;
+        // TODO(weny): add tests for atomic key changed.
+        if !resp.succeeded {
+            let set = TxnOpGetResponseSet::from(&mut resp.responses);
+            let err_msg = format_on_failure_error_message(set, on_failure_kv_and_filters);
+            return error::CasKeyChangedSnafu { err_msg }.fail();
+        }
+
+        Ok(())
+    }
+
+    /// Deletes tombstones for keys.
+    pub(crate) async fn delete(&self, keys: Vec<Vec<u8>>) -> Result<()> {
+        let operations = keys
+            .iter()
+            .map(|key| TxnOp::Delete(TombstoneKey(key).to_tombstone_key()))
+            .collect::<Vec<_>>();
+
+        let txn = Txn::new().and_then(operations);
+        // Always success.
+        let _ = self.kv_backend.txn(txn).await?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use std::sync::Arc;
+
+    use crate::key::tombstone::{Key, TombstoneKey, TombstoneManager};
+    use crate::kv_backend::memory::MemoryKvBackend;
+    use crate::kv_backend::KvBackend;
+    use crate::rpc::store::PutRequest;
+
+    #[tokio::test]
+    async fn test_create_tombstone() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+        tombstone_manager
+            .create(vec![Key::compare_and_swap("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        assert!(!kv_backend.exists(b"bar").await.unwrap());
+        assert!(!kv_backend.exists(b"foo").await.unwrap());
+        assert_eq!(
+            kv_backend
+                .get(&TombstoneKey(&"bar".into()).to_tombstone_key())
+                .await
+                .unwrap()
+                .unwrap()
+                .value,
+            b"baz"
+        );
+        assert_eq!(
+            kv_backend
+                .get(&TombstoneKey(&"foo".into()).to_tombstone_key())
+                .await
+                .unwrap()
+                .unwrap()
+                .value,
+            b"hi"
+        );
+        assert_eq!(kv_backend.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_create_tombstone_without_atomic_key() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+        tombstone_manager
+            .create(vec![Key::new("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        assert!(!kv_backend.exists(b"bar").await.unwrap());
+        assert!(!kv_backend.exists(b"foo").await.unwrap());
+        assert_eq!(
+            kv_backend
+                .get(&TombstoneKey(&"bar".into()).to_tombstone_key())
+                .await
+                .unwrap()
+                .unwrap()
+                .value,
+            b"baz"
+        );
+        assert_eq!(
+            kv_backend
+                .get(&TombstoneKey(&"foo".into()).to_tombstone_key())
+                .await
+                .unwrap()
+                .unwrap()
+                .value,
+            b"hi"
+        );
+        assert_eq!(kv_backend.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_create_tombstone_origin_value_not_found_err() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+
+        let err = tombstone_manager
+            .create(vec![Key::compare_and_swap("bar"), Key::new("baz")])
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("Missing value"));
+    }
+
+    #[tokio::test]
+    async fn test_restore_tombstone() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+        let expected_kvs = kv_backend.dump();
+        tombstone_manager
+            .create(vec![Key::compare_and_swap("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        tombstone_manager
+            .restore(vec![Key::compare_and_swap("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        assert_eq!(expected_kvs, kv_backend.dump());
+    }
+
+    #[tokio::test]
+    async fn test_restore_tombstone_without_atomic_key() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+        let expected_kvs = kv_backend.dump();
+        tombstone_manager
+            .create(vec![Key::compare_and_swap("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        tombstone_manager
+            .restore(vec![Key::new("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        assert_eq!(expected_kvs, kv_backend.dump());
+    }
+
+    #[tokio::test]
+    async fn test_restore_tombstone_origin_value_not_found_err() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+        tombstone_manager
+            .create(vec![Key::compare_and_swap("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        let err = tombstone_manager
+            .restore(vec![Key::new("bar"), Key::new("baz")])
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("Missing value"));
+    }
+
+    #[tokio::test]
+    async fn test_delete_tombstone() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        kv_backend
+            .put(PutRequest::new().with_key("bar").with_value("baz"))
+            .await
+            .unwrap();
+        kv_backend
+            .put(PutRequest::new().with_key("foo").with_value("hi"))
+            .await
+            .unwrap();
+        tombstone_manager
+            .create(vec![Key::compare_and_swap("bar"), Key::new("foo")])
+            .await
+            .unwrap();
+        tombstone_manager
+            .delete(vec![b"bar".to_vec(), b"foo".to_vec()])
+            .await
+            .unwrap();
+        assert!(kv_backend.is_empty());
+    }
+}
--- a/src/common/meta/src/key/txn_helper.rs
+++ b/src/common/meta/src/key/txn_helper.rs
@@ -18,7 +18,69 @@ use serde::Serialize;
 use crate::error::Result;
 use crate::key::{DeserializedValueWithBytes, TableMetaValue};
 use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp, TxnOpResponse};
+use crate::rpc::KeyValue;

+/// The response set of [TxnOpResponse::ResponseGet]
+pub(crate) struct TxnOpGetResponseSet(Vec<KeyValue>);
+
+impl TxnOpGetResponseSet {
+    /// Returns a [TxnOp] to retrieve the value corresponding `key` and
+    /// a filter to consume corresponding [KeyValue] from [TxnOpGetResponseSet].
+    pub(crate) fn build_get_op<T: Into<Vec<u8>>>(
+        key: T,
+    ) -> (
+        TxnOp,
+        impl FnMut(&'_ mut TxnOpGetResponseSet) -> Option<Vec<u8>>,
+    ) {
+        let key = key.into();
+        (TxnOp::Get(key.clone()), TxnOpGetResponseSet::filter(key))
+    }
+
+    /// Returns a filter to consume a [KeyValue] where the key equals `key`.
+    pub(crate) fn filter(key: Vec<u8>) -> impl FnMut(&mut TxnOpGetResponseSet) -> Option<Vec<u8>> {
+        move |set| {
+            let pos = set.0.iter().position(|kv| kv.key == key);
+            match pos {
+                Some(pos) => Some(set.0.remove(pos).value),
+                None => None,
+            }
+        }
+    }
+
+    /// Returns a decoder to decode bytes to `DeserializedValueWithBytes<T>`.
+    pub(crate) fn decode_with<F, T>(
+        mut f: F,
+    ) -> impl FnMut(&mut TxnOpGetResponseSet) -> Result<Option<DeserializedValueWithBytes<T>>>
+    where
+        F: FnMut(&mut TxnOpGetResponseSet) -> Option<Vec<u8>>,
+        T: Serialize + DeserializeOwned + TableMetaValue,
+    {
+        move |set| {
+            f(set)
+                .map(|value| DeserializedValueWithBytes::from_inner_slice(&value))
+                .transpose()
+        }
+    }
+}
+
+impl From<&mut Vec<TxnOpResponse>> for TxnOpGetResponseSet {
+    fn from(value: &mut Vec<TxnOpResponse>) -> Self {
+        let value = value
+            .extract_if(|resp| matches!(resp, TxnOpResponse::ResponseGet(_)))
+            .flat_map(|resp| {
+                // Safety: checked
+                let TxnOpResponse::ResponseGet(r) = resp else {
+                    unreachable!()
+                };
+
+                r.kvs
+            })
+            .collect::<Vec<_>>();
+        TxnOpGetResponseSet(value)
+    }
+}
+
+// TODO(weny): using `TxnOpGetResponseSet`.
 pub(crate) fn build_txn_response_decoder_fn<T>(
    raw_key: Vec<u8>,
 ) -> impl FnOnce(&Vec<TxnOpResponse>) -> Result<Option<DeserializedValueWithBytes<T>>>
--- a/src/common/meta/src/kv_backend/memory.rs
+++ b/src/common/meta/src/kv_backend/memory.rs
@@ -70,6 +70,25 @@ impl<T> MemoryKvBackend<T> {
        let mut kvs = self.kvs.write().unwrap();
        kvs.clear();
    }
+
+    #[cfg(test)]
+    /// Returns true if the `kvs` is empty.
+    pub fn is_empty(&self) -> bool {
+        self.kvs.read().unwrap().is_empty()
+    }
+
+    #[cfg(test)]
+    /// Returns the `kvs`.
+    pub fn dump(&self) -> BTreeMap<Vec<u8>, Vec<u8>> {
+        let kvs = self.kvs.read().unwrap();
+        kvs.clone()
+    }
+
+    #[cfg(test)]
+    /// Returns the length of `kvs`
+    pub fn len(&self) -> usize {
+        self.kvs.read().unwrap().len()
+    }
 }

 #[async_trait]
--- a/src/common/meta/src/lib.rs
+++ b/src/common/meta/src/lib.rs
@@ -16,6 +16,7 @@
 #![feature(btree_extract_if)]
 #![feature(async_closure)]
 #![feature(let_chains)]
+#![feature(extract_if)]

 pub mod cache_invalidator;
 pub mod cluster;
--- a/src/common/meta/src/test_util.rs
+++ b/src/common/meta/src/test_util.rs
@@ -27,6 +27,7 @@ use crate::ddl::DdlContext;
 use crate::error::Result;
 use crate::key::TableMetadataManager;
 use crate::kv_backend::memory::MemoryKvBackend;
+use crate::kv_backend::KvBackendRef;
 use crate::peer::Peer;
 use crate::region_keeper::MemoryRegionKeeper;
 use crate::sequence::SequenceBuilder;
@@ -86,6 +87,14 @@ impl<T: MockDatanodeHandler + 'static> DatanodeManager for MockDatanodeManager<T
 /// Returns a test purpose [DdlContext].
 pub fn new_ddl_context(datanode_manager: DatanodeManagerRef) -> DdlContext {
    let kv_backend = Arc::new(MemoryKvBackend::new());
+    new_ddl_context_with_kv_backend(datanode_manager, kv_backend)
+}
+
+/// Returns a test purpose [DdlContext] with a specified [KvBackendRef].
+pub fn new_ddl_context_with_kv_backend(
+    datanode_manager: DatanodeManagerRef,
+    kv_backend: KvBackendRef,
+) -> DdlContext {
    let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));

    DdlContext {
--- a/src/common/substrait/Cargo.toml
+++ b/src/common/substrait/Cargo.toml
@@ -12,16 +12,14 @@ async-trait.workspace = true
 bytes.workspace = true
 catalog.workspace = true
 common-error.workspace = true
-common-function.workspace = true
 common-macro.workspace = true
+datafusion.workspace = true
 datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datafusion-substrait.workspace = true
-datafusion.workspace = true
 datatypes.workspace = true
 promql.workspace = true
 prost.workspace = true
-session.workspace = true
 snafu.workspace = true

 [dependencies.substrait_proto]
--- a/src/common/substrait/src/df_substrait.rs
+++ b/src/common/substrait/src/df_substrait.rs
@@ -16,9 +16,6 @@ use std::sync::Arc;

 use async_trait::async_trait;
 use bytes::{Buf, Bytes, BytesMut};
-use common_function::scalars::matches::MatchesFunction;
-use common_function::scalars::udf::create_udf;
-use common_function::state::FunctionState;
 use datafusion::catalog::CatalogList;
 use datafusion::execution::context::SessionState;
 use datafusion::execution::runtime_env::RuntimeEnv;
@@ -27,7 +24,6 @@ use datafusion_expr::LogicalPlan;
 use datafusion_substrait::logical_plan::consumer::from_substrait_plan;
 use datafusion_substrait::logical_plan::producer::to_substrait_plan;
 use prost::Message;
-use session::context::QueryContext;
 use snafu::ResultExt;
 use substrait_proto::proto::Plan;

@@ -54,13 +50,6 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor {
        let state = SessionState::new_with_config_rt(state_config, Arc::new(RuntimeEnv::default()))
            .with_serializer_registry(Arc::new(ExtensionSerializer));
        let mut context = SessionContext::new_with_state(state);
-
-        let udf = create_udf(
-            Arc::new(MatchesFunction),
-            QueryContext::arc(),
-            Arc::new(FunctionState::default()),
-        );
-        context.register_udf(udf.into());
        context.register_catalog_list(catalog_list);
        let plan = Plan::decode(message).context(DecodeRelSnafu)?;
        let df_plan = from_substrait_plan(&mut context, &plan)
@@ -76,13 +65,6 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor {
                .with_serializer_registry(Arc::new(ExtensionSerializer));
        let context = SessionContext::new_with_state(session_state);

-        let udf = create_udf(
-            Arc::new(MatchesFunction),
-            QueryContext::arc(),
-            Arc::new(FunctionState::default()),
-        );
-        context.register_udf(udf.into());
-
        let substrait_plan = to_substrait_plan(plan, &context).context(EncodeDfPlanSnafu)?;
        substrait_plan.encode(&mut buf).context(EncodeRelSnafu)?;

--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -545,9 +545,7 @@ impl RegionServerInner {
        match region_change {
            RegionChange::None => {}
            RegionChange::Register(_, _) | RegionChange::Deregisters => {
-                self.region_map
-                    .remove(&region_id)
-                    .map(|(id, engine)| engine.set_writable(id, false));
+                self.region_map.remove(&region_id);
            }
        }
    }
--- a/src/index/Cargo.toml
+++ b/src/index/Cargo.toml
@@ -22,13 +22,12 @@ greptime-proto.workspace = true
 mockall.workspace = true
 pin-project.workspace = true
 prost.workspace = true
-regex-automata.workspace = true
 regex.workspace = true
+regex-automata.workspace = true
 snafu.workspace = true
-tantivy = { version = "0.22", features = ["zstd-compression"] }

 [dev-dependencies]
 rand.workspace = true
 tempfile.workspace = true
-tokio-util.workspace = true
 tokio.workspace = true
+tokio-util.workspace = true
--- a/src/index/src/full_text_index/create.rs
+++ b/src/index/src/full_text_index/create.rs
@@ -1,90 +0,0 @@
-// Copyright 2024 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::path::Path;
-
-use snafu::ResultExt;
-use tantivy::schema::{Schema, INDEXED, STORED, TEXT};
-use tantivy::store::{Compressor, ZstdCompressor};
-use tantivy::{Index, IndexWriter, TantivyDocument};
-
-use super::error::TantivySnafu;
-use crate::full_text_index::error::Result;
-
-pub struct FullTextIndexCreater {
-    index: Index,
-    writer: IndexWriter,
-    count_field: tantivy::schema::Field,
-    text_field: tantivy::schema::Field,
-
-    row_count: usize,
-    segment_size: usize,
-}
-
-impl FullTextIndexCreater {
-    pub fn new<P>(segment_size: usize, path: P) -> Result<Self>
-    where
-        P: AsRef<Path>,
-    {
-        // build schema
-        let mut schema_builder = Schema::builder();
-        let count_field = schema_builder.add_i64_field("seg_count", INDEXED | STORED);
-        let text_field = schema_builder.add_text_field("text", TEXT);
-        let schema = schema_builder.build();
-
-        // create path
-        std::fs::create_dir_all(&path).unwrap();
-        common_telemetry::info!("[DEBUG] create full text index in {:?}", path.as_ref());
-
-        // build index
-        let mut index = Index::create_in_dir(path, schema).context(TantivySnafu)?;
-
-        // tune
-        index.settings_mut().docstore_compression = Compressor::Zstd(ZstdCompressor::default());
-        index.settings_mut().docstore_blocksize = 65_536;
-
-        // build writer
-        // 100 MB
-        let writer = index.writer(400_000_000).context(TantivySnafu)?;
-
-        Ok(Self {
-            index,
-            writer,
-            count_field,
-            text_field,
-            row_count: 0,
-            segment_size,
-        })
-    }
-
-    pub fn push_string(&mut self, content: String) -> Result<()> {
-        let mut doc = TantivyDocument::new();
-        doc.add_text(self.text_field, content);
-        doc.add_i64(self.count_field, (self.row_count / self.segment_size) as _);
-        self.writer.add_document(doc).context(TantivySnafu)?;
-        self.row_count += 1;
-
-        Ok(())
-    }
-
-    pub fn finish(&mut self) -> Result<()> {
-        common_telemetry::info!(
-            "[DEBUG] full text index finish with {} entries",
-            self.row_count
-        );
-        self.row_count = 0;
-        self.writer.commit().context(TantivySnafu)?;
-        Ok(())
-    }
-}
--- a/src/index/src/full_text_index/error.rs
+++ b/src/index/src/full_text_index/error.rs
@@ -1,45 +0,0 @@
-// Copyright 2024 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_macro::stack_trace_debug;
-use snafu::{Location, Snafu};
-use tantivy::directory::error::OpenDirectoryError;
-
-#[derive(Snafu)]
-#[snafu(visibility(pub))]
-#[stack_trace_debug]
-pub enum Error {
-    #[snafu(display("Tantivy error"))]
-    Tantivy {
-        #[snafu(source)]
-        error: tantivy::TantivyError,
-        location: Location,
-    },
-
-    #[snafu(display("Failed to open directory"))]
-    OpenDirectory {
-        #[snafu(source)]
-        error: OpenDirectoryError,
-        location: Location,
-    },
-
-    #[snafu(display("Failed to parse tantivy query"))]
-    ParseQuery {
-        #[snafu(source)]
-        error: tantivy::query::QueryParserError,
-        location: Location,
-    },
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
--- a/src/index/src/full_text_index/search.rs
+++ b/src/index/src/full_text_index/search.rs
@@ -1,74 +0,0 @@
-// Copyright 2024 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashSet;
-use std::path::Path;
-
-use snafu::ResultExt;
-use tantivy::directory::MmapDirectory;
-use tantivy::query::QueryParser;
-use tantivy::schema::Value;
-use tantivy::{Index, IndexReader, TantivyDocument, TantivyError};
-
-use super::error::ParseQuerySnafu;
-use crate::full_text_index::error::{OpenDirectorySnafu, Result, TantivySnafu};
-
-pub struct FullTextIndexSearcher {
-    index: Index,
-    count_field: tantivy::schema::Field,
-    text_field: tantivy::schema::Field,
-    reader: IndexReader,
-}
-
-impl FullTextIndexSearcher {
-    pub fn open<P>(path: P) -> Result<Self>
-    where
-        P: AsRef<Path>,
-    {
-        let index = Index::open_in_dir(path).context(TantivySnafu)?;
-        let schema = index.schema();
-        let count_field = schema.get_field("seg_count").unwrap();
-        let text_field = schema.get_field("text").unwrap();
-        let reader = index.reader().context(TantivySnafu)?;
-
-        Ok(Self {
-            index,
-            count_field,
-            text_field,
-            reader,
-        })
-    }
-
-    pub fn search(&self, query: &str) -> Result<Vec<usize>> {
-        let searcher = self.reader.searcher();
-        let query_parser = QueryParser::for_index(&self.index, vec![self.text_field]);
-        let query = query_parser.parse_query(query).context(ParseQuerySnafu)?;
-        let top_docs = searcher
-            .search(&query, &tantivy::collector::TopDocs::with_limit(1000_0000))
-            .context(TantivySnafu)?;
-        let mut result = HashSet::new();
-        for (_score, doc_address) in top_docs {
-            let retrieved_doc = searcher
-                .doc::<TantivyDocument>(doc_address)
-                .context(TantivySnafu)?;
-            let seg_count = retrieved_doc
-                .get_first(self.count_field)
-                .unwrap()
-                .as_i64()
-                .unwrap();
-            result.insert(seg_count);
-        }
-        Ok(result.into_iter().map(|x| x as _).collect())
-    }
-}
--- a/src/index/src/lib.rs
+++ b/src/index/src/lib.rs
@@ -14,5 +14,4 @@

 #![feature(iter_partition_in_place)]

-pub mod full_text_index;
 pub mod inverted_index;
--- a/src/mito2/src/access_layer.rs
+++ b/src/mito2/src/access_layer.rs
@@ -96,13 +96,6 @@ impl AccessLayer {
                })?;
        }

-        let full_text_index_dir = format!(
-            "/tmp/greptimedb/{}index/{}/full_text_index",
-            self.region_dir, file_meta.file_id
-        );
-        common_telemetry::info!("[DEBUG] removing {}", full_text_index_dir);
-        tokio::fs::remove_dir(full_text_index_dir).await.unwrap();
-
        Ok(())
    }

--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -39,7 +39,7 @@ use crate::read::{compat, Batch, Source};
 use crate::region::version::VersionRef;
 use crate::sst::file::FileHandle;
 use crate::sst::index::applier::builder::SstIndexApplierBuilder;
-use crate::sst::index::applier::{FullTextIndexApplier, SstIndexApplierRef};
+use crate::sst::index::applier::SstIndexApplierRef;

 /// A scanner scans a region and returns a [SendableRecordBatchStream].
 pub(crate) enum Scanner {
@@ -269,7 +269,6 @@ impl ScanRegion {
        );

        let index_applier = self.build_index_applier();
-        let full_text_index_applier = self.build_full_text_index_applier();
        let predicate = Predicate::new(self.request.filters.clone());
        // The mapper always computes projected column ids as the schema of SSTs may change.
        let mapper = match &self.request.projection {
@@ -284,7 +283,6 @@ impl ScanRegion {
            .with_files(files)
            .with_cache(self.cache_manager)
            .with_index_applier(index_applier)
-            .with_full_index_applier(full_text_index_applier)
            .with_parallelism(self.parallelism)
            .with_start_time(self.start_time)
            .with_append_mode(self.version.options.append_mode)
@@ -338,14 +336,6 @@ impl ScanRegion {
        .flatten()
        .map(Arc::new)
    }
-
-    fn build_full_text_index_applier(&self) -> Option<FullTextIndexApplier> {
-        FullTextIndexApplier::new(
-            self.access_layer.region_dir().to_string(),
-            self.version.metadata.region_id,
-            &self.request.filters,
-        )
-    }
 }

 /// Config for parallel scan.
@@ -403,8 +393,6 @@ pub(crate) struct ScanInput {
    pub(crate) append_mode: bool,
    /// Whether to remove deletion markers.
    pub(crate) filter_deleted: bool,
-
-    full_text_index_applier: Option<FullTextIndexApplier>,
 }

 impl ScanInput {
@@ -425,7 +413,6 @@ impl ScanInput {
            query_start: None,
            append_mode: false,
            filter_deleted: true,
-            full_text_index_applier: None,
        }
    }

@@ -485,15 +472,6 @@ impl ScanInput {
        self
    }

-    #[must_use]
-    pub(crate) fn with_full_index_applier(
-        mut self,
-        index_applier: Option<FullTextIndexApplier>,
-    ) -> Self {
-        self.full_text_index_applier = index_applier;
-        self
-    }
-
    /// Sets start time of the query.
    #[must_use]
    pub(crate) fn with_start_time(mut self, now: Option<Instant>) -> Self {
@@ -531,7 +509,6 @@ impl ScanInput {
                .projection(Some(self.mapper.column_ids().to_vec()))
                .cache(self.cache_manager.clone())
                .index_applier(self.index_applier.clone())
-                .full_text_index_applier(self.full_text_index_applier.clone())
                .build()
                .await;
            let reader = match maybe_reader {
--- a/src/mito2/src/sst/index.rs
+++ b/src/mito2/src/sst/index.rs
@@ -63,8 +63,6 @@ impl Indexer {
                // Skip index creation if error occurs.
                self.inner = None;
            }
-        } else {
-            common_telemetry::info!("[DEBUG] Indexer::update: inner is None");
        }

        if let Some(creator) = self.inner.as_ref() {
@@ -191,9 +189,6 @@ impl<'a> IndexerBuilder<'a> {
            segment_row_count = row_group_size;
        }

-        // find a column named "log"
-        let log_column_id = self.metadata.column_by_name("log").map(|c| c.column_id);
-
        let creator = SstIndexCreator::new(
            self.file_path,
            self.file_id,
@@ -202,7 +197,6 @@ impl<'a> IndexerBuilder<'a> {
            self.intermediate_manager,
            self.mem_threshold_index_create,
            segment_row_count,
-            log_column_id,
        )
        .with_buffer_size(self.write_buffer_size)
        .with_ignore_column_ids(
--- a/src/mito2/src/sst/index/applier.rs
+++ b/src/mito2/src/sst/index/applier.rs
@@ -16,10 +16,7 @@ pub mod builder;

 use std::sync::Arc;

-use common_query::logical_plan::Expr;
-use datafusion_expr::Expr as DfExpr;
 use futures::{AsyncRead, AsyncSeek};
-use index::full_text_index::search::FullTextIndexSearcher;
 use index::inverted_index::format::reader::InvertedIndexBlobReader;
 use index::inverted_index::search::index_apply::{
    ApplyOutput, IndexApplier, IndexNotFoundStrategy, SearchContext,
@@ -175,53 +172,6 @@ impl Drop for SstIndexApplier {
    }
 }

-#[derive(Debug, Clone)]
-pub(crate) struct FullTextIndexApplier {
-    region_dir: String,
-    region_id: RegionId,
-    query: String,
-}
-
-impl FullTextIndexApplier {
-    pub fn new(region_dir: String, region_id: RegionId, filters: &[Expr]) -> Option<Self> {
-        let query = Self::extract_from_filter(filters)?;
-        Some(Self {
-            region_dir,
-            region_id,
-            query,
-        })
-    }
-
-    fn extract_from_filter(filters: &[Expr]) -> Option<String> {
-        common_telemetry::info!("[DEBUG] filters in scan request: {:?}", filters);
-        for filter in filters {
-            if let DfExpr::ScalarUDF(udf) = filter.df_expr()
-                && udf.fun.name == "matches"
-            {
-                let pattern = &udf.args[0];
-                if let DfExpr::Literal(literal) = pattern {
-                    return Some(literal.to_string());
-                } else {
-                    return None;
-                }
-            }
-        }
-        None
-    }
-
-    /// Returns the selected row number
-    pub fn apply(&self, file_id: FileId) -> Result<Vec<usize>> {
-        let index_path = format!(
-            "/tmp/greptimedb/{}index/{}/full_text_index",
-            self.region_dir, file_id
-        );
-        common_telemetry::info!("[DEBUG] open index at {index_path}");
-
-        let searcher = FullTextIndexSearcher::open(index_path).unwrap();
-        Ok(searcher.search(&self.query).unwrap())
-    }
-}
-
 #[cfg(test)]
 mod tests {
    use common_base::BitVec;
--- a/src/mito2/src/sst/index/creator.rs
+++ b/src/mito2/src/sst/index/creator.rs
@@ -21,9 +21,6 @@ use std::sync::atomic::AtomicUsize;
 use std::sync::Arc;

 use common_telemetry::warn;
-use datatypes::scalars::ScalarVector;
-use datatypes::vectors::StringVector;
-use index::full_text_index::create::FullTextIndexCreater;
 use index::inverted_index::create::sort::external_sort::ExternalSorter;
 use index::inverted_index::create::sort_create::SortIndexCreator;
 use index::inverted_index::create::InvertedIndexCreator;
@@ -32,7 +29,6 @@ use object_store::ObjectStore;
 use puffin::file_format::writer::{Blob, PuffinAsyncWriter, PuffinFileWriter};
 use snafu::{ensure, ResultExt};
 use store_api::metadata::RegionMetadataRef;
-use store_api::storage::ConcreteDataType;
 use tokio::io::duplex;
 use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};

@@ -87,10 +83,6 @@ pub struct SstIndexCreator {

    /// The memory usage of the index creator.
    memory_usage: Arc<AtomicUsize>,
-
-    // experimental full text index
-    full_text_index_creater: FullTextIndexCreater,
-    log_column_id: Option<u32>,
 }

 impl SstIndexCreator {
@@ -104,7 +96,6 @@ impl SstIndexCreator {
        intermediate_manager: IntermediateManager,
        memory_usage_threshold: Option<usize>,
        segment_row_count: NonZeroUsize,
-        log_column_id: Option<u32>,
    ) -> Self {
        let temp_file_provider = Arc::new(TempFileProvider::new(
            IntermediateLocation::new(&metadata.region_id, &sst_file_id),
@@ -121,12 +112,6 @@ impl SstIndexCreator {
        );
        let index_creator = Box::new(SortIndexCreator::new(sorter, segment_row_count));

-        let file_id = file_path.trim_end_matches(".puffin");
-        let full_text_index_path = format!("/tmp/greptimedb/{file_id}/full_text_index");
-        // let full_text_index_creater =
-        //     FullTextIndexCreater::new(segment_row_count.get(), full_text_index_path).unwrap();
-        let full_text_index_creater = FullTextIndexCreater::new(1, full_text_index_path).unwrap();
-
        let codec = IndexValuesCodec::from_tag_columns(metadata.primary_key_columns());
        Self {
            file_path,
@@ -142,9 +127,6 @@ impl SstIndexCreator {

            ignore_column_ids: HashSet::default(),
            memory_usage,
-
-            full_text_index_creater,
-            log_column_id,
        }
    }

@@ -251,24 +233,6 @@ impl SstIndexCreator {
                .context(PushIndexValueSnafu)?;
        }

-        // try find column named "log" and update it into full text index
-        common_telemetry::info!("[DEBUG] do_update: log_column_id: {:?}", self.log_column_id);
-        if let Some(log_column_id) = self.log_column_id {
-            for col in batch.fields() {
-                if col.column_id == log_column_id {
-                    let vector = &col.data;
-                    if vector.data_type() == ConcreteDataType::string_datatype() {
-                        let vector = vector.as_any().downcast_ref::<StringVector>().unwrap();
-                        for content in vector.iter_data() {
-                            self.full_text_index_creater
-                                .push_string(content.unwrap_or_default().to_string())
-                                .unwrap();
-                        }
-                    }
-                }
-            }
-        }
-
        Ok(())
    }

@@ -332,8 +296,6 @@ impl SstIndexCreator {
            _ => {}
        }

-        self.full_text_index_creater.finish().unwrap();
-
        let byte_count = puffin_writer.finish().await.context(PuffinFinishSnafu)?;
        guard.inc_byte_count(byte_count);
        Ok(())
@@ -459,7 +421,6 @@ mod tests {
            intm_mgr,
            memory_threshold,
            NonZeroUsize::new(segment_row_count).unwrap(),
-            None,
        );

        for (str_tag, i32_tag) in &tags {
--- a/src/mito2/src/sst/location.rs
+++ b/src/mito2/src/sst/location.rs
@@ -29,10 +29,6 @@ pub fn index_file_path(region_dir: &str, sst_file_id: FileId) -> String {
    util::join_path(&dir, &sst_file_id.as_puffin())
 }

-pub fn full_text_index_path(region_dir: &str) -> String {
-    util::join_dir(region_dir, "full_text_index")
-}
-
 #[cfg(test)]
 mod tests {
    use super::*;
--- a/src/mito2/src/sst/parquet/reader.rs
+++ b/src/mito2/src/sst/parquet/reader.rs
@@ -29,7 +29,7 @@ use datafusion_common::arrow::buffer::BooleanBuffer;
 use datatypes::arrow::record_batch::RecordBatch;
 use itertools::Itertools;
 use object_store::ObjectStore;
-use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, RowSelection, RowSelector};
+use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, RowSelection};
 use parquet::arrow::{parquet_to_arrow_field_levels, FieldLevels, ProjectionMask};
 use parquet::file::metadata::ParquetMetaData;
 use parquet::format::KeyValue;
@@ -50,7 +50,7 @@ use crate::metrics::{
 use crate::read::{Batch, BatchReader};
 use crate::row_converter::{McmpRowCodec, RowCodec, SortField};
 use crate::sst::file::FileHandle;
-use crate::sst::index::applier::{FullTextIndexApplier, SstIndexApplierRef};
+use crate::sst::index::applier::SstIndexApplierRef;
 use crate::sst::parquet::format::ReadFormat;
 use crate::sst::parquet::metadata::MetadataLoader;
 use crate::sst::parquet::row_group::InMemoryRowGroup;
@@ -77,8 +77,6 @@ pub(crate) struct ParquetReaderBuilder {
    cache_manager: Option<CacheManagerRef>,
    /// Index applier.
    index_applier: Option<SstIndexApplierRef>,
-
-    full_text_index_applier: Option<FullTextIndexApplier>,
 }

 impl ParquetReaderBuilder {
@@ -97,7 +95,6 @@ impl ParquetReaderBuilder {
            projection: None,
            cache_manager: None,
            index_applier: None,
-            full_text_index_applier: None,
        }
    }

@@ -134,15 +131,6 @@ impl ParquetReaderBuilder {
        self
    }

-    #[must_use]
-    pub fn full_text_index_applier(
-        mut self,
-        full_text_index_applier: Option<FullTextIndexApplier>,
-    ) -> Self {
-        self.full_text_index_applier = full_text_index_applier;
-        self
-    }
-
    /// Builds and initializes a [ParquetReader].
    ///
    /// This needs to perform IO operation.
@@ -292,17 +280,10 @@ impl ParquetReaderBuilder {
        }
        metrics.num_row_groups_before_filtering += num_row_groups;

-        if let Some(full_text_index_result) = self.prune_row_groups_by_full_text_index(parquet_meta)
-        {
-            return full_text_index_result;
-        }
-
        self.prune_row_groups_by_inverted_index(parquet_meta, metrics)
            .await
            .or_else(|| self.prune_row_groups_by_minmax(read_format, parquet_meta, metrics))
            .unwrap_or_else(|| (0..num_row_groups).map(|i| (i, None)).collect())
-
-        // todo: change here
    }

    /// Applies index to prune row groups.
@@ -425,59 +406,6 @@ impl ParquetReaderBuilder {

        Some(row_groups)
    }
-
-    fn prune_row_groups_by_full_text_index(
-        &self,
-        parquet_meta: &ParquetMetaData,
-    ) -> Option<BTreeMap<usize, Option<RowSelection>>> {
-        let applier = self.full_text_index_applier.as_ref()?;
-        let file_id = self.file_handle.file_id();
-        let mut selected_row = applier.apply(file_id).unwrap();
-
-        common_telemetry::info!("[DEBUG] selected_row: {:?}", selected_row.len());
-
-        // Let's assume that the number of rows in the first row group
-        // can represent the `row_group_size` of the Parquet file.
-        //
-        // If the file contains only one row group, i.e. the number of rows
-        // less than the `row_group_size`, the calculation of `row_group_id`
-        // and `rg_begin_row_id` is still correct.
-        let row_group_size = parquet_meta.row_group(0).num_rows() as usize;
-        if row_group_size == 0 {
-            return None;
-        }
-
-        // translate `selected_row` into row groups selection
-        selected_row.sort_unstable();
-        let mut row_groups_selected = BTreeMap::new();
-        for row_id in selected_row.iter() {
-            let row_group_id = row_id / row_group_size;
-            let rg_row_id = row_id % row_group_size;
-
-            row_groups_selected
-                .entry(row_group_id)
-                .or_insert_with(Vec::new)
-                .push(rg_row_id);
-        }
-        let row_group = row_groups_selected
-            .into_iter()
-            .map(|(row_group_id, row_ids)| {
-                let mut current_row = 0;
-                let mut selection = vec![];
-                for row_id in row_ids {
-                    selection.push(RowSelector::skip(row_id - current_row));
-                    selection.push(RowSelector::select(1));
-                    current_row = row_id + 1;
-                }
-
-                (row_group_id, Some(RowSelection::from(selection)))
-            })
-            .collect();
-
-        // common_telemetry::info!("[DEBUG] row_group: {:?}", row_group);
-
-        Some(row_group)
-    }
 }

 /// Parquet reader metrics.
--- a/tests-chaos/Cargo.toml
+++ b/tests-chaos/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "tests-chaos"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+axum-macros = "0.3.8"
+axum.workspace = true
+common-error.workspace = true
+common-macro.workspace = true
+common-telemetry.workspace = true
+common-time = { workspace = true }
+lazy_static.workspace = true
+mysql = "25.0"
+nix = { version = "0.26", features = ["process"] }
+prometheus.workspace = true
+rand = { workspace = true }
+rand_chacha = "0.3.1"
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+snafu.workspace = true
+sqlx = { version = "0.6", features = [
+    "runtime-tokio-rustls",
+    "mysql",
+    "postgres",
+    "chrono",
+] }
+tests-fuzz.workspace = true
+tinytemplate = "1.2"
+tokio.workspace = true
--- a/tests-chaos/conf/standalone-v0.3.2.toml.template
+++ b/tests-chaos/conf/standalone-v0.3.2.toml.template
@@ -0,0 +1,82 @@
+mode = "standalone"
+enable_memory_catalog = false
+skip_write = false
+
+[http_options]
+addr = "127.0.0.1:4000"
+timeout = "30s"
+body_limit = "64MB"
+
+[grpc_options]
+addr = "127.0.0.1:4001"
+runtime_size = 8
+
+[mysql_options]
+addr = "127.0.0.1:4002"
+runtime_size = 2
+
+[mysql_options.tls]
+mode = "disable"
+cert_path = ""
+key_path = ""
+
+[postgres_options]
+addr = "127.0.0.1:4003"
+runtime_size = 2
+
+[postgres_options.tls]
+mode = "disable"
+cert_path = ""
+key_path = ""
+
+[opentsdb_options]
+addr = "127.0.0.1:4242"
+runtime_size = 2
+
+[influxdb_options]
+enable = true
+
+[prometheus_options]
+enable = true
+
+[prom_options]
+addr = "127.0.0.1:4004"
+
+[wal]
+file_size = "256MB"
+purge_threshold = "4GB"
+purge_interval = "10m"
+read_batch_size = 128
+sync_write = false
+
+[storage]
+type = "File"
+data_home = '{data_home}'
+global_ttl = "15m"
+skip_wal = true
+sst_compression = "lz4raw"
+memtable_type = "time_series"
+
+[storage.compaction]
+max_inflight_tasks = 4
+max_files_in_level0 = 5
+max_purge_tasks = 4
+purge_expired_only = true
+
+[storage.manifest]
+checkpoint_margin = 128
+gc_duration = '10m'
+checkpoint_on_startup = false
+
+[storage.flush]
+max_flush_tasks = 2
+region_write_buffer_size = "1MB"
+picker_schedule_interval = "5m"
+global_write_buffer_size = "150MB"
+
+[procedure]
+max_retry_times = 3
+retry_delay = "500ms"
+
+[logging]
+enable_logcat = false
--- a/src/index/src/full_text_index.rs
+++ b/src/index/src/full_text_index.rs
@@ -1,4 +1,4 @@
-// Copyright 2024 Greptime Team
+// Copyright 2023 Greptime Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,4 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-pub mod create;
-pub mod error;
-pub mod search;
+pub(crate) mod process;
--- a/tests-chaos/src/bare/process.rs
+++ b/tests-chaos/src/bare/process.rs
@@ -0,0 +1,132 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::process::{ExitStatus, Stdio};
+use std::sync::{Arc, Mutex};
+
+use common_telemetry::{info, warn};
+use nix::sys::signal::Signal;
+use snafu::ResultExt;
+use tokio::process::Child;
+
+use crate::error::{self, Result};
+
+pub(crate) type Pid = u32;
+
+/// The state of a process.
+#[derive(Debug, Clone)]
+pub(crate) struct Process {
+    pub(crate) exit_status: Option<ExitStatus>,
+    pub(crate) exited: bool,
+}
+
+/// ProcessManager provides the ability to spawn/wait/kill a child process.
+#[derive(Debug, Clone)]
+pub(crate) struct ProcessManager {
+    processes: Arc<Mutex<HashMap<Pid, Process>>>,
+}
+
+/// The callback while the child process exits.
+pub type OnChildExitResult = std::result::Result<ExitStatus, std::io::Error>;
+
+impl ProcessManager {
+    pub fn new() -> Self {
+        Self {
+            processes: Arc::new(Default::default()),
+        }
+    }
+
+    pub(crate) fn get(&self, pid: Pid) -> Option<Process> {
+        self.processes.lock().unwrap().get(&pid).cloned()
+    }
+
+    fn wait<F>(&self, mut child: Child, f: F)
+    where
+        F: FnOnce(Pid, OnChildExitResult) + Send + 'static,
+    {
+        let processes = self.processes.clone();
+        tokio::spawn(async move {
+            // Safety: caller checked
+            let pid = child.id().unwrap();
+            let result = child.wait().await;
+
+            match result {
+                Ok(code) => {
+                    warn!("pid: {pid} exited with status: {}", code);
+                    f(pid, Ok(code));
+                    processes.lock().unwrap().entry(pid).and_modify(|process| {
+                        process.exit_status = Some(code);
+                        process.exited = true;
+                    });
+                }
+                Err(err) => {
+                    warn!("pid: {pid} exited with error: {}", err);
+                    f(pid, Err(err));
+                    processes.lock().unwrap().entry(pid).and_modify(|process| {
+                        process.exited = true;
+                    });
+                }
+            }
+        });
+    }
+
+    /// Spawns a new process.
+    pub fn spawn<T: Into<Stdio>, F>(
+        &self,
+        binary: &str,
+        args: &[String],
+        stdout: T,
+        stderr: T,
+        on_exit: F,
+    ) -> Result<Pid>
+    where
+        F: FnOnce(Pid, OnChildExitResult) + Send + 'static,
+    {
+        info!("starting {} with {:?}", binary, args);
+        let child = tokio::process::Command::new(binary)
+            .args(args)
+            .stdout(stdout)
+            .stderr(stderr)
+            .spawn()
+            .context(error::SpawnChildSnafu)?;
+        let pid = child.id();
+
+        if let Some(pid) = pid {
+            self.processes.lock().unwrap().insert(
+                pid,
+                Process {
+                    exit_status: None,
+                    exited: false,
+                },
+            );
+
+            self.wait(child, on_exit);
+            Ok(pid)
+        } else {
+            error::UnexpectedExitedSnafu {}.fail()
+        }
+    }
+
+    /// Kills a process via [Pid].
+    pub fn kill<T: Into<Option<Signal>>>(pid: Pid, signal: T) -> Result<()> {
+        let signal: Option<Signal> = signal.into();
+        info!("kill pid :{} siganl: {:?}", pid, signal);
+        // Safety: checked.
+        nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), signal)
+            .context(error::KillProcessSnafu)?;
+
+        Ok(())
+    }
+}
--- a/tests-chaos/src/error.rs
+++ b/tests-chaos/src/error.rs
@@ -0,0 +1,77 @@
+use common_macro::stack_trace_debug;
+use snafu::{location, Location, Snafu};
+
+#[derive(Snafu)]
+#[snafu(visibility(pub))]
+#[stack_trace_debug]
+pub enum Error {
+    #[snafu(display("Failed to spawn a child process"))]
+    SpawnChild {
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[snafu(display("Child process exited unexpected"))]
+    UnexpectedExited { location: Location },
+
+    #[snafu(display("Unexpected: {err_msg}"))]
+    Unexpected { err_msg: String, location: Location },
+
+    #[snafu(display("Failed to kill a process"))]
+    KillProcess {
+        location: Location,
+        #[snafu(source)]
+        error: nix::Error,
+    },
+
+    #[snafu(display("Failed to create a file: {}", path))]
+    CreateFile {
+        path: String,
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[snafu(display("Failed to create dir all"))]
+    CreateDirAll {
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[snafu(display("Failed to write a file: {}", path))]
+    WriteFile {
+        path: String,
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[snafu(display("Failed to execute query: {}", sql))]
+    ExecuteQuery {
+        sql: String,
+        #[snafu(source)]
+        error: sqlx::error::Error,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to request mysql, error: {}", err_msg))]
+    RequestMysql {
+        err_msg: String,
+        #[snafu(source)]
+        error: mysql::Error,
+        location: Location,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<tests_fuzz::error::Error> for Error {
+    fn from(e: tests_fuzz::error::Error) -> Self {
+        Self::Unexpected {
+            err_msg: e.to_string(),
+            location: location!(),
+        }
+    }
+}
--- a/tests-chaos/src/main.rs
+++ b/tests-chaos/src/main.rs
@@ -0,0 +1,505 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use axum::extract::{Query, State};
+use axum::Router;
+use bare::process::{Pid, ProcessManager};
+use common_telemetry::{info, warn};
+use mysql::prelude::Queryable;
+use nix::sys::signal::Signal;
+use rand::{Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
+use serde::Serialize;
+use snafu::{ensure, ResultExt};
+use sqlx::mysql::MySqlPoolOptions;
+use sqlx::{MySql, Pool};
+use tests_fuzz::context::{Rows, TableContext};
+use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder;
+use tests_fuzz::generator::Generator;
+use tests_fuzz::ir::select_expr::{Direction, SelectExpr};
+use tests_fuzz::ir::AlterTableOperation;
+use tests_fuzz::translator::mysql::alter_expr::AlterTableExprTranslator;
+use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator;
+use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator;
+use tests_fuzz::translator::mysql::select_expr::SelectExprTranslator;
+use tests_fuzz::translator::DslTranslator;
+use tests_fuzz::validator;
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
+use utils::generate_alter_table_expr;
+mod bare;
+mod error;
+mod utils;
+
+use axum::routing::get;
+use prometheus::{register_int_counter, Encoder, IntCounter, TextEncoder};
+
+use crate::error::{Error, RequestMysqlSnafu, Result};
+use crate::utils::{generate_create_table_expr, get_conf_path, path_to_stdio, render_config_file};
+const DEFAULT_LOG_LEVEL: &str = "--log-level=debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info";
+
+#[derive(Copy, Clone)]
+pub struct MetricsHandler;
+
+impl MetricsHandler {
+    pub fn render(&self) -> String {
+        let mut buffer = Vec::new();
+        let encoder = TextEncoder::new();
+        // Gather the metrics.
+        let metric_families = prometheus::gather();
+        // Encode them to send.
+        match encoder.encode(&metric_families, &mut buffer) {
+            Ok(_) => match String::from_utf8(buffer) {
+                Ok(s) => s,
+                Err(e) => e.to_string(),
+            },
+            Err(e) => e.to_string(),
+        }
+    }
+}
+
+#[axum_macros::debug_handler]
+pub async fn metrics(
+    State(state): State<MetricsHandler>,
+    Query(_params): Query<HashMap<String, String>>,
+) -> String {
+    state.render()
+}
+
+lazy_static::lazy_static! {
+    static ref UP_COUNTER: IntCounter = register_int_counter!("up", "up counter").unwrap();
+}
+
+// cargo run --package tests-chaos --bin tests-chaos
+#[tokio::main]
+async fn main() {
+    common_telemetry::init_default_ut_logging();
+
+    tokio::spawn(async move {
+        let app = Router::new()
+            .route("/metric", get(metrics))
+            .with_state(MetricsHandler);
+        let addr = SocketAddr::from(([0, 0, 0, 0], 30000));
+        axum::Server::bind(&addr)
+            .serve(app.into_make_service())
+            .await
+            .unwrap();
+    });
+
+    let test_dir = "/home/lfc/test-cuckoo/";
+    // Remove everything in the test directory, to make sure we have a clean start.
+    match std::fs::remove_dir_all(test_dir) {
+        Err(e) if e.kind() != std::io::ErrorKind::NotFound => panic!("{e:?}"),
+        _ => {}
+    }
+    std::fs::create_dir_all(test_dir).unwrap();
+
+    let state = Arc::new(TestState {
+        killed: AtomicBool::new(false),
+    });
+
+    let moved_state = state.clone();
+    tokio::spawn(async move {
+        let mut rng = ChaChaRng::seed_from_u64(0);
+        loop {
+            warn!("Staring");
+            UP_COUNTER.inc();
+            let pid = start_database(test_dir).await.unwrap();
+            let secs = rng.gen_range(100..300);
+            moved_state.killed.store(false, Ordering::Relaxed);
+            tokio::time::sleep(Duration::from_millis(secs)).await;
+            warn!("After {secs}ms, Killing pid: {pid}");
+            moved_state.killed.store(true, Ordering::Relaxed);
+
+            // Flush the database before restarting it. Because cuckoo does not enable WAL,
+            // data may not survive the restart if not flush them.
+            flush_db().await;
+
+            ProcessManager::kill(pid, Signal::SIGKILL).expect("Failed to kill");
+        }
+    });
+    let mut rng = ChaChaRng::seed_from_u64(0);
+    let mut sqlx = sqlx_connections().await;
+    let mut mysql = mysql_connections().await;
+    let mut created_table = HashSet::new();
+
+    // Runs maximum 10000 times.
+    for _i in 0..10000 {
+        if let Err(e) = run_test(&sqlx, &mysql, &mut created_table, &state, &mut rng).await {
+            if matches!(e, Error::ExecuteQuery { .. } | Error::RequestMysql { .. })
+                && state.killed.load(Ordering::Relaxed)
+            {
+                // If the query error is caused by restarting the database
+                // (which is an intended action), reconnect.
+                sqlx = sqlx_connections().await;
+                mysql = mysql_connections().await;
+            } else {
+                panic!("{e:?}");
+            }
+        }
+    }
+    info!("Successfully runs DDL chaos testing for cuckoo!");
+}
+
+async fn flush_db() {
+    info!("Start flushing the database ...");
+    let _ = reqwest::get("http://127.0.0.1:4000/v1/admin/flush?db=public")
+        .await
+        .unwrap()
+        .bytes()
+        .await
+        .unwrap();
+}
+
+async fn mysql_connections() -> mysql::Pool {
+    let mut max_retry = 10;
+    loop {
+        match mysql::Pool::new("mysql://127.0.0.1:4002/public") {
+            Ok(x) => return x,
+            Err(e) => {
+                max_retry -= 1;
+                if max_retry == 0 {
+                    panic!("{e:?}")
+                } else {
+                    info!("GreptimeDB is not connectable, maybe during restart. Wait 1 second to retry");
+                    tokio::time::sleep(Duration::from_secs(1)).await;
+                }
+            }
+        }
+    }
+}
+
+async fn sqlx_connections() -> Pool<MySql> {
+    let mut max_retry = 10;
+    loop {
+        match MySqlPoolOptions::new()
+            .connect("mysql://127.0.0.1:4002/public")
+            .await
+        {
+            Ok(x) => return x,
+            Err(e) => {
+                max_retry -= 1;
+                if max_retry == 0 {
+                    panic!("{e:?}")
+                } else {
+                    info!("GreptimeDB is not connectable, maybe during restart. Wait 1 second to retry");
+                    tokio::time::sleep(Duration::from_secs(1)).await;
+                }
+            }
+        }
+    }
+}
+
+struct TestState {
+    killed: AtomicBool,
+}
+
+async fn run_test<R: Rng + 'static>(
+    client: &Pool<MySql>,
+    pool: &mysql::Pool,
+    created_table: &mut HashSet<String>,
+    state: &Arc<TestState>,
+    rng: &mut R,
+) -> Result<()> {
+    let expr = generate_create_table_expr(rng);
+    let table_name = expr.table_name.to_string();
+    if !created_table.insert(table_name.clone()) {
+        warn!("ignores same name table: {table_name}");
+        // ignores.
+        return Ok(());
+    }
+
+    let mut table_ctx = TableContext::from(&expr);
+
+    let translator = CreateTableExprTranslator;
+    let sql = translator.translate(&expr).unwrap();
+    let result = sqlx::query(&sql).execute(client).await;
+
+    match result {
+        Ok(result) => {
+            validate_mysql(client, state, &table_ctx).await;
+            info!("Create table: {sql}, result: {result:?}");
+        }
+        Err(err) => {
+            ensure!(
+                state.killed.load(Ordering::Relaxed),
+                error::UnexpectedSnafu {
+                    err_msg: err.to_string(),
+                }
+            );
+            created_table.insert(table_name);
+            return Ok(());
+        }
+    }
+
+    let actions = rng.gen_range(1..20);
+
+    for _ in 0..actions {
+        let expr = generate_alter_table_expr(Arc::new(table_ctx.clone()), rng);
+        if let AlterTableOperation::RenameTable { new_table_name } = &expr.alter_options {
+            let table_name = new_table_name.to_string();
+            if created_table.contains(&table_name) {
+                warn!("ignores altering to same name table: {table_name}");
+                continue;
+            }
+        };
+
+        insert_rows(pool, &mut table_ctx, rng)?;
+
+        let translator = AlterTableExprTranslator;
+        let sql = translator.translate(&expr).unwrap();
+        let result = sqlx::query(&sql).execute(client).await;
+        match result {
+            Ok(result) => {
+                info!("alter table: {sql}, result: {result:?}");
+                let table_name = table_ctx.name.to_string();
+                created_table.remove(&table_name);
+                table_ctx.alter(expr)?;
+                validate_mysql(client, state, &table_ctx).await;
+                let table_name = table_ctx.name.to_string();
+                created_table.insert(table_name);
+            }
+            Err(err) => {
+                table_ctx.alter(expr)?;
+                let table_name = table_ctx.name.to_string();
+                created_table.insert(table_name);
+                ensure!(
+                    state.killed.load(Ordering::Relaxed),
+                    error::UnexpectedSnafu {
+                        err_msg: err.to_string(),
+                    }
+                );
+                break;
+            }
+        }
+
+        let actual_rows = fetch_all_rows(&table_ctx, pool)?;
+        info!("fetch all rows after alter: {actual_rows}");
+        if actual_rows.empty() && state.killed.load(Ordering::Relaxed) {
+            // Cuckoo does not have WAL enabled; therefore the data could be cleared across restart.
+            // When that happened, clear the saved data that are used for comparison, too.
+            table_ctx.clear_data();
+        } else {
+            assert_eq!(
+                &table_ctx.rows, &actual_rows,
+                r#"rows not equal:
+expect: {}
+actual: {}"#,
+                &table_ctx.rows, &actual_rows
+            )
+        }
+    }
+
+    Ok(())
+}
+
+fn insert_rows<R: Rng + 'static>(
+    pool: &mysql::Pool,
+    table_ctx: &mut TableContext,
+    rng: &mut R,
+) -> Result<()> {
+    let insert_expr = InsertExprGeneratorBuilder::default()
+        .table_ctx(Arc::new(table_ctx.clone()))
+        .build()
+        .unwrap()
+        .generate(rng)
+        .unwrap();
+    let sql = InsertIntoExprTranslator.translate(&insert_expr).unwrap();
+    info!("executing insertion: {sql}");
+
+    let mut conn = pool.get_conn().context(RequestMysqlSnafu {
+        err_msg: "get connection",
+    })?;
+    conn.query_drop(&sql).with_context(|_| RequestMysqlSnafu {
+        err_msg: format!("executing sql '{}'", sql),
+    })?;
+
+    if conn.affected_rows() > 0 {
+        table_ctx.insert(insert_expr)?;
+    }
+    Ok(())
+}
+
+// Sqlx treats all queries as prepared, we have to switch to another mysql client library here.
+// There's a error when trying to query GreptimeDB with prepared statement:
+// "tried to use [50, 48, ..., 56] as MYSQL_TYPE_TIMESTAMP"
+// Besides, sqlx is suited for cases where table schema is known (and representable in codes),
+// definitely not here.
+fn fetch_all_rows(table_ctx: &TableContext, pool: &mysql::Pool) -> Result<Rows> {
+    let select_expr = SelectExpr {
+        table_name: table_ctx.name.to_string(),
+        columns: table_ctx.columns.clone(),
+        order_by: vec![table_ctx
+            .columns
+            .iter()
+            .find_map(|c| {
+                if c.is_time_index() {
+                    Some(c.name.to_string())
+                } else {
+                    None
+                }
+            })
+            .unwrap()],
+        direction: Direction::Asc,
+        limit: usize::MAX,
+    };
+    let sql = SelectExprTranslator.translate(&select_expr).unwrap();
+    info!("executing selection: {sql}");
+
+    let mut conn = pool.get_conn().context(RequestMysqlSnafu {
+        err_msg: "get connection",
+    })?;
+    let rows: Vec<mysql::Row> = conn.query(&sql).with_context(|_| RequestMysqlSnafu {
+        err_msg: format!("executing sql: {}", sql),
+    })?;
+
+    Ok(Rows::fill(rows))
+}
+
+async fn validate_mysql(client: &Pool<MySql>, _state: &Arc<TestState>, table_ctx: &TableContext) {
+    loop {
+        match validator::column::fetch_columns_via_mysql(
+            client,
+            "public".into(),
+            table_ctx.name.clone(),
+        )
+        .await
+        {
+            Ok(mut column_entries) => {
+                column_entries.sort_by(|a, b| a.column_name.cmp(&b.column_name));
+                let mut columns = table_ctx.columns.clone();
+                columns.sort_by(|a, b| a.name.value.cmp(&b.name.value));
+                validator::column::assert_eq(&column_entries, &columns).unwrap();
+                return;
+            }
+            Err(err) => warn!(
+                "Failed to fetch table '{}' columns, error: {}",
+                table_ctx.name, err
+            ),
+        }
+    }
+}
+
+async fn start_database(test_dir: &str) -> Result<Pid> {
+    let binary_path = "/home/lfc/greptimedb-cuckoo/target/debug/greptime";
+    let template_filename = "standalone-v0.3.2.toml.template";
+    let health_url = "http://127.0.0.1:4000/health";
+
+    let process_manager = ProcessManager::new();
+    for _ in 0..3 {
+        let pid = start_process(&process_manager, binary_path, test_dir, template_filename)
+            .await
+            .unwrap();
+        match tokio::time::timeout(Duration::from_secs(10), health_check(health_url)).await {
+            Ok(_) => {
+                info!("GreptimeDB started, pid: {pid}");
+                return Ok(pid);
+            }
+            Err(_) => {
+                ensure!(
+                    process_manager.get(pid).unwrap().exited,
+                    error::UnexpectedSnafu {
+                        err_msg: format!("Failed to start database: pid: {pid}")
+                    }
+                );
+                // retry alter
+                warn!("Wait for staring timeout, retry later...");
+            }
+        };
+    }
+    error::UnexpectedSnafu {
+        err_msg: "Failed to start datanode",
+    }
+    .fail()
+}
+
+async fn start_process(
+    process_manager: &ProcessManager,
+    binary: &str,
+    test_dir: &str,
+    template_filename: &str,
+) -> Result<Pid> {
+    tokio::fs::create_dir_all(test_dir)
+        .await
+        .context(error::CreateDirAllSnafu)?;
+
+    let data_home = format!("{test_dir}data_home");
+    info!("data home: {}", data_home);
+
+    // Prepares the config file
+    let mut conf_path = get_conf_path();
+    conf_path.push(template_filename);
+    let template_path = conf_path.to_str().unwrap().to_string();
+
+    let conf_path = format!("{test_dir}config.toml");
+    info!("conf path: {}", conf_path);
+    #[derive(Serialize)]
+    struct Context {
+        data_home: String,
+    }
+    let conf_content = render_config_file(&template_path, &Context { data_home });
+    let mut config_file = File::create(&conf_path)
+        .await
+        .context(error::CreateFileSnafu { path: &conf_path })?;
+    config_file
+        .write_all(conf_content.as_bytes())
+        .await
+        .context(error::WriteFileSnafu { path: &conf_path })?;
+
+    let args = vec![
+        DEFAULT_LOG_LEVEL.to_string(),
+        "standalone".to_string(),
+        "start".to_string(),
+        format!("--config-file={conf_path}"),
+    ];
+
+    let now = common_time::util::current_time_millis();
+    let stdout = format!("{test_dir}stdout-{}", now);
+    let stderr = format!("{test_dir}stderr-{}", now);
+    info!("stdout: {}, stderr: {}", stdout, stderr);
+    let stdout = path_to_stdio(&stdout).await?;
+    let stderr = path_to_stdio(&stderr).await?;
+
+    let on_exit = move |pid, result| {
+        info!("The pid: {pid} exited, result: {result:?}");
+    };
+
+    process_manager.spawn(binary, &args, stdout, stderr, on_exit)
+}
+
+async fn health_check(url: &str) {
+    loop {
+        match reqwest::get(url).await {
+            Ok(resp) => {
+                if resp.status() == 200 {
+                    info!("health checked!");
+                    return;
+                }
+                info!("failed to health, status: {}", resp.status());
+            }
+            Err(err) => {
+                info!("failed to health, err: {err:?}");
+            }
+        }
+
+        info!("checking health later...");
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}
--- a/tests-chaos/src/utils.rs
+++ b/tests-chaos/src/utils.rs
@@ -0,0 +1,111 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+
+use rand::Rng;
+use serde::Serialize;
+use snafu::ResultExt;
+use tests_fuzz::context::TableContextRef;
+use tests_fuzz::fake::{
+    merge_two_word_map_fn, random_capitalize_map, uppercase_and_keyword_backtick_map,
+    MappedGenerator, WordGenerator,
+};
+use tests_fuzz::generator::alter_expr::{
+    AlterExprAddColumnGeneratorBuilder, AlterExprDropColumnGeneratorBuilder,
+    AlterExprRenameGeneratorBuilder,
+};
+use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder;
+use tests_fuzz::generator::Generator;
+use tests_fuzz::ir::{droppable_columns, AlterTableExpr, CreateTableExpr};
+use tinytemplate::TinyTemplate;
+use tokio::fs::OpenOptions;
+
+use crate::error::{self, Result};
+
+/// Creates an file
+pub(crate) async fn path_to_stdio(path: &str) -> Result<std::fs::File> {
+    Ok(OpenOptions::new()
+        .append(true)
+        .create(true)
+        .read(true)
+        .write(true)
+        .open(path)
+        .await
+        .context(error::CreateFileSnafu { path })?
+        .into_std()
+        .await)
+}
+
+/// Get the path of config dir `tests/conf`.
+pub(crate) fn get_conf_path() -> PathBuf {
+    let mut root_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    root_path.push("conf");
+    root_path
+}
+
+/// Returns rendered config file.
+pub(crate) fn render_config_file<C: Serialize>(template_path: &str, context: &C) -> String {
+    let mut tt = TinyTemplate::new();
+    let template = std::fs::read_to_string(template_path).unwrap();
+    tt.add_template(template_path, &template).unwrap();
+    tt.render(template_path, context).unwrap()
+}
+
+pub(crate) fn generate_create_table_expr<R: Rng + 'static>(rng: &mut R) -> CreateTableExpr {
+    let columns = rng.gen_range(2..30);
+    let create_table_generator = CreateTableExprGeneratorBuilder::default()
+        .name_generator(Box::new(MappedGenerator::new(
+            WordGenerator,
+            merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
+        )))
+        .columns(columns)
+        .engine("mito")
+        .build()
+        .unwrap();
+    create_table_generator.generate(rng).unwrap()
+}
+
+#[allow(dead_code)]
+pub fn generate_alter_table_expr<R: Rng + 'static>(
+    table_ctx: TableContextRef,
+    rng: &mut R,
+) -> AlterTableExpr {
+    let rename = rng.gen_bool(0.2);
+    if rename {
+        let expr_generator = AlterExprRenameGeneratorBuilder::default()
+            .table_ctx(table_ctx)
+            .name_generator(Box::new(WordGenerator))
+            .build()
+            .unwrap();
+        expr_generator.generate(rng).unwrap()
+    } else {
+        let drop_column = rng.gen_bool(0.5) && !droppable_columns(&table_ctx.columns).is_empty();
+        if drop_column {
+            let expr_generator = AlterExprDropColumnGeneratorBuilder::default()
+                .table_ctx(table_ctx)
+                .build()
+                .unwrap();
+            expr_generator.generate(rng).unwrap()
+        } else {
+            let location = rng.gen_bool(0.5);
+            let expr_generator = AlterExprAddColumnGeneratorBuilder::default()
+                .table_ctx(table_ctx)
+                .location(location)
+                .build()
+                .unwrap();
+            expr_generator.generate(rng).unwrap()
+        }
+    }
+}
--- a/tests-fuzz/Cargo.toml
+++ b/tests-fuzz/Cargo.toml
@@ -24,6 +24,7 @@ derive_builder = { workspace = true }
 dotenv = "0.15"
 lazy_static = { workspace = true }
 libfuzzer-sys = "0.4"
+mysql = "25.0"
 partition = { workspace = true }
 rand = { workspace = true }
 rand_chacha = "0.3.1"
--- a/tests-fuzz/src/context.rs
+++ b/tests-fuzz/src/context.rs
@@ -12,9 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use std::fmt::Display;
 use std::sync::Arc;
+use std::vec;

 use common_query::AddColumnLocation;
+use common_telemetry::info;
+use common_time::timezone::parse_timezone;
+use datatypes::value::Value;
 use partition::partition::PartitionDef;
 use rand::Rng;
 use snafu::{ensure, OptionExt};
@@ -22,15 +27,171 @@ use snafu::{ensure, OptionExt};
 use crate::error::{self, Result};
 use crate::generator::Random;
 use crate::ir::alter_expr::AlterTableOperation;
-use crate::ir::{AlterTableExpr, Column, CreateTableExpr, Ident};
+use crate::ir::create_expr::ColumnOption;
+use crate::ir::insert_expr::RowValue;
+use crate::ir::{AlterTableExpr, Column, CreateTableExpr, Ident, InsertIntoExpr};

 pub type TableContextRef = Arc<TableContext>;

+#[derive(Debug, Clone, PartialEq, PartialOrd)]
+struct Cell {
+    value: mysql::Value,
+}
+
+impl Cell {
+    fn null() -> Self {
+        Self {
+            value: mysql::Value::NULL,
+        }
+    }
+}
+
+impl Display for Cell {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.value.as_sql(true))
+    }
+}
+
+impl From<Value> for Cell {
+    fn from(value: Value) -> Self {
+        if value.is_null() {
+            Self::null()
+        } else {
+            let s = match value {
+                Value::Timestamp(t) => {
+                    t.to_timezone_aware_string(Some(&parse_timezone(Some("+08:00"))))
+                }
+                Value::Boolean(b) => if b { "1" } else { "0" }.to_string(),
+                _ => value.to_string(),
+            };
+            Self {
+                value: mysql::Value::Bytes(s.into_bytes()),
+            }
+        }
+    }
+}
+
+impl From<mysql::Value> for Cell {
+    fn from(value: mysql::Value) -> Self {
+        Self { value }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+struct Row {
+    cells: Vec<Cell>,
+}
+
+impl Row {
+    fn with(columns_len: usize) -> Self {
+        Self {
+            cells: vec![Cell::null(); columns_len],
+        }
+    }
+
+    fn set_cell(&mut self, at: usize, cell: Cell) {
+        self.cells[at] = cell;
+    }
+
+    fn add_column(&mut self, at: usize, cell: Cell) {
+        self.cells.insert(at, cell);
+    }
+
+    fn drop_column(&mut self, at: usize) {
+        self.cells.remove(at);
+    }
+}
+
+impl Display for Row {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "[")?;
+        for (i, v) in self.cells.iter().enumerate() {
+            write!(f, "{}\t", v)?;
+            if i + 1 < self.cells.len() {
+                write!(f, "\t")?;
+            }
+        }
+        write!(f, "]")
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct Rows {
+    rows: Vec<Row>,
+}
+
+impl Rows {
+    pub fn empty(&self) -> bool {
+        self.rows.is_empty()
+    }
+
+    pub fn new() -> Self {
+        Self { rows: vec![] }
+    }
+
+    pub fn fill(rows: Vec<mysql::Row>) -> Self {
+        Self {
+            rows: rows
+                .into_iter()
+                .map(|r| Row {
+                    cells: r.unwrap().into_iter().map(Into::into).collect(),
+                })
+                .collect(),
+        }
+    }
+
+    fn with(rows: usize) -> Self {
+        Self {
+            rows: Vec::with_capacity(rows),
+        }
+    }
+
+    fn add_column(&mut self, at: usize, column: &Column) {
+        let cell = column
+            .options
+            .iter()
+            .find_map(|x| {
+                if let ColumnOption::DefaultValue(v) = x {
+                    Some(v.clone().into())
+                } else {
+                    None
+                }
+            })
+            .unwrap_or(Cell::null());
+        self.rows
+            .iter_mut()
+            .for_each(|x| x.add_column(at, cell.clone()))
+    }
+
+    fn drop_column(&mut self, at: usize) {
+        self.rows.iter_mut().for_each(|x| x.drop_column(at))
+    }
+
+    fn add_row(&mut self, row: Row) {
+        self.rows.push(row)
+    }
+
+    fn extend(&mut self, rows: Rows) {
+        self.rows.extend(rows.rows)
+    }
+}
+
+impl Display for Rows {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "Rows")?;
+        for r in &self.rows {
+            writeln!(f, "{}", r)?;
+        }
+        Ok(())
+    }
+}
+
 /// TableContext stores table info.
 #[derive(Debug, Clone)]
 pub struct TableContext {
    pub name: Ident,
    pub columns: Vec<Column>,
+    pub rows: Rows,

    // GreptimeDB specific options
    pub partition: Option<PartitionDef>,
@@ -50,6 +211,7 @@ impl From<&CreateTableExpr> for TableContext {
        Self {
            name: name.clone(),
            columns: columns.clone(),
+            rows: Rows::new(),
            partition: partition.clone(),
            primary_keys: primary_keys.clone(),
        }
@@ -57,8 +219,72 @@ impl From<&CreateTableExpr> for TableContext {
 }

 impl TableContext {
+    pub fn clear_data(&mut self) {
+        self.rows = Rows::new();
+    }
+
+    pub fn insert(&mut self, expr: InsertIntoExpr) -> Result<()> {
+        fn find_default_value(column: &Column) -> Cell {
+            column
+                .options
+                .iter()
+                .find_map(|opt| {
+                    if let ColumnOption::DefaultValue(v) = opt {
+                        Some(v.clone().into())
+                    } else {
+                        None
+                    }
+                })
+                .unwrap_or(Cell::null())
+        }
+
+        let mut rows = Rows::with(expr.values_list.len());
+
+        for insert_values in expr.values_list.into_iter() {
+            let mut row = Row::with(self.columns.len());
+
+            for (i, column) in self.columns.iter().enumerate() {
+                let cell = if let Some(v) = expr
+                    .columns
+                    .iter()
+                    .zip(insert_values.iter())
+                    .find_map(|(x, y)| if x.name == column.name { Some(y) } else { None })
+                {
+                    match v {
+                        RowValue::Value(v) => v.clone().into(),
+                        RowValue::Default => find_default_value(column),
+                    }
+                } else {
+                    find_default_value(column)
+                };
+                row.set_cell(i, cell);
+            }
+            rows.add_row(row);
+        }
+        self.rows.extend(rows);
+
+        let time_index = self.columns.iter().position(|x| x.is_time_index()).unwrap();
+        self.rows.rows.sort_by(|x, y| {
+            x.cells[time_index]
+                .partial_cmp(&y.cells[time_index])
+                .unwrap_or(std::cmp::Ordering::Equal)
+        });
+
+        info!("after insertion, expected rows: {}", self.rows);
+        Ok(())
+    }
+
+    fn find_column_index(&self, column_name: &str) -> Result<usize> {
+        self.columns
+            .iter()
+            .position(|col| col.name.to_string() == column_name)
+            .context(error::UnexpectedSnafu {
+                violated: format!("Column: {column_name} not found"),
+            })
+    }
+
    /// Applies the [AlterTableExpr].
-    pub fn alter(mut self, expr: AlterTableExpr) -> Result<TableContext> {
+    pub fn alter(&mut self, expr: AlterTableExpr) -> Result<()> {
        match expr.alter_options {
            AlterTableOperation::AddColumn { column, location } => {
                ensure!(
@@ -69,23 +295,18 @@ impl TableContext {
                );
                match location {
                    Some(AddColumnLocation::First) => {
-                        let mut columns = Vec::with_capacity(self.columns.len() + 1);
-                        columns.push(column);
-                        columns.extend(self.columns);
-                        self.columns = columns;
+                        self.rows.add_column(0, &column);
+                        self.columns.insert(0, column);
                    }
                    Some(AddColumnLocation::After { column_name }) => {
-                        let index = self
-                            .columns
-                            .iter()
-                            // TODO(weny): find a better way?
-                            .position(|col| col.name.to_string() == column_name)
-                            .context(error::UnexpectedSnafu {
-                                violated: format!("Column: {column_name} not found"),
-                            })?;
+                        let index = self.find_column_index(&column_name)?;
+                        self.rows.add_column(index + 1, &column);
                        self.columns.insert(index + 1, column);
                    }
-                    None => self.columns.push(column),
+                    None => {
+                        self.rows.add_column(self.columns.len(), &column);
+                        self.columns.push(column);
+                    }
                }
                // Re-generates the primary_keys
                self.primary_keys = self
@@ -100,10 +321,12 @@ impl TableContext {
                        }
                    })
                    .collect();
-                Ok(self)
            }
            AlterTableOperation::DropColumn { name } => {
-                self.columns.retain(|col| col.name != name);
+                let at = self.find_column_index(&name.to_string())?;
+                self.columns.remove(at);
+                self.rows.drop_column(at);
+
                // Re-generates the primary_keys
                self.primary_keys = self
                    .columns
@@ -117,7 +340,6 @@ impl TableContext {
                        }
                    })
                    .collect();
-                Ok(self)
            }
            AlterTableOperation::RenameTable { new_table_name } => {
                ensure!(
@@ -127,9 +349,9 @@ impl TableContext {
                    }
                );
                self.name = new_table_name;
-                Ok(self)
            }
        }
+        Ok(())
    }

    pub fn generate_unique_column_name<R: Rng>(
@@ -162,16 +384,17 @@ mod tests {
    use common_query::AddColumnLocation;
    use datatypes::data_type::ConcreteDataType;

-    use super::TableContext;
+    use super::*;
    use crate::ir::alter_expr::AlterTableOperation;
    use crate::ir::create_expr::ColumnOption;
    use crate::ir::{AlterTableExpr, Column, Ident};

    #[test]
    fn test_table_context_alter() {
-        let table_ctx = TableContext {
+        let mut table_ctx = TableContext {
            name: "foo".into(),
            columns: vec![],
+            rows: Rows::new(),
            partition: None,
            primary_keys: vec![],
        };
@@ -187,7 +410,7 @@ mod tests {
                location: None,
            },
        };
-        let table_ctx = table_ctx.alter(expr).unwrap();
+        table_ctx.alter(expr).unwrap();
        assert_eq!(table_ctx.columns[0].name, Ident::new("a"));
        assert_eq!(table_ctx.primary_keys, vec![0]);

@@ -203,7 +426,7 @@ mod tests {
                location: Some(AddColumnLocation::First),
            },
        };
-        let table_ctx = table_ctx.alter(expr).unwrap();
+        table_ctx.alter(expr).unwrap();
        assert_eq!(table_ctx.columns[0].name, Ident::new("b"));
        assert_eq!(table_ctx.primary_keys, vec![0, 1]);

@@ -221,7 +444,7 @@ mod tests {
                }),
            },
        };
-        let table_ctx = table_ctx.alter(expr).unwrap();
+        table_ctx.alter(expr).unwrap();
        assert_eq!(table_ctx.columns[1].name, Ident::new("c"));
        assert_eq!(table_ctx.primary_keys, vec![0, 1, 2]);

@@ -230,7 +453,7 @@ mod tests {
            table_name: "foo".into(),
            alter_options: AlterTableOperation::DropColumn { name: "b".into() },
        };
-        let table_ctx = table_ctx.alter(expr).unwrap();
+        table_ctx.alter(expr).unwrap();
        assert_eq!(table_ctx.columns[1].name, Ident::new("a"));
        assert_eq!(table_ctx.primary_keys, vec![0, 1]);
    }
--- a/tests-fuzz/src/data/lorem_words
+++ b/tests-fuzz/src/data/lorem_words
@@ -97,7 +97,6 @@ eum
 iure
 reprehenderit
 qui
-in
 ea
 voluptate
 velit
@@ -138,10 +137,8 @@ unde
 omnis
 iste
 natus
-error
 similique
 sunt
-in
 culpa
 qui
 officia
@@ -210,7 +207,6 @@ quo
 voluptas
 nulla
 pariatur
-at
 vero
 eos
 et
--- a/tests-fuzz/src/generator/alter_expr.rs
+++ b/tests-fuzz/src/generator/alter_expr.rs
@@ -27,7 +27,7 @@ use crate::generator::{ColumnOptionGenerator, ConcreteDataTypeGenerator, Generat
 use crate::ir::alter_expr::{AlterTableExpr, AlterTableOperation};
 use crate::ir::create_expr::ColumnOption;
 use crate::ir::{
-    droppable_columns, generate_columns, generate_random_value, ColumnTypeGenerator, Ident,
+    droppable_columns, generate_columns, generate_random_value_abs, ColumnTypeGenerator, Ident,
 };

 fn add_column_options_generator<R: Rng>(
@@ -41,7 +41,7 @@ fn add_column_options_generator<R: Rng>(
    match idx {
        0 => vec![ColumnOption::Null],
        1 => {
-            vec![ColumnOption::DefaultValue(generate_random_value(
+            vec![ColumnOption::DefaultValue(generate_random_value_abs(
                rng,
                column_type,
                None,
@@ -50,7 +50,7 @@ fn add_column_options_generator<R: Rng>(
        2 => {
            vec![
                ColumnOption::PrimaryKey,
-                ColumnOption::DefaultValue(generate_random_value(rng, column_type, None)),
+                ColumnOption::DefaultValue(generate_random_value_abs(rng, column_type, None)),
            ]
        }
        _ => unreachable!(),
--- a/tests-fuzz/src/generator/insert_expr.rs
+++ b/tests-fuzz/src/generator/insert_expr.rs
@@ -94,11 +94,7 @@ impl<R: Rng + 'static> Generator<InsertIntoExpr, R> for InsertExprGenerator<R> {

        Ok(InsertIntoExpr {
            table_name: self.table_ctx.name.to_string(),
-            columns: if omit_column_list {
-                vec![]
-            } else {
-                values_columns
-            },
+            columns: values_columns,
            values_list,
        })
    }
--- a/tests-fuzz/src/ir.rs
+++ b/tests-fuzz/src/ir.rs
@@ -15,14 +15,15 @@
 //! The intermediate representation

 pub(crate) mod alter_expr;
-pub(crate) mod create_expr;
-pub(crate) mod insert_expr;
-pub(crate) mod select_expr;
+pub mod create_expr;
+pub mod insert_expr;
+pub mod select_expr;

 use core::fmt;

-pub use alter_expr::AlterTableExpr;
-use common_time::{Date, DateTime, Timestamp};
+pub use alter_expr::{AlterTableExpr, AlterTableOperation};
+use common_time::timestamp::TimeUnit;
+use common_time::{Date, DateTime, Interval, Timestamp};
 pub use create_expr::{CreateDatabaseExpr, CreateTableExpr};
 use datatypes::data_type::ConcreteDataType;
 use datatypes::types::TimestampType;
@@ -102,29 +103,65 @@ pub fn generate_random_value<R: Rng>(
    }
 }

+pub fn generate_random_value_abs<R: Rng>(
+    rng: &mut R,
+    datatype: &ConcreteDataType,
+    random_str: Option<&dyn Random<Ident, R>>,
+) -> Value {
+    match datatype {
+        &ConcreteDataType::Boolean(_) => Value::from(rng.gen::<bool>()),
+        ConcreteDataType::Int16(_) => Value::from(i16::abs(rng.gen::<i16>())),
+        ConcreteDataType::Int32(_) => Value::from(i32::abs(rng.gen::<i32>())),
+        ConcreteDataType::Int64(_) => Value::from(i64::abs(rng.gen::<i64>())),
+        ConcreteDataType::Float32(_) => Value::from(f32::abs(rng.gen::<f32>())),
+        ConcreteDataType::Float64(_) => Value::from(f64::abs(rng.gen::<f64>())),
+        ConcreteDataType::String(_) => match random_str {
+            Some(random) => Value::from(random.gen(rng).value),
+            None => Value::from(rng.gen::<char>().to_string()),
+        },
+        ConcreteDataType::Date(_) => generate_random_date(rng),
+        ConcreteDataType::DateTime(_) => generate_random_datetime(rng),
+        &ConcreteDataType::Timestamp(ts_type) => generate_random_timestamp(rng, ts_type),
+
+        _ => unimplemented!("unsupported type: {datatype}"),
+    }
+}
+
 fn generate_random_timestamp<R: Rng>(rng: &mut R, ts_type: TimestampType) -> Value {
    let v = match ts_type {
        TimestampType::Second(_) => {
-            let min = i64::from(Timestamp::MIN_SECOND);
-            let max = i64::from(Timestamp::MAX_SECOND);
+            let now = Timestamp::current_time(TimeUnit::Second);
+            let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
+            let max = now.add_interval(Interval::from_year_month(12)).unwrap();
+            let min = i64::from(min);
+            let max = i64::from(max);
            let value = rng.gen_range(min..=max);
            Timestamp::new_second(value)
        }
        TimestampType::Millisecond(_) => {
-            let min = i64::from(Timestamp::MIN_MILLISECOND);
-            let max = i64::from(Timestamp::MAX_MILLISECOND);
+            let now = Timestamp::current_time(TimeUnit::Millisecond);
+            let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
+            let max = now.add_interval(Interval::from_year_month(12)).unwrap();
+            let min = i64::from(min);
+            let max = i64::from(max);
            let value = rng.gen_range(min..=max);
            Timestamp::new_millisecond(value)
        }
        TimestampType::Microsecond(_) => {
-            let min = i64::from(Timestamp::MIN_MICROSECOND);
-            let max = i64::from(Timestamp::MAX_MICROSECOND);
+            let now = Timestamp::current_time(TimeUnit::Microsecond);
+            let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
+            let max = now.add_interval(Interval::from_year_month(12)).unwrap();
+            let min = i64::from(min);
+            let max = i64::from(max);
            let value = rng.gen_range(min..=max);
            Timestamp::new_microsecond(value)
        }
        TimestampType::Nanosecond(_) => {
-            let min = i64::from(Timestamp::MIN_NANOSECOND);
-            let max = i64::from(Timestamp::MAX_NANOSECOND);
+            let now = Timestamp::current_time(TimeUnit::Nanosecond);
+            let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
+            let max = now.add_interval(Interval::from_year_month(12)).unwrap();
+            let min = i64::from(min);
+            let max = i64::from(max);
            let value = rng.gen_range(min..=max);
            Timestamp::new_nanosecond(value)
        }
@@ -278,7 +315,7 @@ pub fn column_options_generator<R: Rng>(
    match option_idx {
        0 => vec![ColumnOption::Null],
        1 => vec![ColumnOption::NotNull],
-        2 => vec![ColumnOption::DefaultValue(generate_random_value(
+        2 => vec![ColumnOption::DefaultValue(generate_random_value_abs(
            rng,
            column_type,
            None,
--- a/tests-fuzz/src/ir/insert_expr.rs
+++ b/tests-fuzz/src/ir/insert_expr.rs
@@ -18,6 +18,7 @@ use datatypes::value::Value;

 use crate::ir::Column;

+#[derive(Debug)]
 pub struct InsertIntoExpr {
    pub table_name: String,
    pub columns: Vec<Column>,
@@ -26,6 +27,7 @@ pub struct InsertIntoExpr {

 pub type RowValues = Vec<RowValue>;

+#[derive(Debug)]
 pub enum RowValue {
    Value(Value),
    Default,
--- a/tests-fuzz/src/test_utils.rs
+++ b/tests-fuzz/src/test_utils.rs
@@ -14,7 +14,7 @@

 use datatypes::data_type::ConcreteDataType;

-use crate::context::TableContext;
+use crate::context::{Rows, TableContext};
 use crate::ir::create_expr::ColumnOption;
 use crate::ir::Column;

@@ -53,6 +53,7 @@ pub fn new_test_ctx() -> TableContext {
                options: vec![ColumnOption::TimeIndex],
            },
        ],
+        rows: Rows::new(),
        partition: None,
        primary_keys: vec![],
    }
--- a/tests-fuzz/src/validator.rs
+++ b/tests-fuzz/src/validator.rs
@@ -12,4 +12,5 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#[allow(dead_code)]
 pub mod column;
--- a/tests-fuzz/src/validator/column.rs
+++ b/tests-fuzz/src/validator/column.rs
@@ -16,7 +16,9 @@ use common_telemetry::debug;
 use datatypes::data_type::DataType;
 use snafu::{ensure, ResultExt};
 use sqlx::database::HasArguments;
-use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, Type};
+use sqlx::{
+    ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, MySql, Pool, Row, Type,
+};

 use crate::error::{self, Result};
 use crate::ir::create_expr::ColumnOption;
@@ -24,12 +26,15 @@ use crate::ir::{Column, Ident};

 #[derive(Debug, sqlx::FromRow)]
 pub struct ColumnEntry {
-    pub table_schema: String,
-    pub table_name: String,
+    #[sqlx(rename = "Field")]
    pub column_name: String,
+    #[sqlx(rename = "Type")]
    pub data_type: String,
+    #[sqlx(rename = "Semantic Type")]
    pub semantic_type: String,
+    #[sqlx(rename = "Default")]
    pub column_default: Option<String>,
+    #[sqlx(rename = "Null")]
    pub is_nullable: String,
 }

@@ -45,6 +50,7 @@ enum SemanticType {

 fn semantic_type(str: &str) -> Option<SemanticType> {
    match str {
+        "TIME INDEX" => Some(SemanticType::Timestamp),
        "TIMESTAMP" => Some(SemanticType::Timestamp),
        "FIELD" => Some(SemanticType::Field),
        "TAG" => Some(SemanticType::Tag),
@@ -127,43 +133,43 @@ impl PartialEq<Column> for ColumnEntry {
                return false;
            }
        }
-        //TODO: Checks `semantic_type`
-        match semantic_type(&self.semantic_type) {
-            Some(SemanticType::Tag) => {
-                if !other
-                    .options
-                    .iter()
-                    .any(|opt| matches!(opt, ColumnOption::PrimaryKey))
-                {
-                    debug!("ColumnOption::PrimaryKey is not found");
-                    return false;
-                }
-            }
-            Some(SemanticType::Field) => {
-                if other
-                    .options
-                    .iter()
-                    .any(|opt| matches!(opt, ColumnOption::PrimaryKey | ColumnOption::TimeIndex))
-                {
-                    debug!("unexpected ColumnOption::PrimaryKey or ColumnOption::TimeIndex");
-                    return false;
-                }
-            }
-            Some(SemanticType::Timestamp) => {
-                if !other
-                    .options
-                    .iter()
-                    .any(|opt| matches!(opt, ColumnOption::TimeIndex))
-                {
-                    debug!("ColumnOption::TimeIndex is not found");
-                    return false;
-                }
-            }
-            None => {
-                debug!("unknown semantic type: {}", self.semantic_type);
-                return false;
-            }
-        };
+        // //TODO: Checks `semantic_type`
+        // match semantic_type(&self.semantic_type) {
+        //     Some(SemanticType::Tag) => {
+        //         if !other
+        //             .options
+        //             .iter()
+        //             .any(|opt| matches!(opt, ColumnOption::PrimaryKey))
+        //         {
+        //             debug!("ColumnOption::PrimaryKey is not found");
+        //             return false;
+        //         }
+        //     }
+        //     Some(SemanticType::Field) => {
+        //         if other
+        //             .options
+        //             .iter()
+        //             .any(|opt| matches!(opt, ColumnOption::PrimaryKey | ColumnOption::TimeIndex))
+        //         {
+        //             debug!("unexpected ColumnOption::PrimaryKey or ColumnOption::TimeIndex");
+        //             return false;
+        //         }
+        //     }
+        //     Some(SemanticType::Timestamp) => {
+        //         if !other
+        //             .options
+        //             .iter()
+        //             .any(|opt| matches!(opt, ColumnOption::TimeIndex))
+        //         {
+        //             debug!("ColumnOption::TimeIndex is not found");
+        //             return false;
+        //         }
+        //     }
+        //     None => {
+        //         debug!("unknown semantic type: {}", self.semantic_type);
+        //         return false;
+        //     }
+        // };

        true
    }
@@ -220,6 +226,37 @@ where
        .context(error::ExecuteQuerySnafu { sql })
 }

+pub async fn fetch_columns_via_mysql(
+    db: &Pool<MySql>,
+    _schema_name: Ident,
+    table_name: Ident,
+) -> Result<Vec<ColumnEntry>> {
+    let sql = format!("DESC TABLE {table_name}");
+    let rows = sqlx::query(&sql)
+        .fetch_all(db)
+        .await
+        .context(error::ExecuteQuerySnafu { sql })?;
+    Ok(rows
+        .into_iter()
+        .map(|row| {
+            let default_value: String = row.get(3);
+            let column_default = if default_value.is_empty() {
+                None
+            } else {
+                Some(default_value)
+            };
+
+            ColumnEntry {
+                column_name: row.get(0),
+                data_type: row.get(1),
+                is_nullable: row.get(2),
+                column_default,
+                semantic_type: row.get(4),
+            }
+        })
+        .collect::<Vec<_>>())
+}
+
 #[cfg(test)]
 mod tests {
    use datatypes::data_type::{ConcreteDataType, DataType};
@@ -233,8 +270,6 @@ mod tests {
    fn test_column_eq() {
        common_telemetry::init_default_ut_logging();
        let column_entry = ColumnEntry {
-            table_schema: String::new(),
-            table_name: String::new(),
            column_name: "test".to_string(),
            data_type: ConcreteDataType::int8_datatype().name(),
            semantic_type: "FIELD".to_string(),
@@ -257,8 +292,6 @@ mod tests {
        assert!(column_entry == column);
        // With default value
        let column_entry = ColumnEntry {
-            table_schema: String::new(),
-            table_name: String::new(),
            column_name: "test".to_string(),
            data_type: ConcreteDataType::int8_datatype().to_string(),
            semantic_type: "FIELD".to_string(),
@@ -273,8 +306,6 @@ mod tests {
        assert!(column_entry == column);
        // With default function
        let column_entry = ColumnEntry {
-            table_schema: String::new(),
-            table_name: String::new(),
            column_name: "test".to_string(),
            data_type: ConcreteDataType::int8_datatype().to_string(),
            semantic_type: "FIELD".to_string(),
--- a/tests-fuzz/targets/fuzz_alter_table.rs
+++ b/tests-fuzz/targets/fuzz_alter_table.rs
@@ -21,23 +21,22 @@ use common_telemetry::info;
 use libfuzzer_sys::fuzz_target;
 use rand::{Rng, SeedableRng};
 use rand_chacha::ChaChaRng;
-use snafu::ResultExt;
-use sqlx::{MySql, Pool};
+use snafu::{ensure, ResultExt};
+use sqlx::{Executor, MySql, Pool};
 use tests_fuzz::context::{TableContext, TableContextRef};
 use tests_fuzz::error::{self, Result};
-use tests_fuzz::fake::{
-    merge_two_word_map_fn, random_capitalize_map, uppercase_and_keyword_backtick_map,
-    MappedGenerator, WordGenerator,
-};
+use tests_fuzz::fake::WordGenerator;
 use tests_fuzz::generator::alter_expr::{
    AlterExprAddColumnGeneratorBuilder, AlterExprDropColumnGeneratorBuilder,
    AlterExprRenameGeneratorBuilder,
 };
 use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder;
+use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder;
 use tests_fuzz::generator::Generator;
-use tests_fuzz::ir::{droppable_columns, AlterTableExpr, CreateTableExpr};
+use tests_fuzz::ir::{droppable_columns, AlterTableExpr, CreateTableExpr, InsertIntoExpr};
 use tests_fuzz::translator::mysql::alter_expr::AlterTableExprTranslator;
 use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator;
+use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator;
 use tests_fuzz::translator::DslTranslator;
 use tests_fuzz::utils::{init_greptime_connections, Connections};
 use tests_fuzz::validator;
@@ -52,19 +51,17 @@ impl FuzzContext {
    }
 }

-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Copy)]
 struct FuzzInput {
    seed: u64,
    actions: usize,
+    rows: usize,
 }

 fn generate_create_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTableExpr> {
    let columns = rng.gen_range(2..30);
    let create_table_generator = CreateTableExprGeneratorBuilder::default()
-        .name_generator(Box::new(MappedGenerator::new(
-            WordGenerator,
-            merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
-        )))
+        .name_generator(Box::new(WordGenerator))
        .columns(columns)
        .engine("mito")
        .build()
@@ -80,10 +77,7 @@ fn generate_alter_table_expr<R: Rng + 'static>(
    if rename {
        let expr_generator = AlterExprRenameGeneratorBuilder::default()
            .table_ctx(table_ctx)
-            .name_generator(Box::new(MappedGenerator::new(
-                WordGenerator,
-                merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
-            )))
+            .name_generator(Box::new(WordGenerator))
            .build()
            .unwrap();
        expr_generator.generate(rng)
@@ -112,11 +106,29 @@ impl Arbitrary<'_> for FuzzInput {
        let seed = u.int_in_range(u64::MIN..=u64::MAX)?;
        let mut rng = ChaChaRng::seed_from_u64(seed);
        let actions = rng.gen_range(1..256);
+        let insertions = rng.gen_range(1..1024);

-        Ok(FuzzInput { seed, actions })
+        Ok(FuzzInput {
+            seed,
+            actions,
+            rows: insertions,
+        })
    }
 }

+fn generate_insert_expr<R: Rng + 'static>(
+    input: FuzzInput,
+    rng: &mut R,
+    table_ctx: TableContextRef,
+) -> Result<InsertIntoExpr> {
+    let insert_generator = InsertExprGeneratorBuilder::default()
+        .table_ctx(table_ctx)
+        .rows(input.rows)
+        .build()
+        .unwrap();
+    insert_generator.generate(rng)
+}
+
 async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
    info!("input: {input:?}");
    let mut rng = ChaChaRng::seed_from_u64(input.seed);
@@ -143,10 +155,12 @@ async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
            .context(error::ExecuteQuerySnafu { sql: &sql })?;
        info!("Alter table: {sql}, result: {result:?}");
        // Applies changes
-        table_ctx = Arc::new(Arc::unwrap_or_clone(table_ctx).alter(expr).unwrap());
+        let mut t = Arc::unwrap_or_clone(table_ctx);
+        t.alter(expr).unwrap();
+        table_ctx = Arc::new(t);

        // Validates columns
-        let mut column_entries = validator::column::fetch_columns(
+        let mut column_entries = validator::column::fetch_columns_via_mysql(
            &ctx.greptime,
            "public".into(),
            table_ctx.name.clone(),
@@ -156,6 +170,28 @@ async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
        let mut columns = table_ctx.columns.clone();
        columns.sort_by(|a, b| a.name.value.cmp(&b.name.value));
        validator::column::assert_eq(&column_entries, &columns)?;
+
+        // insertions
+        let insert_expr = generate_insert_expr(input, &mut rng, table_ctx.clone())?;
+        let translator = InsertIntoExprTranslator;
+        let sql = translator.translate(&insert_expr)?;
+        let result = ctx
+            .greptime
+            // unprepared query, see <https://github.com/GreptimeTeam/greptimedb/issues/3500>
+            .execute(sql.as_str())
+            .await
+            .context(error::ExecuteQuerySnafu { sql: &sql })?;
+
+        ensure!(
+            result.rows_affected() == input.rows as u64,
+            error::AssertSnafu {
+                reason: format!(
+                    "expected rows affected: {}, actual: {}",
+                    input.rows,
+                    result.rows_affected(),
+                )
+            }
+        );
    }

    // Cleans up
--- a/tests/cases/standalone/common/create/create_metric_table.result
+++ b/tests/cases/standalone/common/create/create_metric_table.result
@@ -71,7 +71,26 @@ DESC TABLE t2;
 | val    | Float64              |     | YES  |         | FIELD         |
 +--------+----------------------+-----+------+---------+---------------+

-- TODO(ruihang): add a case that drops phy before t1
+-- should be failed
+-- SQLNESS REPLACE (region\s\d+\(\d+\,\s\d+\)) region
+DROP TABLE phy;
+
+Error: 1004(InvalidArguments), Physical region is busy, there are still some logical regions using it
+
+-- metadata should be restored
+DESC TABLE phy;
+
+------------+----------------------+-----+------+---------+---------------+
+| Column     | Type                 | Key | Null | Default | Semantic Type |
+------------+----------------------+-----+------+---------+---------------+
+| ts         | TimestampMillisecond |     | NO   |         | FIELD         |
+| val        | Float64              |     | YES  |         | FIELD         |
+| __table_id | UInt32               | PRI | NO   |         | TAG           |
+| __tsid     | UInt64               | PRI | NO   |         | TAG           |
+| host       | String               | PRI | YES  |         | TAG           |
+| job        | String               | PRI | YES  |         | TAG           |
+------------+----------------------+-----+------+---------+---------------+
+
 DROP TABLE t1;

 Affected Rows: 0
--- a/tests/cases/standalone/common/create/create_metric_table.sql
+++ b/tests/cases/standalone/common/create/create_metric_table.sql
@@ -16,7 +16,11 @@ DESC TABLE t1;

 DESC TABLE t2;

-- TODO(ruihang): add a case that drops phy before t1
+-- should be failed
+-- SQLNESS REPLACE (region\s\d+\(\d+\,\s\d+\)) region
+DROP TABLE phy;
+-- metadata should be restored
+DESC TABLE phy;

 DROP TABLE t1;
Author	SHA1	Message	Date
luofucong	ee67ce10c9	insert some rows and query them across DDL to test the integrity of data	2024-05-14 20:41:47 +08:00
WenyXu	2ba721cc82	fix: fix bug	2024-05-13 08:25:18 +00:00
WenyXu	de468ee595	fix: test program	2024-05-13 07:46:30 +00:00
WenyXu	bb9bdf74ec	feat: export metric endpoint	2024-05-13 02:44:34 +00:00
WenyXu	be5574fdb3	fuzz alter	2024-04-19 14:14:34 +00:00
WenyXu	f9afc5dbbf	feat: `start_database` auto retry	2024-04-19 02:40:15 +00:00
WenyXu	c7400a4182	chore: reduce sleep time	2024-04-18 13:59:20 +00:00
WenyXu	bf07dd275a	test: reproduce bugs	2024-04-18 13:48:11 +00:00
WenyXu	7e1eed4b18	feat: adapt for cuckoo	2024-04-16 14:52:06 +00:00
Weny Xu	d12379106e	feat(drop_table): support to rollback table metadata (#3692 ) * feat: support to rollback table metadata * refactor: store table route value instead of physical table route * feat(drop_table): support to rollback table metadata * test: add rollback tests for drop table * fix: do not set region to readonly * test: add sqlness tests * feat: implement TombstoneManager * test: add tests for TombstoneManager * refactor: using TombstoneManager * chore: remove unused code * fix: fix typo * refactor: using `on_restore_metadata` * refactor: add `executor` to `DropTableProcedure` * refactor: simplify the `TombstoneManager` * refactor: refactor `Key` * refactor: carry more info * feat: add `destroy_table_metadata` * refactor: remove redundant table_route_value * feat: ensure the key is empty * feat: introcude `table_metadata_keys` * chore: carry more info * chore: remove clone * chore: apply suggestions from CR * feat: delete metadata tombstone	2024-04-16 09:22:41 +00:00