feat: resolve unused dependencies with cargo-udeps (#6578) (#6619)

* feat:resolve unused dependencies with cargo-udeps (#6578)

Signed-off-by: Arshdeep54 <balarsh535@gmail.com>

* Apply suggestion from @zyy17

Co-authored-by: zyy17 <zyylsxm@gmail.com>

* Apply suggestion from @zyy17

Co-authored-by: zyy17 <zyylsxm@gmail.com>

---------

Signed-off-by: Arshdeep54 <balarsh535@gmail.com>
Co-authored-by: Ning Sun <classicning@gmail.com>
Co-authored-by: zyy17 <zyylsxm@gmail.com>

.github/workflows/develop.yml

@@ -683,6 +683,30 @@ jobs:
       - name: Run cargo clippy
         run: make clippy
 
+  check-udeps:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    name: Check Unused Dependencies
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "check-udeps"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Install cargo-udeps
+        run: cargo install cargo-udeps --locked
+      - name: Check unused dependencies
+        run: make check-udeps
+
   conflict-check:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check for conflict
@@ -698,7 +722,7 @@ jobs:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'merge_group' }}
     runs-on: ubuntu-22.04-arm
     timeout-minutes: 60
-    needs:  [conflict-check, clippy, fmt]
+    needs: [conflict-check, clippy, fmt, check-udeps]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -824,3 +848,4 @@ jobs:
   #         mkdir -p ./bins/current
   #         tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current
   #     - run: ./tests/compat/test-compat.sh 0.6.0
+  

.github/workflows/docs.yml

@@ -67,6 +67,12 @@ jobs:
     steps:
       - run: 'echo "No action required"'
 
+  check-udeps:
+    name: Unused Dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No action required"'
+
   coverage:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -52,6 +52,9 @@ venv/
 tests-fuzz/artifacts/
 tests-fuzz/corpus/
 
+# cargo-udeps reports
+udeps-report.json
+
 # Nix
 .direnv
 .envrc

CONTRIBUTING.md

@@ -57,6 +57,9 @@ GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptim
 - Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/) and [style guide](docs/style-guide.md).
 - Make sure all unit tests are passed using [nextest](https://nexte.st/index.html) `cargo nextest run --workspace --features pg_kvbackend,mysql_kvbackend` or `make test`.
 - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings` or `make clippy`).
+- Ensure there are no unused dependencies by running `make check-udeps` (clean them up with `make fix-udeps` if reported).
+- If you must keep a target-specific dependency (e.g. under `[target.'cfg(...)'.dev-dependencies]`), add a cargo-udeps ignore entry in the same `Cargo.toml`, for example:
+  `[package.metadata.cargo-udeps.ignore]` with `development = ["rexpect"]` (or `dependencies`/`build` as appropriate).
 - When modifying sample configuration files in `config/`, run `make config-docs` (which requires Docker to be installed) to update the configuration documentation and include it in your commit.
 
 #### `pre-commit` Hooks

Cargo.lock

@@ -2189,7 +2189,7 @@ dependencies = [
  "hyperloglogplus",
  "jsonb",
  "memchr",
- "nalgebra 0.33.2",
+ "nalgebra",
  "num",
  "num-traits",
  "once_cell",
@@ -2201,7 +2201,6 @@ dependencies = [
  "session",
  "snafu 0.8.6",
  "sql",
- "statrs",
  "store-api",
  "table",
  "tokio",
@@ -2283,15 +2282,10 @@ dependencies = [
 name = "common-macro"
 version = "0.17.0"
 dependencies = [
- "arc-swap",
- "common-query",
- "datatypes",
  "greptime-proto",
  "once_cell",
  "proc-macro2",
  "quote",
- "snafu 0.8.6",
- "static_assertions",
  "syn 2.0.104",
 ]
 
@@ -2469,7 +2463,6 @@ dependencies = [
  "snafu 0.8.6",
  "sqlparser 0.55.0-greptime",
  "sqlparser_derive 0.1.1",
- "statrs",
  "store-api",
  "tokio",
 ]
@@ -2512,7 +2505,6 @@ dependencies = [
  "paste",
  "pin-project",
  "prometheus",
- "rand 0.9.1",
  "ratelimit",
  "serde",
  "serde_json",
@@ -2538,7 +2530,6 @@ name = "common-sql"
 version = "0.17.0"
 dependencies = [
  "common-base",
- "common-datasource",
  "common-decimal",
  "common-error",
  "common-macro",
@@ -2650,7 +2641,6 @@ dependencies = [
 name = "common-workload"
 version = "0.17.0"
 dependencies = [
- "api",
  "common-telemetry",
  "serde",
 ]
@@ -7054,7 +7044,6 @@ dependencies = [
  "protobuf-build",
  "raft-engine",
  "rand 0.9.1",
- "rand_distr",
  "rskafka",
  "serde",
  "serde_json",
@@ -7407,7 +7396,6 @@ dependencies = [
  "itertools 0.14.0",
  "lazy_static",
  "local-ip-address",
- "log-store",
  "once_cell",
  "parking_lot 0.12.4",
  "prometheus",
@@ -7902,24 +7890,6 @@ dependencies = [
  "zstd 0.13.3",
 ]
 
-[[package]]
-name = "nalgebra"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d506eb7e08d6329505faa8a3a00a5dcc6de9f76e0c77e4b75763ae3c770831ff"
-dependencies = [
- "approx 0.5.1",
- "matrixmultiply",
- "nalgebra-macros 0.1.0",
- "num-complex",
- "num-rational",
- "num-traits",
- "rand 0.8.5",
- "rand_distr",
- "simba 0.6.0",
- "typenum",
-]
-
 [[package]]
 name = "nalgebra"
 version = "0.33.2"
@@ -7928,25 +7898,14 @@ checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
 dependencies = [
  "approx 0.5.1",
  "matrixmultiply",
- "nalgebra-macros 0.2.2",
+ "nalgebra-macros",
  "num-complex",
  "num-rational",
  "num-traits",
- "simba 0.9.0",
+ "simba",
  "typenum",
 ]
 
-[[package]]
-name = "nalgebra-macros"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01fcc0b8149b4632adc89ac3b7b31a12fb6099a0317a4eb2ebff574ef7de7218"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "nalgebra-macros"
 version = "0.2.2"
@@ -9382,7 +9341,6 @@ dependencies = [
  "serde",
  "serde_json",
  "session",
- "simd-json",
  "snafu 0.8.6",
  "sql",
  "table",
@@ -10157,7 +10115,7 @@ dependencies = [
  "log-query",
  "meter-core",
  "meter-macros",
- "nalgebra 0.33.2",
+ "nalgebra",
  "num",
  "num-traits",
  "object-store",
@@ -10177,7 +10135,6 @@ dependencies = [
  "snafu 0.8.6",
  "sql",
  "sqlparser 0.55.0-greptime",
- "statrs",
  "store-api",
  "substrait 0.17.0",
  "table",
@@ -11563,7 +11520,6 @@ dependencies = [
  "client",
  "common-base",
  "common-catalog",
- "common-config",
  "common-error",
  "common-frontend",
  "common-grpc",
@@ -11775,19 +11731,6 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
-[[package]]
-name = "simba"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0b7840f121a46d63066ee7a99fc81dcabbc6105e437cae43528cea199b5a05f"
-dependencies = [
- "approx 0.5.1",
- "num-complex",
- "num-traits",
- "paste",
- "wide",
-]
-
 [[package]]
 name = "simba"
 version = "0.9.0"
@@ -12376,19 +12319,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
-[[package]]
-name = "statrs"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b35a062dbadac17a42e0fc64c27f419b25d6fae98572eb43c8814c9e873d7721"
-dependencies = [
- "approx 0.5.1",
- "lazy_static",
- "nalgebra 0.29.0",
- "num-traits",
- "rand 0.8.5",
-]
-
 [[package]]
 name = "store-api"
 version = "0.17.0"

Makefile

@@ -193,6 +193,17 @@ clippy: ## Check clippy rules.
 fix-clippy: ## Fix clippy violations.
 	cargo clippy --workspace --all-targets --all-features --fix
 
+.PHONY: check-udeps
+check-udeps: ## Check unused dependencies.
+	cargo udeps --workspace --all-targets
+
+.PHONY: fix-udeps
+fix-udeps: ## Remove unused dependencies automatically.
+	@echo "Running cargo-udeps to find unused dependencies..."
+	@cargo udeps --workspace --all-targets --output json > udeps-report.json || true
+	@echo "Removing unused dependencies..."
+	@python3 scripts/fix-udeps.py udeps-report.json
+	
 .PHONY: fmt-check
 fmt-check: ## Check code format.
 	cargo fmt --all -- --check

scripts/fix-udeps.py

@@ -0,0 +1,265 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import sys
+
+
+def load_udeps_report(report_path):
+    try:
+        with open(report_path, "r") as f:
+            return json.load(f)
+    except FileNotFoundError:
+        print(f"Error: Report file '{report_path}' not found.")
+        return None
+    except json.JSONDecodeError as e:
+        print(f"Error: Invalid JSON in report file: {e}")
+        return None
+
+
+def extract_unused_dependencies(report):
+    """
+    Extract and organize unused dependencies from the cargo-udeps JSON report.
+
+    The cargo-udeps report has this structure:
+    {
+        "unused_deps": {
+            "package_name v0.1.0 (/path/to/package)": {
+                "normal": ["dep1", "dep2"],
+                "development": ["dev_dep1"],
+                "build": ["build_dep1"],
+                "manifest_path": "/path/to/Cargo.toml"
+            }
+        }
+    }
+
+    Args:
+        report (dict): The parsed JSON report from cargo-udeps
+
+    Returns:
+        dict: Organized unused dependencies by package name:
+        {
+            "package_name": {
+                "dependencies": [("dep1", "normal"), ("dev_dep1", "dev")],
+                "manifest_path": "/path/to/Cargo.toml"
+            }
+        }
+    """
+    if not report or "unused_deps" not in report:
+        return {}
+
+    unused_deps = {}
+    for package_full_name, deps_info in report["unused_deps"].items():
+        package_name = package_full_name.split(" ")[0]
+
+        all_unused = []
+        if deps_info.get("normal"):
+            all_unused.extend([(dep, "normal") for dep in deps_info["normal"]])
+        if deps_info.get("development"):
+            all_unused.extend([(dep, "dev") for dep in deps_info["development"]])
+        if deps_info.get("build"):
+            all_unused.extend([(dep, "build") for dep in deps_info["build"]])
+
+        if all_unused:
+            unused_deps[package_name] = {
+                "dependencies": all_unused,
+                "manifest_path": deps_info.get("manifest_path", "unknown"),
+            }
+
+    return unused_deps
+
+
+def get_section_pattern(dep_type):
+    """
+    Get regex patterns to identify different dependency sections in Cargo.toml.
+
+    Args:
+        dep_type (str): Type of dependency ("normal", "dev", or "build")
+
+    Returns:
+        list: List of regex patterns to match the appropriate section headers
+
+    """
+    patterns = {
+        "normal": [r"\[dependencies\]", r"\[dependencies\..*?\]"],
+        "dev": [r"\[dev-dependencies\]", r"\[dev-dependencies\..*?\]"],
+        "build": [r"\[build-dependencies\]", r"\[build-dependencies\..*?\]"],
+    }
+    return patterns.get(dep_type, [])
+
+
+def remove_dependency_line(content, dep_name, section_start, section_end):
+    """
+    Remove a dependency line from a specific section of a Cargo.toml file.
+
+    Args:
+        content (str): The entire content of the Cargo.toml file
+        dep_name (str): Name of the dependency to remove (e.g., "serde", "tokio")
+        section_start (int): Starting position of the section in the content
+        section_end (int): Ending position of the section in the content
+
+    Returns:
+        tuple: (new_content, removed) where:
+            - new_content (str): The modified content with dependency removed
+            - removed (bool): True if dependency was found and removed, False otherwise
+
+    Example input content format:
+        content = '''
+        [package]
+        name = "my-crate"
+        version = "0.1.0"
+
+        [dependencies]
+        serde = "1.0"
+        tokio = { version = "1.0", features = ["full"] }
+        serde_json.workspace = true
+
+        [dev-dependencies]
+        tempfile = "3.0"
+        '''
+
+        # If dep_name = "serde", section_start = start of [dependencies],
+        # section_end = start of [dev-dependencies], this function will:
+        # 1. Extract the section: "serde = "1.0"\ntokio = { version = "1.0", features = ["full"] }\nserde_json.workspace = true\n"
+        # 2. Find and remove the line: "serde = "1.0""
+        # 3. Return the modified content with that line removed
+    """
+    section_content = content[section_start:section_end]
+
+    dep_patterns = [
+        rf"^{re.escape(dep_name)}\s*=.*$",  # e.g., "serde = "1.0""
+        rf"^{re.escape(dep_name)}\.workspace\s*=.*$",  # e.g., "serde_json.workspace = true"
+    ]
+
+    for pattern in dep_patterns:
+        match = re.search(pattern, section_content, re.MULTILINE)
+        if match:
+            line_start = section_start + match.start()  # Start of the matched line
+            line_end = section_start + match.end()  # End of the matched line
+
+            if line_end < len(content) and content[line_end] == "\n":
+                line_end += 1
+
+            return content[:line_start] + content[line_end:], True
+
+    return content, False
+
+
+def remove_dependency_from_toml(file_path, dep_name, dep_type):
+    """
+    Remove a specific dependency from a Cargo.toml file.
+
+    Args:
+        file_path (str): Path to the Cargo.toml file
+        dep_name (str): Name of the dependency to remove
+        dep_type (str): Type of dependency ("normal", "dev", or "build")
+
+    Returns:
+        bool: True if dependency was successfully removed, False otherwise
+    """
+    try:
+        with open(file_path, "r") as f:
+            content = f.read()
+
+        section_patterns = get_section_pattern(dep_type)
+        if not section_patterns:
+            return False
+
+        for pattern in section_patterns:
+            section_match = re.search(pattern, content, re.IGNORECASE)
+            if not section_match:
+                continue
+
+            section_start = section_match.end()
+            next_section = re.search(r"\n\s*\[", content[section_start:])
+            section_end = (
+                section_start + next_section.start() if next_section else len(content)
+            )
+
+            new_content, removed = remove_dependency_line(
+                content, dep_name, section_start, section_end
+            )
+            if removed:
+                with open(file_path, "w") as f:
+                    f.write(new_content)
+                return True
+
+        return False
+
+    except Exception as e:
+        print(f"Error processing {file_path}: {e}")
+        return False
+
+
+def process_unused_dependencies(unused_deps):
+    """
+    Process and remove all unused dependencies from their respective Cargo.toml files.
+
+    Args:
+        unused_deps (dict): Dictionary of unused dependencies organized by package:
+            {
+                "package_name": {
+                    "dependencies": [("dep1", "normal"), ("dev_dep1", "dev")],
+                    "manifest_path": "/path/to/Cargo.toml"
+                }
+            }
+
+    """
+    if not unused_deps:
+        print("No unused dependencies found.")
+        return
+
+    total_removed = 0
+    total_failed = 0
+
+    for package, info in unused_deps.items():
+        deps = info["dependencies"]
+        manifest_path = info["manifest_path"]
+
+        if not os.path.exists(manifest_path):
+            print(f"Manifest file not found: {manifest_path}")
+            total_failed += len(deps)
+            continue
+
+        for dep, dep_type in deps:
+            if remove_dependency_from_toml(manifest_path, dep, dep_type):
+                print(f"Removed {dep} from {package}")
+                total_removed += 1
+            else:
+                print(f"Failed to remove {dep} from {package}")
+                total_failed += 1
+
+    print(f"Removed {total_removed} dependencies")
+    if total_failed > 0:
+        print(f"Failed to remove {total_failed} dependencies")
+
+
+def main():
+    if len(sys.argv) > 1:
+        report_path = sys.argv[1]
+    else:
+        report_path = "udeps-report.json"
+
+    report = load_udeps_report(report_path)
+    if report is None:
+        sys.exit(1)
+
+    unused_deps = extract_unused_dependencies(report)
+    process_unused_dependencies(unused_deps)
+
+
+if __name__ == "__main__":
+    main()

src/cmd/Cargo.toml

@@ -103,3 +103,6 @@ tempfile.workspace = true
 
 [target.'cfg(not(windows))'.dev-dependencies]
 rexpect = "0.5"
+
+[package.metadata.cargo-udeps.ignore]
+development = ["rexpect"]

src/common/function/Cargo.toml

@@ -57,7 +57,6 @@ serde_json.workspace = true
 session.workspace = true
 snafu.workspace = true
 sql.workspace = true
-statrs = "0.16"
 store-api.workspace = true
 table.workspace = true
 uddsketch = { git = "https://github.com/GreptimeTeam/timescaledb-toolkit.git", rev = "84828fe8fb494a6a61412a3da96517fc80f7bb20" }

src/common/macro/Cargo.toml

@@ -21,8 +21,3 @@ syn = { version = "2.0", features = [
 ] }
 
 [dev-dependencies]
-arc-swap = "1.0"
-common-query.workspace = true
-datatypes.workspace = true
-snafu.workspace = true
-static_assertions = "1.1.0"

src/common/query/Cargo.toml

@@ -26,7 +26,6 @@ serde.workspace = true
 snafu.workspace = true
 sqlparser.workspace = true
 sqlparser_derive = "0.1"
-statrs = "0.16"
 store-api.workspace = true
 
 [dev-dependencies]

src/common/runtime/Cargo.toml

@@ -28,7 +28,6 @@ parking_lot.workspace = true
 paste.workspace = true
 pin-project.workspace = true
 prometheus.workspace = true
-rand.workspace = true
 ratelimit.workspace = true
 serde.workspace = true
 serde_json.workspace = true

src/common/sql/Cargo.toml

@@ -6,7 +6,6 @@ license.workspace = true
 
 [dependencies]
 common-base.workspace = true
-common-datasource.workspace = true
 common-decimal.workspace = true
 common-error.workspace = true
 common-macro.workspace = true

src/common/workload/Cargo.toml

@@ -8,6 +8,5 @@ license.workspace = true
 workspace = true
 
 [dependencies]
-api.workspace = true
 common-telemetry.workspace = true
 serde.workspace = true

src/log-store/Cargo.toml

@@ -51,6 +51,5 @@ common-test-util.workspace = true
 common-wal = { workspace = true, features = ["testing"] }
 itertools.workspace = true
 rand.workspace = true
-rand_distr = "0.4"
 rskafka = { workspace = true, features = ["unstable-fuzzing"] }
 uuid.workspace = true

src/meta-srv/Cargo.toml

@@ -59,7 +59,6 @@ humantime-serde.workspace = true
 hyper-util = { workspace = true, features = ["tokio"] }
 itertools.workspace = true
 lazy_static.workspace = true
-log-store.workspace = true
 once_cell.workspace = true
 parking_lot.workspace = true
 prometheus.workspace = true

src/metric-engine/Cargo.toml

@@ -46,3 +46,6 @@ tracing.workspace = true
 common-meta = { workspace = true, features = ["testing"] }
 common-test-util.workspace = true
 mito2 = { workspace = true, features = ["test"] }
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["aquamarine"]

src/mito2/Cargo.toml

@@ -114,3 +114,6 @@ required-features = ["test"]
 name = "simple_bulk_memtable"
 harness = false
 required-features = ["test"]
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["aquamarine"]

src/pipeline/Cargo.toml

@@ -54,7 +54,6 @@ query.workspace = true
 regex.workspace = true
 serde_json.workspace = true
 session.workspace = true
-simd-json.workspace = true
 snafu.workspace = true
 sql.workspace = true
 table.workspace = true

src/query/Cargo.toml

@@ -89,7 +89,6 @@ paste.workspace = true
 pretty_assertions.workspace = true
 rand.workspace = true
 session = { workspace = true, features = ["testing"] }
-statrs = "0.16"
 store-api.workspace = true
 table = { workspace = true, features = ["testing"] }
 tokio-stream.workspace = true

src/servers/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 
 [features]
 default = []
-dashboard = []
+dashboard = ["dep:rust-embed"]
 mem-prof = ["dep:common-mem-prof"]
 pprof = ["dep:common-pprof"]
 testing = []
@@ -35,7 +35,6 @@ catalog.workspace = true
 chrono.workspace = true
 common-base.workspace = true
 common-catalog.workspace = true
-common-config.workspace = true
 common-error.workspace = true
 common-frontend.workspace = true
 common-grpc.workspace = true
@@ -101,7 +100,7 @@ quoted-string = "0.6"
 rand.workspace = true
 regex.workspace = true
 reqwest.workspace = true
-rust-embed = { version = "6.6", features = ["debug-embed"] }
+rust-embed = { version = "6.6", optional = true, features = ["debug-embed"] }
 rustls = { workspace = true, default-features = false, features = ["ring", "logging", "std", "tls12"] }
 rustls-pemfile = "2.0"
 rustls-pki-types = "1.0"

src/store-api/Cargo.toml

@@ -41,3 +41,6 @@ tokio.workspace = true
 async-stream.workspace = true
 common-meta.workspace = true
 serde_json.workspace = true
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["aquamarine"]