feat: add GcsConfig credential field (#4568)

.env.example

@@ -14,10 +14,11 @@ GT_AZBLOB_CONTAINER=AZBLOB container
 GT_AZBLOB_ACCOUNT_NAME=AZBLOB account name
 GT_AZBLOB_ACCOUNT_KEY=AZBLOB account key
 GT_AZBLOB_ENDPOINT=AZBLOB endpoint
-# Settings for gcs test 
+# Settings for gcs test
-GT_GCS_BUCKET = GCS bucket 
+GT_GCS_BUCKET = GCS bucket
 GT_GCS_SCOPE  = GCS scope
-GT_GCS_CREDENTIAL_PATH = GCS credential path 
+GT_GCS_CREDENTIAL_PATH = GCS credential path
+GT_GCS_CREDENTIAL = GCS credential
 GT_GCS_ENDPOINT = GCS end point
 # Settings for kafka wal test
 GT_KAFKA_ENDPOINTS = localhost:9092

config/config.md

@@ -98,6 +98,7 @@
 | `storage.account_key` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.scope` | String | `None` | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.credential_path` | String | `None` | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential` | String | `None` | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.container` | String | `None` | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.sas_token` | String | `None` | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.endpoint` | String | `None` | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
@@ -387,6 +388,7 @@
 | `storage.account_key` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.scope` | String | `None` | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.credential_path` | String | `None` | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential` | String | `None` | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.container` | String | `None` | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.sas_token` | String | `None` | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.endpoint` | String | `None` | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |

config/datanode.example.toml

@@ -241,6 +241,7 @@ backoff_deadline = "5mins"
 # root = "data"
 # scope = "test"
 # credential_path = "123456"
+# credential = "base64-credential"
 # endpoint = "https://storage.googleapis.com"
 
 ## The data storage options.
@@ -312,6 +313,11 @@ scope = "test"
 ## +toml2docs:none-default
 credential_path = "test"
 
+## The credential of the google cloud storage.
+## **It's only used when the storage type is `Gcs`**.
+## +toml2docs:none-default
+credential= "base64-credential"
+
 ## The container of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
 ## +toml2docs:none-default

config/standalone.example.toml

@@ -286,6 +286,7 @@ retry_delay = "500ms"
 # root = "data"
 # scope = "test"
 # credential_path = "123456"
+# credential = "base64-credential"
 # endpoint = "https://storage.googleapis.com"
 
 ## The data storage options.
@@ -357,6 +358,11 @@ scope = "test"
 ## +toml2docs:none-default
 credential_path = "test"
 
+## The credential of the google cloud storage.
+## **It's only used when the storage type is `Gcs`**.
+## +toml2docs:none-default
+credential = "base64-credential"
+
 ## The container of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
 ## +toml2docs:none-default

src/datanode/src/config.rs

@@ -179,6 +179,8 @@ pub struct GcsConfig {
     pub scope: String,
     #[serde(skip_serializing)]
     pub credential_path: SecretString,
+    #[serde(skip_serializing)]
+    pub credential: SecretString,
     pub endpoint: String,
     #[serde(flatten)]
     pub cache: ObjectStorageCacheConfig,
@@ -190,6 +192,7 @@ impl PartialEq for GcsConfig {
             && self.bucket == other.bucket
             && self.scope == other.scope
             && self.credential_path.expose_secret() == other.credential_path.expose_secret()
+            && self.credential.expose_secret() == other.credential.expose_secret()
             && self.endpoint == other.endpoint
             && self.cache == other.cache
     }
@@ -243,6 +246,7 @@ impl Default for GcsConfig {
             bucket: String::default(),
             scope: String::default(),
             credential_path: SecretString::from(String::default()),
+            credential: SecretString::from(String::default()),
             endpoint: String::default(),
             cache: ObjectStorageCacheConfig::default(),
         }

src/datanode/src/store/gcs.rs

@@ -34,6 +34,7 @@ pub(crate) async fn new_gcs_object_store(gcs_config: &GcsConfig) -> Result<Objec
         .bucket(&gcs_config.bucket)
         .scope(&gcs_config.scope)
         .credential_path(gcs_config.credential_path.expose_secret())
+        .credential(gcs_config.credential.expose_secret())
         .endpoint(&gcs_config.endpoint)
         .http_client(build_http_client()?);
 

src/object-store/tests/object_store_test.rs

@@ -200,6 +200,7 @@ async fn test_gcs_backend() -> Result<()> {
                 .bucket(&env::var("GT_GCS_BUCKET").unwrap())
                 .scope(&env::var("GT_GCS_SCOPE").unwrap())
                 .credential_path(&env::var("GT_GCS_CREDENTIAL_PATH").unwrap())
+                .credential(&env::var("GT_GCS_CREDENTIAL").unwrap())
                 .endpoint(&env::var("GT_GCS_ENDPOINT").unwrap());
 
             let store = ObjectStore::new(builder).unwrap().finish();

tests-integration/src/test_util.rs

@@ -160,6 +160,7 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te
                 bucket: env::var("GT_GCS_BUCKET").unwrap(),
                 scope: env::var("GT_GCS_SCOPE").unwrap(),
                 credential_path: env::var("GT_GCS_CREDENTIAL_PATH").unwrap().into(),
+                credential: env::var("GT_GCS_CREDENTIAL").unwrap().into(),
                 endpoint: env::var("GT_GCS_ENDPOINT").unwrap(),
                 ..Default::default()
             };
@@ -169,6 +170,7 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te
                 .bucket(&gcs_config.bucket)
                 .scope(&gcs_config.scope)
                 .credential_path(gcs_config.credential_path.expose_secret())
+                .credential(gcs_config.credential.expose_secret())
                 .endpoint(&gcs_config.endpoint);
 
             let config = ObjectStoreConfig::Gcs(gcs_config);