mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
* fix: parse handler references once with RunnableRef * chore: update ee-repo-ref to 0c8882d93f3d49cb144e539378d0bc167f2b89c5 This commit updates the EE repository reference after PR #949 was merged in windmill-ee-private. Previous ee-repo-ref: bb2621c9059d51a8a2e5d097a1e7a51a4db2decc New ee-repo-ref: 0c8882d93f3d49cb144e539378d0bc167f2b89c5 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
535 lines
19 KiB
Rust
535 lines
19 KiB
Rust
//! Tests for the variable expiration sweep. No worker: the sweep ends at the push, and the
|
|
//! queued job is what the tests assert on.
|
|
|
|
use sqlx::{Pool, Postgres};
|
|
use windmill_queue::variable_expiration::dispatch_expiring_variables;
|
|
use windmill_test_utils::{initialize_tracing, ApiServer};
|
|
|
|
const WS: &str = "test-workspace";
|
|
|
|
async fn configure_handler(db: &Pool<Postgres>, handler: &str) -> anyhow::Result<()> {
|
|
sqlx::query("UPDATE workspace_settings SET variable_expiration_handler = $2::jsonb WHERE workspace_id = $1")
|
|
.bind(WS)
|
|
.bind(handler)
|
|
.execute(db)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Seed a deployed script the handler setting can point at.
|
|
async fn seed_handler_script(db: &Pool<Postgres>, path: &str) -> anyhow::Result<()> {
|
|
sqlx::query(
|
|
r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, language, lock)
|
|
VALUES ($1, $2, $3, '', '', 'export function main() {}', 'test-user', 'bun', '')"#,
|
|
)
|
|
.bind(WS)
|
|
.bind(path.bytes().fold(0i64, |h, b| h.wrapping_mul(31).wrapping_add(b as i64)))
|
|
.bind(path)
|
|
.execute(db)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn seed_variable(db: &Pool<Postgres>, path: &str, expires_in: &str) -> anyhow::Result<()> {
|
|
sqlx::query(&format!(
|
|
r#"INSERT INTO variable (workspace_id, path, value, is_secret, description, value_valid_until)
|
|
VALUES ($1, $2, 'MUST_NOT_LEAK', true, 'a credential', now() + interval '{expires_in}')"#
|
|
))
|
|
.bind(WS)
|
|
.bind(path)
|
|
.execute(db)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn queued_handler_jobs(db: &Pool<Postgres>, path: &str) -> anyhow::Result<i64> {
|
|
Ok(
|
|
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM v2_job WHERE runnable_path = $1")
|
|
.bind(path)
|
|
.fetch_one(db)
|
|
.await?,
|
|
)
|
|
}
|
|
|
|
/// The sweep dispatches inside the handler's `notify_before_secs`, leaves everything outside it
|
|
/// alone, and does not fire twice for the same expiry.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_dispatch_within_lead_time_once(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
configure_handler(
|
|
&db,
|
|
r#"{"path": "script/u/test-user/expiry_handler", "notify_before_secs": 3600}"#,
|
|
)
|
|
.await?;
|
|
|
|
seed_variable(&db, "u/test-user/due", "30 minutes").await?;
|
|
seed_variable(&db, "u/test-user/far", "2 hours").await?;
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
1,
|
|
"exactly the variable inside the lead window should dispatch"
|
|
);
|
|
|
|
// Admins grant this group access to the secrets it rotates, so the job must run as it.
|
|
let (path, is_secret, permissioned_as, permissioned_as_email): (String, bool, String, String) =
|
|
sqlx::query_as(
|
|
"SELECT args->>'variable_path', (args->>'is_secret')::boolean, permissioned_as,
|
|
permissioned_as_email FROM v2_job WHERE runnable_path = $1",
|
|
)
|
|
.bind("u/test-user/expiry_handler")
|
|
.fetch_one(&db)
|
|
.await?;
|
|
assert_eq!(path, "u/test-user/due");
|
|
assert!(is_secret);
|
|
assert_eq!(permissioned_as, "g/variable_expiration_handler");
|
|
assert_eq!(
|
|
permissioned_as_email,
|
|
"variable_expiration_handler@windmill.dev"
|
|
);
|
|
|
|
let args: String = sqlx::query_scalar("SELECT args::text FROM v2_job WHERE runnable_path = $1")
|
|
.bind("u/test-user/expiry_handler")
|
|
.fetch_one(&db)
|
|
.await?;
|
|
assert!(
|
|
!args.contains("MUST_NOT_LEAK"),
|
|
"handler args leaked the value: {args}"
|
|
);
|
|
|
|
// The claim is what makes concurrent replicas safe.
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
1,
|
|
"a dispatched variable must not fire again"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// A variable's writer must not get the handler to resolve a reference as its own identity;
|
|
/// the admin's `extra_args` still resolve.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_variable_fields_do_not_resolve_in_handler(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
configure_handler(
|
|
&db,
|
|
r#"{"path": "script/u/test-user/expiry_handler", "extra_args": {"slack": "$res:f/slack_bot/bot_token"}}"#,
|
|
)
|
|
.await?;
|
|
seed_variable(&db, "u/test-user/due", "30 minutes").await?;
|
|
sqlx::query(
|
|
"UPDATE variable SET description = '$var:u/test-user/other' WHERE path = 'u/test-user/due'",
|
|
)
|
|
.execute(&db)
|
|
.await?;
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
let (description, slack): (String, String) = sqlx::query_as(
|
|
"SELECT args->>'description', args->>'slack' FROM v2_job WHERE runnable_path = $1",
|
|
)
|
|
.bind("u/test-user/expiry_handler")
|
|
.fetch_one(&db)
|
|
.await?;
|
|
assert!(
|
|
!description.starts_with("$var:"),
|
|
"description still resolves: {description}"
|
|
);
|
|
assert_eq!(slack, "$res:f/slack_bot/bot_token");
|
|
Ok(())
|
|
}
|
|
|
|
/// `remind_every_secs` runs the handler again once the interval has elapsed, and not before.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_remind_every_repeats_the_notice(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
configure_handler(
|
|
&db,
|
|
r#"{"path": "script/u/test-user/expiry_handler", "remind_every_secs": 600}"#,
|
|
)
|
|
.await?;
|
|
seed_variable(&db, "u/test-user/due", "30 minutes").await?;
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
1,
|
|
"no reminder before the interval has elapsed"
|
|
);
|
|
|
|
// A sweep landing a few seconds short of the interval must still remind.
|
|
sqlx::query(
|
|
"UPDATE variable SET expiration_notified_at = now() - interval '9 minutes 50 seconds'",
|
|
)
|
|
.execute(&db)
|
|
.await?;
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
2
|
|
);
|
|
|
|
sqlx::query(
|
|
"UPDATE variable SET value_valid_until = now() - interval '1 minute',
|
|
expiration_dispatched_for = now() - interval '1 minute',
|
|
expiration_notified_at = now() - interval '11 minutes'",
|
|
)
|
|
.execute(&db)
|
|
.await?;
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
2
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// Moving `value_valid_until` re-arms the variable; re-writing the same date does not.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_rearm_on_date_change(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
let server = ApiServer::start(db.clone()).await?;
|
|
let base_url = format!("http://localhost:{}", server.addr.port());
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
configure_handler(&db, r#"{"path": "script/u/test-user/expiry_handler"}"#).await?;
|
|
seed_variable(&db, "u/test-user/due", "30 minutes").await?;
|
|
|
|
// Through the real endpoint: writing the column directly would assert nothing.
|
|
let update = async |body: serde_json::Value| -> anyhow::Result<u16> {
|
|
Ok(reqwest::Client::new()
|
|
.post(format!(
|
|
"{base_url}/api/w/{WS}/variables/update/u/test-user/due"
|
|
))
|
|
.bearer_auth("SECRET_TOKEN")
|
|
.json(&body)
|
|
.send()
|
|
.await?
|
|
.status()
|
|
.as_u16())
|
|
};
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
1
|
|
);
|
|
|
|
let stored: chrono::DateTime<chrono::Utc> = sqlx::query_scalar(
|
|
"SELECT value_valid_until FROM variable WHERE workspace_id = $1 AND path = $2",
|
|
)
|
|
.bind(WS)
|
|
.bind("u/test-user/due")
|
|
.fetch_one(&db)
|
|
.await?;
|
|
|
|
// An unchanged `wmill sync push` rewrites the same date.
|
|
assert_eq!(
|
|
update(serde_json::json!({ "value_valid_until": stored })).await?,
|
|
200
|
|
);
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
1,
|
|
"an unchanged expiry must not re-arm"
|
|
);
|
|
|
|
assert_eq!(
|
|
update(serde_json::json!({ "description": "touched" })).await?,
|
|
200
|
|
);
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
1,
|
|
"an update that omits value_valid_until must not re-arm"
|
|
);
|
|
|
|
assert_eq!(
|
|
update(serde_json::json!({ "value_valid_until": stored + chrono::Duration::minutes(5) }))
|
|
.await?,
|
|
200
|
|
);
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
2,
|
|
"a moved expiry must re-arm the handler"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// `muted_on_user_path` keeps personal-path variables out.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_muted_user_paths(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
seed_variable(&db, "u/test-user/personal", "30 minutes").await?;
|
|
seed_variable(&db, "f/test_folder/shared", "30 minutes").await?;
|
|
configure_handler(
|
|
&db,
|
|
r#"{"path": "script/u/test-user/expiry_handler", "muted_on_user_path": true}"#,
|
|
)
|
|
.await?;
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
let dispatched: Vec<String> =
|
|
sqlx::query_scalar("SELECT args->>'variable_path' FROM v2_job WHERE runnable_path = $1")
|
|
.bind("u/test-user/expiry_handler")
|
|
.fetch_all(&db)
|
|
.await?;
|
|
assert_eq!(
|
|
dispatched,
|
|
vec!["f/test_folder/shared".to_string()],
|
|
"a muted personal path must not dispatch"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Broken workspaces' due rows must not hold back anyone else's first notice or reminder: every
|
|
/// due row is tried each pass, and a broken handler only once per pass. The broken rows go in
|
|
/// first, so a cap on the pass would cut off the working workspace.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_broken_handlers_do_not_hold_back_others(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
sqlx::raw_sql(
|
|
r#"INSERT INTO workspace (id, name, owner) SELECT 'broken-' || i, 'broken-' || i, 'test-user' FROM generate_series(1, 100) i;
|
|
INSERT INTO workspace_key (workspace_id, kind, key) SELECT 'broken-' || i, 'cloud', 'k' FROM generate_series(1, 100) i;
|
|
INSERT INTO workspace_settings (workspace_id, variable_expiration_handler)
|
|
SELECT 'broken-' || i, '{"path": "script/u/test-user/does_not_exist"}'::jsonb FROM generate_series(1, 100) i;
|
|
INSERT INTO variable (workspace_id, path, value, is_secret, description, value_valid_until)
|
|
SELECT 'broken-' || i, 'f/creds/v' || j, 'v', false, '', now() - interval '1 day'
|
|
FROM generate_series(1, 100) i, generate_series(1, 2) j;"#,
|
|
)
|
|
.execute(&db)
|
|
.await?;
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
configure_handler(
|
|
&db,
|
|
r#"{"path": "script/u/test-user/expiry_handler", "remind_every_secs": 60}"#,
|
|
)
|
|
.await?;
|
|
seed_variable(&db, "u/test-user/fresh", "30 minutes").await?;
|
|
sqlx::query(
|
|
"INSERT INTO variable (workspace_id, path, value, is_secret, description, value_valid_until,
|
|
expiration_dispatched_for, expiration_notified_at)
|
|
VALUES ($1, 'u/test-user/reminded', 'v', false, '', now() + interval '10 minutes',
|
|
now() + interval '10 minutes', now() - interval '2 minutes')",
|
|
)
|
|
.bind(WS)
|
|
.execute(&db)
|
|
.await?;
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/expiry_handler").await?,
|
|
2,
|
|
"the first notice and the reminder must both go out"
|
|
);
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/does_not_exist").await?,
|
|
0
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// A broken handler alerts once, however many passes it fails, runs on the first pass after
|
|
/// it is deployed, and that resolves its alert.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_deploying_the_handler_ends_the_failure(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
configure_handler(&db, r#"{"path": "script/u/test-user/redeployed_handler"}"#).await?;
|
|
seed_variable(&db, "u/test-user/due", "30 minutes").await?;
|
|
let alerts = || async {
|
|
Ok::<_, anyhow::Error>(sqlx::query_as::<_, (i64, i64)>(
|
|
"SELECT count(*), count(*) FILTER (WHERE NOT acknowledged) FROM alerts
|
|
WHERE alert_type = 'critical_error' AND resource = 'variable_expiration_handler:test-workspace'",
|
|
)
|
|
.fetch_one(&db)
|
|
.await?)
|
|
};
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
alerts().await?,
|
|
(1, 1),
|
|
"one alert per failure, not per pass"
|
|
);
|
|
|
|
seed_handler_script(&db, "u/test-user/redeployed_handler").await?;
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/redeployed_handler").await?,
|
|
1,
|
|
"the deployed handler did not run on the next pass"
|
|
);
|
|
assert_eq!(alerts().await?, (1, 0), "starting again resolves the alert");
|
|
Ok(())
|
|
}
|
|
|
|
/// A handler script still waiting on its dependency job runs on the first pass after its lock
|
|
/// lands, though nothing about the script changes but the lock.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_handler_runs_once_its_lock_lands(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
configure_handler(
|
|
&db,
|
|
r#"{"path": "script/u/test-user/pending_lock_handler"}"#,
|
|
)
|
|
.await?;
|
|
seed_variable(&db, "u/test-user/due", "30 minutes").await?;
|
|
seed_handler_script(&db, "u/test-user/pending_lock_handler").await?;
|
|
sqlx::query("UPDATE script SET lock = NULL WHERE workspace_id = $1")
|
|
.bind(WS)
|
|
.execute(&db)
|
|
.await?;
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/pending_lock_handler").await?,
|
|
0
|
|
);
|
|
sqlx::query("UPDATE script SET lock = '' WHERE workspace_id = $1")
|
|
.bind(WS)
|
|
.execute(&db)
|
|
.await?;
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
assert_eq!(
|
|
queued_handler_jobs(&db, "u/test-user/pending_lock_handler").await?,
|
|
1,
|
|
"the handler did not run once its lock landed"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// A removed handler has nothing left to retry, so the next pass ends its failure and resolves
|
|
/// its alert.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_removing_a_failing_handler_resolves_its_alert(
|
|
db: Pool<Postgres>,
|
|
) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
sqlx::query(
|
|
"UPDATE workspace_settings SET variable_expiration_handler = NULL,
|
|
variable_expiration_failed_at = now() WHERE workspace_id = $1",
|
|
)
|
|
.bind(WS)
|
|
.execute(&db)
|
|
.await?;
|
|
sqlx::query(
|
|
"INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)
|
|
VALUES ('critical_error', 'failed', false, false, $1, 'variable_expiration_handler:' || $1)",
|
|
)
|
|
.bind(WS)
|
|
.execute(&db)
|
|
.await?;
|
|
|
|
dispatch_expiring_variables(&db).await;
|
|
|
|
let (failing, open_alerts): (bool, i64) = sqlx::query_as(
|
|
"SELECT ws.variable_expiration_failed_at IS NOT NULL,
|
|
(SELECT count(*) FROM alerts WHERE alert_type = 'critical_error' AND NOT acknowledged
|
|
AND workspace_id = $1)
|
|
FROM workspace_settings ws WHERE ws.workspace_id = $1",
|
|
)
|
|
.bind(WS)
|
|
.fetch_one(&db)
|
|
.await?;
|
|
assert!(!failing, "the removed handler's failure must end");
|
|
assert_eq!(
|
|
open_alerts, 0,
|
|
"the removed handler's alert must be resolved"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// The handler endpoint: admin-only, existence-checked, stored in its clean form, and clearing
|
|
/// by omission, which `wmill sync push` relies on to propagate an absent handler.
|
|
#[sqlx::test(fixtures("base"))]
|
|
async fn test_edit_handler_endpoint(db: Pool<Postgres>) -> anyhow::Result<()> {
|
|
initialize_tracing().await;
|
|
let server = ApiServer::start(db.clone()).await?;
|
|
let base_url = format!("http://localhost:{}", server.addr.port());
|
|
|
|
let edit = async |token: &str, body: serde_json::Value| -> anyhow::Result<u16> {
|
|
Ok(reqwest::Client::new()
|
|
.post(format!(
|
|
"{base_url}/api/w/{WS}/workspaces/edit_variable_expiration_handler"
|
|
))
|
|
.bearer_auth(token)
|
|
.json(&body)
|
|
.send()
|
|
.await?
|
|
.status()
|
|
.as_u16())
|
|
};
|
|
let stored = async || -> anyhow::Result<Option<serde_json::Value>> {
|
|
Ok(sqlx::query_scalar(
|
|
"SELECT variable_expiration_handler FROM workspace_settings WHERE workspace_id = $1",
|
|
)
|
|
.bind(WS)
|
|
.fetch_one(&db)
|
|
.await?)
|
|
};
|
|
|
|
seed_handler_script(&db, "u/test-user/expiry_handler").await?;
|
|
let handler = serde_json::json!({ "path": "script//u/test-user/expiry_handler" });
|
|
assert_eq!(edit("SECRET_TOKEN_2", handler.clone()).await?, 403);
|
|
assert!(stored().await?.is_none());
|
|
|
|
for path in [
|
|
"u/test-user/expiry_handler",
|
|
"script/u/test-user/does_not_exist",
|
|
] {
|
|
assert_eq!(
|
|
edit("SECRET_TOKEN", serde_json::json!({ "path": path })).await?,
|
|
400,
|
|
"{path}"
|
|
);
|
|
}
|
|
|
|
// A recorded failure is the sweep's to end, so the endpoint leaves it alone either way.
|
|
sqlx::query("UPDATE workspace_settings SET variable_expiration_failed_at = now() WHERE workspace_id = $1")
|
|
.bind(WS)
|
|
.execute(&db)
|
|
.await?;
|
|
let failing = || async {
|
|
Ok::<_, anyhow::Error>(sqlx::query_scalar::<_, bool>(
|
|
"SELECT variable_expiration_failed_at IS NOT NULL FROM workspace_settings WHERE workspace_id = $1",
|
|
)
|
|
.bind(WS)
|
|
.fetch_one(&db)
|
|
.await?)
|
|
};
|
|
assert_eq!(edit("SECRET_TOKEN", handler).await?, 200);
|
|
assert_eq!(
|
|
stored()
|
|
.await?
|
|
.and_then(|h| h["path"].as_str().map(String::from)),
|
|
Some("script/u/test-user/expiry_handler".to_string())
|
|
);
|
|
assert!(failing().await?, "saving the handler must keep its failure");
|
|
|
|
assert_eq!(edit("SECRET_TOKEN", serde_json::json!({})).await?, 200);
|
|
assert!(
|
|
stored().await?.is_none(),
|
|
"an omitted path clears the handler"
|
|
);
|
|
|
|
Ok(())
|
|
}
|