mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-06 08:09:24 +00:00
feat: add js for claim parser (#9873)
### **User description** Design at : security/sso-claim-parser/configuration.md ___ ### **PR Type** Enhancement, Tests ___ ### **Description** - Introduce rquickjs-based JS runtime for functions - Add compile_js_function and apply_js_fn utilities - Extend pipeline to support JS via CompiledFunctionRuntime - Update web UI: select JS/VRL transforms, show JS errors ___ ### Diagram Walkthrough ```mermaid flowchart LR A["FunctionNode"] -->|JS| B["compile_js_function"] A -->|VRL| C["compile_vrl_function"] B --> D["apply_js_fn"] C --> E["apply_vrl_fn"] D --> F["send to children"] E --> F ``` <details> <summary><h3> File Walkthrough</h3></summary> <table><thead><tr><th></th><th align="left">Relevant files</th></tr></thead><tbody><tr><td><strong>Enhancement</strong></td><td><details><summary>13 files</summary><table> <tr> <td><strong>function.ts</strong><dd><code>Add optional `trans_type` to TestFunctionPayload</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-c6766f6c671adce05f9eef5d4a57a32371fa7da35628ef5c142fa503d191cb16">+16/-0</a> </td> </tr> <tr> <td><strong>js.rs</strong><dd><code>Introduce JS runtime, compile and apply functions</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-88e490bed5fa6c9c8e0aa2738b95c5284bb97899298f54b9cd587713e3ef0109">+756/-0</a> </td> </tr> <tr> <td><strong>batch_execution.rs</strong><dd><code>Extend pipeline execution to handle JS functions</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-3f6824a6eddaf1603e5cb0f1a4311ac6a45e1ffb967266ccc1f0bba944625e69">+465/-96</a></td> </tr> <tr> <td><strong>functions.rs</strong><dd><code>Validate and test-run JS transforms, restrict to _meta</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-ed5611a9aa6b5c033e70d7631ba72fafad3cc842672077a7fcf965d11a4fef23">+319/-35</a></td> </tr> <tr> <td><strong>mod.rs</strong><dd><code>Forbid JavaScript functions in pipelines globally</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-c186c796a7bdfd9d1301b1429df11456430f1d0d4758783810b9e2e44384e3b2">+39/-1</a> </td> </tr> <tr> <td><strong>jwt.rs</strong><dd><code>Add JS claim parser execution function</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-232cab74f4cbf135af424744272e17732fbda8ac005d33c0acb4597e65479d02">+25/-2</a> </td> </tr> <tr> <td><strong>mod.rs</strong><dd><code>Wrap JS compile/apply in ingestion API</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-7c7b5bca85f0ced652965cae66540c96f56dfcca4f4c16b77bf7b45aac203d1b">+24/-1</a> </td> </tr> <tr> <td><strong>mod.rs</strong><dd><code>Include `trans_type` in test_function API payload</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-a5e1b3644cef9feb689dfe66382e44e5434ecadbff3469127346a333d4733102">+8/-3</a> </td> </tr> <tr> <td><strong>AddFunction.vue</strong><dd><code>UI: add JS/VRL transform type selector</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-ebfd6c538f3cd3ecb9ab17550fd76e2e89e7a1abccc83a7505077e4af3eee110">+42/-25</a> </td> </tr> <tr> <td><strong>FunctionsToolbar.vue</strong><dd><code>Toolbar: transform type radio buttons</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-8283d2ff9c4a66b4b59eac306672aca2d54655a462e2ddb88e14e039d79dcdaf">+51/-3</a> </td> </tr> <tr> <td><strong>DomainManagement.vue</strong><dd><code>Track claim-parser function changes for save button</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-5e47cd01fcb63f7ef27dc64d2ecf66c39e17097e29944951c3ae1bfe715bb97c">+17/-9</a> </td> </tr> <tr> <td><strong>TestFunction.vue</strong><dd><code>Support JS error messaging in test UI</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-b9af9a2d1bf04d23422c26a13e6a763c6953c8713357a2f64ed8073ce6c926a3">+15/-5</a> </td> </tr> <tr> <td><strong>PipelineEditor.vue</strong><dd><code>Filter out JS functions in pipeline editor UI</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-68c21f647336067eb79737ce9aca2d49632f2fa1c0878bcfdae55607e3473a15">+12/-4</a> </td> </tr> </table></details></td></tr><tr><td><strong>Configuration changes</strong></td><td><details><summary>2 files</summary><table> <tr> <td><strong>function.rs</strong><dd><code>Extend `Transform` model for JS support</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-27c9f3d93723397a0e234bc45c9f543b43eff5fd518d76e735f78d95a58f3f1c">+16/-6</a> </td> </tr> <tr> <td><strong>mod.rs</strong><dd><code>Register new `js` utils module</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-2aa59ce4464e7dbabdb9367fc8bb8c1b489c914b8690577d4916870463ab12db">+1/-0</a> </td> </tr> </table></details></td></tr><tr><td><strong>Bug fix</strong></td><td><details><summary>1 files</summary><table> <tr> <td><strong>errors.rs</strong><dd><code>Rename `MemoryTableOverflowError` display message</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-bf45932466265b732f13d0cd9b9cffbaefbe37c5861ace9315449eeca9250713">+1/-1</a> </td> </tr> </table></details></td></tr><tr><td><strong>Tests</strong></td><td><details><summary>3 files</summary><table> <tr> <td><strong>result-array-functions.spec.js</strong><dd><code>Add E2E tests for `#ResultArray#` functions in UI</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-e9911625ad2bab76f2b760f0ad254791331993a6a874b96f6a11fc6d88038b10">+470/-0</a> </td> </tr> <tr> <td><strong>cleanup.spec.js</strong><dd><code>Clean up JS function test data in UI tests</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-c214275a512fa285a7e4f2ab61dfcb42d71a0436b0e4241f94d7b45693587983">+31/-0</a> </td> </tr> <tr> <td><strong>page-manager.js</strong><dd><code>Register `FunctionsPage` in `PageManager`</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-565eab8b3c767ae50c9d05e591621d14f66b81af84d6b5ea6efe136c1c3933df">+6/-0</a> </td> </tr> </table></details></td></tr><tr><td><strong>Dependencies</strong></td><td><details><summary>1 files</summary><table> <tr> <td><strong>Cargo.toml</strong><dd><code>Add `rquickjs` dependency for JS runtime</code> </dd></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-2e9d962a08321605940b5a657135052fbcef87b5e360662bb527c96d9a615542">+2/-0</a> </td> </tr> </table></details></td></tr><tr><td><strong>Additional files</strong></td><td><details><summary>18 files</summary><table> <tr> <td><strong>test_functions_meta_org_restriction.py</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-99ef1207d6ae6d7eeaed37d70c8cf0c4770ba78f6f51fbdc067d35492d2fe9ea">+401/-0</a> </td> </tr> <tr> <td><strong>test_js_enrichment.py</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-c5547d46bec60b888d947d1e9b7c5445aee72cf8e837182f7138e0fc903eedbb">+356/-0</a> </td> </tr> <tr> <td><strong>test_js_result_array.py</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-9ed1363a1f58c94e018bb2b0aaa9d08fa71f89e7e55a6ff6be4a82bbeed8985c">+390/-0</a> </td> </tr> <tr> <td><strong>functionsPage.js</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-7bab9cc2eb8f36d3ee5d0aab453c353ec6d6dfecbc401cf401322ff57faa8440">+326/-0</a> </td> </tr> <tr> <td><strong>javascript-functions.spec.js</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-bdae6529346a7222bdec19f75f490cb9d0f3578a260ba835225efcd27db3f64e">+314/-0</a> </td> </tr> <tr> <td><strong>meta-org-js-restriction.spec.js</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-16cc2373f6f1955f8344a59d5f2edcd6ce121c4fdee1d432b1f0ec8e27f25881">+483/-0</a> </td> </tr> <tr> <td><strong>FunctionList.vue</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-6ff9af50a1597504b7f0dca91e61cf30c800b4a17b4be2ed2c3a4f7a7b2d7068">+1/-0</a> </td> </tr> <tr> <td><strong>de.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-78f24ecb2dbcfc38fdf3ec22b2559a5b40d236ea689d1c55f5d3e87168310264">+30/-18</a> </td> </tr> <tr> <td><strong>en.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-892aa3d75ab6e129ef9116aa21bec1b9a15ba69ce5366d2a19128078c690d824">+15/-7</a> </td> </tr> <tr> <td><strong>es.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-ac2a2167b8b973e3f776f9fe3b43bef4576470d8ad33f3d7cfaff6edff0e5778">+30/-18</a> </td> </tr> <tr> <td><strong>fr.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-db80e762b88ddf183820c050cdc6b215e6b02b365b5e73f5152af46b0d26f1a2">+30/-18</a> </td> </tr> <tr> <td><strong>it.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-2de78074a5f887afddce60d694ead9322e6ab223eb70251c13819f3028bdc70d">+30/-18</a> </td> </tr> <tr> <td><strong>ja.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-092e49c5f9fbc1b539ab9c2866d108d0ce1d314022b4c294eb4611b285e7bbd3">+30/-18</a> </td> </tr> <tr> <td><strong>ko.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-50e27ca1b4825a5026f3a251bfcf79c97cab07cfade2ebd512d332453d707c48">+30/-18</a> </td> </tr> <tr> <td><strong>nl.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-a1706021687ae5d8a1c006c49eab3946174fecd895f9b70a89c2e76d97250906">+30/-18</a> </td> </tr> <tr> <td><strong>pt.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-38e804f64cb5e0e451849fc84e9b46e53c954a4a4263d51c18665ad45f4b4815">+30/-18</a> </td> </tr> <tr> <td><strong>tr.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-6f0f131e9e2591480c4c5d1eaffc72d02dc31cbd8d3f90d8f012f08cabac8818">+30/-18</a> </td> </tr> <tr> <td><strong>zh.json</strong></td> <td><a href="https://github.com/openobserve/openobserve/pull/9873/files#diff-8818055db58f83230aeea4ebfd4cec31d924e8c47b6f91298e356959ab25339e">+254/-242</a></td> </tr> </table></details></td></tr></tr></tbody></table> </details> ___ --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Ashish Kolhe
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
0f4dcbe971
commit
ce09cd0a4d
@@ -979,7 +979,7 @@ async fn process_custom_claim_parsing(
|
||||
let function_name = function_name.to_string();
|
||||
Box::pin(async move {
|
||||
match db::functions::get("_meta", &function_name).await {
|
||||
Ok(func) => Ok(func.function),
|
||||
Ok(func) => Ok(func), // Return full Transform object with trans_type
|
||||
Err(e) => Err(anyhow::anyhow!(
|
||||
"Failed to load function '{}' from _meta org: {}",
|
||||
function_name,
|
||||
@@ -988,7 +988,11 @@ async fn process_custom_claim_parsing(
|
||||
}
|
||||
})
|
||||
as std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = Result<String, anyhow::Error>> + Send>,
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<config::meta::function::Transform, anyhow::Error>,
|
||||
> + Send,
|
||||
>,
|
||||
>
|
||||
};
|
||||
|
||||
@@ -1034,6 +1038,22 @@ async fn process_custom_claim_parsing(
|
||||
}
|
||||
};
|
||||
|
||||
let js_execute_fn = |function_content: &str, claims: Value| -> Result<Value, anyhow::Error> {
|
||||
// Compile the JavaScript function
|
||||
let js_config = crate::service::ingestion::compile_js_function(function_content, "_meta")
|
||||
.map_err(|e| anyhow::anyhow!("JavaScript compilation failed: {}", e))?;
|
||||
|
||||
// Execute JavaScript with claims as input
|
||||
let (result, error) =
|
||||
crate::service::ingestion::apply_js_fn(&js_config, claims, "_meta", &[String::new()]);
|
||||
|
||||
if let Some(err) = error {
|
||||
return Err(anyhow::anyhow!("JavaScript execution failed: {}", err));
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
};
|
||||
|
||||
let error_publish_fn =
|
||||
|error: o2_enterprise::enterprise::auth::claim_parser::ClaimParserError| {
|
||||
Box::pin(async move {
|
||||
@@ -1070,6 +1090,7 @@ async fn process_custom_claim_parsing(
|
||||
function_loader_fn,
|
||||
vrl_compile_fn,
|
||||
vrl_execute_fn,
|
||||
js_execute_fn,
|
||||
error_publish_fn,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -351,10 +351,15 @@ pub async fn test_function(
|
||||
req_body: web::Json<TestVRLRequest>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let org_id = path.into_inner();
|
||||
let TestVRLRequest { function, events } = req_body.into_inner();
|
||||
let TestVRLRequest {
|
||||
function,
|
||||
events,
|
||||
trans_type,
|
||||
} = req_body.into_inner();
|
||||
|
||||
// Assuming `test_function` applies the VRL function to each event
|
||||
match crate::service::functions::test_run_function(&org_id, function, events).await {
|
||||
// test_run_function will auto-detect VRL vs JS if trans_type is None
|
||||
match crate::service::functions::test_run_function(&org_id, function, events, trans_type).await
|
||||
{
|
||||
Ok(result) => Ok(result),
|
||||
Err(err) => Ok(HttpResponse::BadRequest().body(err.to_string())),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user