Annotations refactor (#4512)

* Refactor annotation system

* Move tests to different location

* Limit annotations per line (15)

* Write initial benchmark

* Optimize

* More optimizations

* Rewrite with 3x speed!

* Remove benchmarks

They were in wrong place and for development

* Polishing

* Remove unused leftover

* Integrate new annotation system

* Add comment explanation

* Remove `hashe-based matching`

Turned to be it is impure and non-deterministic method

* Add `tracing::error` for `// native` without `deno_core`

If you build backend without `deno_core` feature flag, and add //native
annotation to ts script, it would just silently exit.

This commit prints error to logs

* add more tests

* Update annotations.rs

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "Inflector"
@@ -10555,6 +10555,7 @@ dependencies = [
  "tracing-loki",
  "tracing-subscriber",
  "uuid 1.11.0",
+ "windmill-macros",
 ]
 
 [[package]]
@@ -10594,6 +10595,18 @@ dependencies = [
  "windmill-common",
 ]
 
+[[package]]
+name = "windmill-macros"
+version = "1.405.5"
+dependencies = [
+ "itertools 0.10.5",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "syn 2.0.79",
+]
+
 [[package]]
 name = "windmill-parser"
 version = "1.409.2"

backend/Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "./windmill-audit",
     "./windmill-git-sync",
     "./windmill-indexer",
+    "./windmill-macros",
     "./parsers/windmill-parser",
     "./parsers/windmill-parser-ts",
     "./parsers/windmill-parser-wasm",
@@ -23,7 +24,7 @@ members = [
     "./parsers/windmill-parser-py",
     "./parsers/windmill-parser-py-imports",
     "./parsers/windmill-sql-datatype-parser-wasm",
-    "./parsers/windmill-parser-yaml",
+    "./parsers/windmill-parser-yaml", "windmill-macros",
 ]
 
 [workspace.package]
@@ -116,6 +117,7 @@ windmill-common = { path = "./windmill-common", default-features = false }
 windmill-audit = { path = "./windmill-audit" }
 windmill-git-sync = { path = "./windmill-git-sync" }
 windmill-indexer = {path = "./windmill-indexer"}
+windmill-macros = {path = "./windmill-macros"}
 windmill-parser = { path = "./parsers/windmill-parser" }
 windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
 windmill-parser-py = { path = "./parsers/windmill-parser-py" }
@@ -281,6 +283,8 @@ triomphe = "^0"
 
 tantivy = "0.22.0"
 
+# Macro-related
+proc-macro2 = "1.0"
 pulldown-cmark = "0.9"
 toml = "0.7"
 syn = { version = "2.0.74", features = ["full"] }

backend/windmill-api/src/jobs.rs

@@ -4193,10 +4193,10 @@ async fn run_dependencies_job(
             JsonRawValue::from_string("true".to_string()).unwrap(),
         );
         if language == ScriptLang::Bun {
-            let annotation = windmill_common::worker::get_annotation_ts(&raw_code);
+            let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code);
             hm.insert(
                 "npm_mode".to_string(),
-                JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(),
+                JsonRawValue::from_string(annotation.npm.to_string()).unwrap(),
             );
         }
         (PushArgs { extra: Some(hm), args: &ehm }, deps)

backend/windmill-api/src/scripts.rs

@@ -56,7 +56,7 @@ use windmill_common::{
     utils::{
         not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
-    worker::{get_annotation_ts, to_raw_value},
+    worker::to_raw_value,
     HUB_BASE_URL,
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -606,8 +606,8 @@ async fn create_script_internal<'c>(
     };
 
     let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative {
-        let anns = get_annotation_ts(&ns.content);
-        if anns.native_mode {
+        let anns = windmill_common::worker::TypeScriptAnnotations::parse(&ns.content);
+        if anns.native {
             ScriptLang::Bunnative
         } else {
             ScriptLang::Bun

backend/windmill-common/Cargo.toml

@@ -57,6 +57,7 @@ futures-core.workspace = true
 async-stream.workspace = true
 const_format.workspace = true
 crc.workspace = true
+windmill-macros.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemalloc-ctl = { optional = true, workspace = true }

backend/windmill-common/src/worker.rs

@@ -13,6 +13,7 @@ use std::{
     sync::{atomic::AtomicBool, Arc},
 };
 use tokio::sync::RwLock;
+use windmill_macros::annotations;
 
 use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB};
 
@@ -303,64 +304,25 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
         .flatten()
 }
 
+#[annotations("#")]
+pub struct PythonAnnotations {
+    pub no_cache: bool,
+    pub no_uv: bool,
+}
+
+#[annotations("//")]
 pub struct TypeScriptAnnotations {
-    pub npm_mode: bool,
-    pub nodejs_mode: bool,
-    pub native_mode: bool,
+    pub npm: bool,
+    pub nodejs: bool,
+    pub native: bool,
     pub nobundling: bool,
 }
 
-pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations {
-    let annotations = inner_content
-        .lines()
-        .take_while(|x| x.starts_with("//"))
-        .map(|x| x.to_string().replace("//", "").trim().to_string())
-        .collect_vec();
-    let nodejs_mode: bool = annotations.contains(&"nodejs".to_string());
-    let npm_mode: bool = annotations.contains(&"npm".to_string());
-    let native_mode: bool = annotations.contains(&"native".to_string());
-
-    //TODO: remove || npm_mode when bun build is more powerful
-    let nobundling: bool =
-        annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING;
-
-    TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling }
-}
-
-pub struct PythonAnnotations {
-    pub no_uv: bool,
-    pub no_cache: bool,
-}
-
-pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations {
-    let annotations = inner_content
-        .lines()
-        .take_while(|x| x.starts_with("#"))
-        .map(|x| x.to_string().replace("#", "").trim().to_string())
-        .collect_vec();
-
-    let no_uv: bool = annotations.contains(&"no_uv".to_string());
-    let no_cache: bool = annotations.contains(&"no_cache".to_string());
-
-    PythonAnnotations { no_uv, no_cache }
-}
-
+#[annotations("--")]
 pub struct SqlAnnotations {
     pub return_last_result: bool,
 }
 
-pub fn get_sql_annotations(inner_content: &str) -> SqlAnnotations {
-    let annotations = inner_content
-        .lines()
-        .take_while(|x| x.starts_with("--"))
-        .map(|x| x.to_string().replace("--", "").trim().to_string())
-        .collect_vec();
-
-    let return_last_result: bool = annotations.contains(&"return_last_result".to_string());
-
-    SqlAnnotations { return_last_result }
-}
-
 pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
     if tokio::fs::metadata(&bin_path).await.is_ok() {
         (true, format!("loaded from local cache: {}\n", bin_path))

backend/windmill-macros/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "windmill-macros"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2.workspace = true
+quote.workspace = true
+syn.workspace = true
+
+# Dependencies for tests
+[dev-dependencies]
+# tests/annotation.rs
+lazy_static.workspace = true
+itertools.workspace = true
+regex.workspace = true

backend/windmill-macros/src/lib.rs

@@ -0,0 +1,95 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, Ident, ItemStruct, Lit};
+
+#[proc_macro_attribute]
+pub fn annotations(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(item as ItemStruct);
+    let name = input.ident.clone();
+    let fields = input
+        .fields
+        .iter()
+        .map(|f| f.ident.clone().unwrap())
+        .collect::<Vec<Ident>>();
+
+    // Match on the literal to extract the string value
+    let comm_lit = match parse_macro_input!(attr as Lit) {
+        Lit::Str(lit_str) => lit_str.value(), // This will give "#" without quotes
+        _ => panic!("Expected a string literal"),
+    };
+
+    // Generate regex
+    let mut reg = format!("^{}|", &comm_lit);
+    {
+        for field in fields.iter() {
+            reg.push_str(&(field.to_string()));
+            reg.push_str("\\b");
+        }
+
+        reg.push_str(r#"|\w+"#);
+    }
+    // Example of generated regex:
+    // ^#
+    // |ann1\b|ann2\b|ann3\b|ann4\b
+    // |\w+
+
+    TokenStream::from(quote! {
+        #[derive(Default, Debug)]
+        #input
+
+        impl std::ops::BitOrAssign for #name{
+            fn bitor_assign(&mut self, rhs: Self) {
+                // Unfold fields
+                // Read more: https://docs.rs/quote/latest/quote/macro.quote.html#interpolation
+                #( self.#fields |= rhs.#fields; )*
+            }
+        }
+
+        impl #name {
+            /// Autogenerated by windmill-macros
+            pub fn parse(inner_content: &str) -> Self{
+                let mut res = Self::default();
+                lazy_static::lazy_static! {
+                    static ref RE: regex::Regex = regex::Regex::new(#reg).unwrap();
+                }
+                // Create lines stream
+                let mut lines = inner_content.lines();
+                'outer: while let Some(line) = lines.next() {
+                    // If comment sign(s) on the right place
+                    let mut comms = false;
+                    // New instance
+                    // We will apply it if in line only annotations
+                    let mut new = Self::default();
+
+                    'inner: for (i, mat) in RE.find_iter(line).enumerate() {
+
+                        match mat.as_str(){
+                            #comm_lit if i == 0 => {
+                                comms = true;
+                                continue 'inner;
+                            },
+
+                            // Will expand into something like:
+                            //            "ann1"   => new.ann1    = true,
+                            //            "ann2"   => new.ann2    = true,
+                            //            "ann3"   => new.ann3    = true,
+                            #( stringify!(#fields) => new.#fields = true, )*
+                            // Non annotations
+                            _ => continue 'outer,
+                        };
+                    }
+
+                    if !comms {
+                        // We dont want to continue if line does not start with #
+                        return res;
+                    }
+
+                    // Apply changes
+                    res |= new;
+                }
+
+                res
+            }
+        }
+    })
+}

backend/windmill-macros/tests/annotations.rs

@@ -0,0 +1,169 @@
+#[cfg(test)]
+mod annotations_tests {
+
+    extern crate windmill_macros;
+    use itertools::Itertools;
+    use windmill_macros::annotations;
+
+    // Previous implementation.
+    // We have to make sure that new one works the same as old one
+    fn old(inner_content: &str) -> Annotations {
+        let annotations = inner_content
+            .lines()
+            .take_while(|x| x.starts_with("#"))
+            .map(|x| x.to_string().replace("#", "").trim().to_string())
+            .collect_vec();
+
+        let ann1: bool = annotations.contains(&"ann1".to_string());
+        let ann2: bool = annotations.contains(&"ann2".to_string());
+        let ann3: bool = annotations.contains(&"ann3".to_string());
+        let ann4: bool = annotations.contains(&"ann4".to_string());
+        let ann5: bool = annotations.contains(&"ann5".to_string());
+
+        Annotations { ann1, ann2, ann3, ann4, ann5 }
+    }
+
+    #[annotations("#")]
+    #[derive(Eq, PartialEq, Copy, Clone)]
+    pub struct Annotations {
+        pub ann1: bool,
+        pub ann2: bool,
+        pub ann3: bool,
+        pub ann4: bool,
+        pub ann5: bool,
+    }
+
+    #[annotations("//")]
+    #[derive(Eq, PartialEq, Copy, Clone)]
+    pub struct SlashedAnnotations {
+        pub ann1: bool,
+        pub ann2: bool,
+        pub ann3: bool,
+        pub ann4: bool,
+    }
+
+    #[annotations("--")]
+    #[derive(Eq, PartialEq, Copy, Clone)]
+    pub struct MinusedAnnotations {
+        pub ann1: bool,
+        pub ann2: bool,
+    }
+
+    // e.g. rust, TS and JS
+    #[test]
+    fn slashed_annotations() {
+        let cont = "// ann1
+// ann2
+//ann3";
+        assert_eq!(
+            SlashedAnnotations { ann1: true, ann2: true, ann3: true, ann4: false },
+            SlashedAnnotations::parse(cont)
+        );
+    }
+
+    // e.g. Haskell, SQL
+    #[test]
+    fn minused_annotations() {
+        let cont = "-- ann1
+-- ann2";
+        assert_eq!(
+            MinusedAnnotations { ann1: true, ann2: true },
+            MinusedAnnotations::parse(cont)
+        );
+    }
+
+    #[test]
+    fn simple_integration() {
+        let cont = "# ann1";
+        let expected = Annotations { ann1: true, ..Default::default() };
+        assert_eq!(expected, old(cont));
+        assert_eq!(expected, Annotations::parse(cont));
+    }
+
+    #[test]
+    fn multiline_integration() {
+        let cont = "# ann2
+# ann3
+# ann4
+# ann5
+            ";
+        let expected = Annotations {
+            ann1: false,
+            ann2: true,
+            ann3: true,
+            ann4: true,
+            ann5: true,
+            //
+        };
+        assert_eq!(expected, old(cont));
+        assert_eq!(expected, Annotations::parse(cont));
+    }
+
+    #[test]
+    fn spacing_integration() {
+        // First line is ignored and not used
+        {
+            let cont = "
+# ann2";
+            let expected = Annotations { ..Default::default() };
+            assert_eq!(expected, old(cont));
+            assert_eq!(expected, Annotations::parse(cont));
+        }
+        // Wrong spacing for ann3
+        {
+            let cont = "# ann2
+                        # ann3";
+
+            let expected = Annotations { ann2: true, ..Default::default() };
+            assert_eq!(expected, old(cont));
+            assert_eq!(expected, Annotations::parse(cont));
+        }
+
+        // Drunk but valid spacing
+        {
+            let cont = "#ann1
+#                        ann2";
+            let expected = Annotations { ann2: true, ann1: true, ..Default::default() };
+            assert_eq!(expected, old(cont));
+            assert_eq!(expected, Annotations::parse(cont));
+        }
+    }
+
+    #[test]
+    fn comments_inbetween_integration() {
+        let cont = "# ann2
+# Just comment, has nothing to do with annotations
+# Another comment: ann1 ann2 ann3
+# ann4 is not valid annotation
+# Actual annotation next line:
+# ann5
+
+# Should be ignored
+# ann3
+            ";
+        let expected = Annotations { ann2: true, ann5: true, ..Default::default() };
+        assert_eq!(expected, old(cont));
+        assert_eq!(expected, Annotations::parse(cont));
+    }
+
+    #[test]
+    fn hash_collision() {
+        // TODO
+    }
+    #[test]
+    fn non_matching_integration() {
+        {
+            let cont = r#" "ann1", ann2 "#;
+            let expected = Annotations::default();
+            assert_eq!(expected, old(cont));
+            assert_eq!(expected, Annotations::parse(cont));
+        }
+        // Empty
+        {
+            let cont = "";
+            let expected = Annotations::default();
+            assert_eq!(expected, old(cont));
+            assert_eq!(expected, Annotations::parse(cont));
+        }
+    }
+}

backend/windmill-worker/src/bigquery_executor.rs

@@ -5,7 +5,6 @@ use futures::{FutureExt, TryFutureExt};
 use serde_json::{json, value::RawValue, Value};
 use windmill_common::error::to_anyhow;
 use windmill_common::jobs::QueuedJob;
-use windmill_common::worker::get_sql_annotations;
 use windmill_common::{error::Error, worker::to_raw_value};
 use windmill_parser_sql::{
     parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params,
@@ -238,7 +237,7 @@ pub async fn do_bigquery(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
+    let annotations = windmill_common::worker::SqlAnnotations::parse(query);
 
     let service_account = CustomServiceAccount::from_json(&database)
         .map_err(|e| Error::ExecutionErr(e.to_string()))?;

backend/windmill-worker/src/bun_executor.rs

@@ -47,7 +47,7 @@ use windmill_common::{
     get_latest_hash_for_path,
     jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
     scripts::ScriptLang,
-    worker::{exists_in_cache, get_annotation_ts, save_cache, write_file},
+    worker::{exists_in_cache, save_cache, write_file},
     DB,
 };
 
@@ -663,7 +663,7 @@ pub async fn prebundle_bun_script(
     if exists_in_cache(&local_path, &remote_path).await {
         return Ok(());
     }
-    let annotation = get_annotation_ts(inner_content);
+    let annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
     if annotation.nobundling {
         return Ok(());
     }
@@ -676,9 +676,9 @@ pub async fn prebundle_bun_script(
         &token,
         w_id,
         script_path,
-        if annotation.nodejs_mode {
+        if annotation.nodejs {
             LoaderMode::NodeBundle
-        } else if annotation.native_mode {
+        } else if annotation.native {
             LoaderMode::BrowserBundle
         } else {
             LoaderMode::BunBundle
@@ -800,7 +800,7 @@ pub async fn handle_bun_job(
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
     occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
-    let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);
+    let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
 
     let (mut has_bundle_cache, cache_logs, local_path, remote_path) =
         if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() {
@@ -822,7 +822,7 @@ pub async fn handle_bun_job(
 
     if !codebase.is_some() && !has_bundle_cache {
         let _ = write_file(job_dir, "main.ts", inner_content)?;
-    } else if !annotation.native_mode && codebase.is_none() {
+    } else if !annotation.native && codebase.is_none() {
         let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?;
     };
 
@@ -830,7 +830,7 @@ pub async fn handle_bun_job(
         get_common_bun_proc_envs(Some(&base_internal_url)).await;
 
     if codebase.is_some() {
-        annotation.nodejs_mode = true
+        annotation.nodejs = true
     }
     let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) {
         Some(main_override) => {
@@ -844,7 +844,7 @@ pub async fn handle_bun_job(
     };
 
     #[cfg(not(feature = "enterprise"))]
-    if annotation.nodejs_mode || annotation.npm_mode {
+    if annotation.nodejs || annotation.npm {
         return Err(error::Error::ExecutionErr(
             "Nodejs / npm mode is an EE feature".to_string(),
         ));
@@ -875,20 +875,20 @@ pub async fn handle_bun_job(
         pull_codebase(&job.workspace_id, codebase, job_dir).await?;
     } else if let Some(reqs) = requirements_o.as_ref() {
         let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
-        if splitted.len() != 2 && !annotation.npm_mode {
+        if splitted.len() != 2 && !annotation.npm {
             return Err(error::Error::ExecutionErr(
                 format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
             ));
         }
 
         let _ = write_file(job_dir, "package.json", &splitted[0])?;
-        let lockb = if annotation.npm_mode { "" } else { splitted[1] };
+        let lockb = if annotation.npm { "" } else { splitted[1] };
         if lockb != EMPTY_FILE {
             let mut skip_install = false;
             let mut create_buntar = false;
             let mut buntar_path = "".to_string();
 
-            if !annotation.npm_mode {
+            if !annotation.npm {
                 let _ = write_lockb(&splitted[1], job_dir).await?;
 
                 let mut sha_path = sha2::Sha256::new();
@@ -921,7 +921,7 @@ pub async fn handle_bun_job(
                     job_dir,
                     worker_name,
                     common_bun_proc_envs.clone(),
-                    annotation.npm_mode,
+                    annotation.npm,
                     &mut Some(occupancy_metrics),
                 )
                 .await?;
@@ -963,7 +963,7 @@ pub async fn handle_bun_job(
             worker_name,
             false,
             None,
-            annotation.npm_mode,
+            annotation.npm,
             &mut Some(occupancy_metrics),
         )
         .await?;
@@ -971,19 +971,19 @@ pub async fn handle_bun_job(
         // }
     }
 
-    let mut init_logs = if annotation.native_mode {
+    let mut init_logs = if annotation.native {
         "\n\n--- NATIVE CODE EXECUTION ---\n".to_string()
     } else if has_bundle_cache {
-        if annotation.nodejs_mode {
+        if annotation.nodejs {
             "\n\n--- NODE BUNDLE SNAPSHOT EXECUTION ---\n".to_string()
         } else {
             "\n\n--- BUN BUNDLE SNAPSHOT EXECUTION ---\n".to_string()
         }
     } else if codebase.is_some() {
         "\n\n--- NODE CODEBASE SNAPSHOT EXECUTION ---\n".to_string()
-    } else if annotation.native_mode {
+    } else if annotation.native {
         "\n\n--- NATIVE CODE EXECUTION ---\n".to_string()
-    } else if annotation.nodejs_mode {
+    } else if annotation.nodejs {
         write_file(job_dir, "main.ts", &remove_pinned_imports(inner_content)?)?;
         "\n\n--- NODE CODE EXECUTION ---\n".to_string()
     } else {
@@ -1003,7 +1003,7 @@ pub async fn handle_bun_job(
     }
 
     let write_wrapper_f = async {
-        if !has_bundle_cache && annotation.native_mode {
+        if !has_bundle_cache && annotation.native {
             return Ok(()) as error::Result<()>;
         }
         // let mut start = Instant::now();
@@ -1126,7 +1126,7 @@ try {{
 
     let reserved_variables_args_out_f = async {
         let args_and_out_f = async {
-            if !annotation.native_mode {
+            if !annotation.native {
                 create_args_and_out_file(&client, job, job_dir, db).await?;
             }
             Ok(()) as Result<()>
@@ -1143,7 +1143,7 @@ try {{
     let build_cache = !has_bundle_cache
         && !annotation.nobundling
         && !codebase.is_some()
-        && (requirements_o.is_some() || annotation.native_mode);
+        && (requirements_o.is_some() || annotation.native);
 
     let write_loader_f = async {
         if build_cache {
@@ -1153,9 +1153,9 @@ try {{
                 &client.get_token().await,
                 &job.workspace_id,
                 &job.script_path(),
-                if annotation.nodejs_mode {
+                if annotation.nodejs {
                     LoaderMode::NodeBundle
-                } else if annotation.native_mode {
+                } else if annotation.native {
                     LoaderMode::BrowserBundle
                 } else {
                     LoaderMode::BunBundle
@@ -1171,7 +1171,7 @@ try {{
                 &client.get_token().await,
                 &job.workspace_id,
                 &job.script_path(),
-                if annotation.nodejs_mode {
+                if annotation.nodejs {
                     LoaderMode::Node
                 } else {
                     LoaderMode::Bun
@@ -1217,7 +1217,7 @@ try {{
                     }
                 }
             }
-            if !annotation.native_mode {
+            if !annotation.native {
                 let ex_wrapper = read_file_content(&format!("{job_dir}/wrapper.mjs")).await?;
                 write_file(
                     job_dir,
@@ -1231,7 +1231,7 @@ try {{
             }
             fs::remove_file(format!("{job_dir}/main.ts"))?;
             has_bundle_cache = true;
-        } else if annotation.nodejs_mode {
+        } else if annotation.nodejs {
             generate_wrapper_mjs(
                 job_dir,
                 &job.workspace_id,
@@ -1247,9 +1247,14 @@ try {{
             .await?;
         }
     }
-    if annotation.native_mode {
+    if annotation.native {
         #[cfg(not(feature = "deno_core"))]
-        return Ok(to_raw_value("").unwrap());
+        {
+            tracing::error!(
+                r#""deno_core" feature is not activated, but "//native" annotation used. Returning empty value..."#
+            );
+            return Ok(to_raw_value("").unwrap());
+        }
 
         #[cfg(feature = "deno_core")]
         {
@@ -1309,14 +1314,7 @@ try {{
             job_dir,
             "run.config.proto",
             &NSJAIL_CONFIG_RUN_BUN_CONTENT
-                .replace(
-                    "{LANG}",
-                    if annotation.nodejs_mode {
-                        "nodejs"
-                    } else {
-                        "bun"
-                    },
-                )
+                .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" })
                 .replace("{JOB_DIR}", job_dir)
                 .replace("{CACHE_DIR}", BUN_CACHE_DIR)
                 .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
@@ -1324,7 +1322,7 @@ try {{
                     "{SHARED_MOUNT}",
                     &shared_mount.replace(
                         "/tmp/shared",
-                        if annotation.nodejs_mode {
+                        if annotation.nodejs {
                             "/tmp/nodejs/shared"
                         } else {
                             "/tmp/bun/shared"
@@ -1334,7 +1332,7 @@ try {{
         )?;
 
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
-        let args = if annotation.nodejs_mode {
+        let args = if annotation.nodejs {
             vec![
                 "--config",
                 "run.config.proto",
@@ -1378,7 +1376,7 @@ try {{
             .stderr(Stdio::piped());
         start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
     } else {
-        let cmd = if annotation.nodejs_mode {
+        let cmd = if annotation.nodejs {
             let script_path = format!("{job_dir}/wrapper.mjs");
 
             let mut bun_cmd = Command::new(&*NODE_BIN_PATH);
@@ -1430,7 +1428,7 @@ try {{
 
         start_child_process(
             cmd,
-            if annotation.nodejs_mode {
+            if annotation.nodejs {
                 &*NODE_BIN_PATH
             } else {
                 &*BUN_PATH
@@ -1535,10 +1533,10 @@ pub async fn start_worker(
     let common_bun_proc_envs: HashMap<String, String> =
         get_common_bun_proc_envs(Some(&base_internal_url)).await;
 
-    let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);
+    let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
 
     //TODO: remove this when bun dedicated workers work without issues
-    annotation.nodejs_mode = true;
+    annotation.nodejs = true;
 
     let context = variables::get_reserved_variables(
         db,
@@ -1592,7 +1590,7 @@ pub async fn start_worker(
                 job_dir,
                 worker_name,
                 common_bun_proc_envs.clone(),
-                annotation.npm_mode,
+                annotation.npm,
                 &mut None,
             )
             .await?;
@@ -1613,7 +1611,7 @@ pub async fn start_worker(
             worker_name,
             false,
             None,
-            annotation.npm_mode,
+            annotation.npm,
             &mut None,
         )
         .await?;
@@ -1691,7 +1689,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
             token,
             w_id,
             script_path,
-            if annotation.nodejs_mode {
+            if annotation.nodejs {
                 LoaderMode::Node
             } else {
                 LoaderMode::Bun
@@ -1700,7 +1698,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
         .await?;
     }
 
-    if annotation.nodejs_mode && !codebase.is_some() {
+    if annotation.nodejs && !codebase.is_some() {
         generate_wrapper_mjs(
             job_dir,
             w_id,
@@ -1716,7 +1714,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
         .await?;
     }
 
-    if annotation.nodejs_mode {
+    if annotation.nodejs {
         let script_path = format!("{job_dir}/wrapper.mjs");
 
         handle_dedicated_process(

backend/windmill-worker/src/mssql_executor.rs

@@ -9,7 +9,7 @@ use tokio::net::TcpStream;
 use tokio_util::compat::TokioAsyncWriteCompatExt;
 use uuid::Uuid;
 use windmill_common::error::{self, Error};
-use windmill_common::worker::{get_sql_annotations, to_raw_value};
+use windmill_common::worker::to_raw_value;
 use windmill_common::{error::to_anyhow, jobs::QueuedJob};
 use windmill_parser_sql::{parse_db_resource, parse_mssql_sig};
 use windmill_queue::{append_logs, CanceledBy};
@@ -68,7 +68,7 @@ pub async fn do_mssql(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
+    let annotations = windmill_common::worker::SqlAnnotations::parse(query);
 
     let mut config = Config::new();
 

backend/windmill-worker/src/mysql_executor.rs

@@ -13,7 +13,7 @@ use tokio::sync::Mutex;
 use windmill_common::{
     error::{to_anyhow, Error},
     jobs::QueuedJob,
-    worker::{get_sql_annotations, to_raw_value},
+    worker::to_raw_value,
 };
 use windmill_parser_sql::{
     parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params,
@@ -148,7 +148,7 @@ pub async fn do_mysql(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
+    let annotations = windmill_common::worker::SqlAnnotations::parse(query);
 
     let opts = OptsBuilder::default()
         .db_name(Some(database.database))

backend/windmill-worker/src/pg_executor.rs

@@ -30,7 +30,7 @@ use tokio_postgres::{
 };
 use uuid::Uuid;
 use windmill_common::error::{self, Error};
-use windmill_common::worker::{get_sql_annotations, to_raw_value, CLOUD_HOSTED};
+use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 use windmill_common::{error::to_anyhow, jobs::QueuedJob};
 use windmill_parser::{Arg, Typ};
 use windmill_parser_sql::{
@@ -191,7 +191,7 @@ pub async fn do_postgresql(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
+    let annotations = windmill_common::worker::SqlAnnotations::parse(query);
 
     let sslmode = match database.sslmode.as_deref() {
         Some("allow") => "prefer".to_string(),

backend/windmill-worker/src/python_executor.rs

@@ -898,7 +898,7 @@ async fn handle_python_deps(
     let requirements = match requirements_o {
         Some(r) => r,
         None => {
-            let annotation = windmill_common::worker::get_annotation_python(inner_content);
+            let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content);
             let mut already_visited = vec![];
 
             let requirements = windmill_parser_py_imports::parse_python_imports(

backend/windmill-worker/src/snowflake_executor.rs

@@ -9,7 +9,6 @@ use serde_json::{json, value::RawValue, Value};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use windmill_common::error::to_anyhow;
-use windmill_common::worker::get_sql_annotations;
 
 use windmill_common::jobs::QueuedJob;
 use windmill_common::{error::Error, worker::to_raw_value};
@@ -266,7 +265,7 @@ pub async fn do_snowflake(
         return Err(Error::BadRequest("Missing database argument".to_string()));
     };
 
-    let annotations = get_sql_annotations(query);
+    let annotations = windmill_common::worker::SqlAnnotations::parse(query);
 
     let qualified_username = format!(
         "{}.{}",

backend/windmill-worker/src/worker_lockfiles.rs

@@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue};
 use windmill_common::get_latest_deployed_hash_for_path;
 use windmill_common::jobs::JobPayload;
 use windmill_common::scripts::ScriptHash;
-use windmill_common::worker::{get_annotation_ts, to_raw_value, to_raw_value_owned, write_file};
+use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file};
 use windmill_common::{
     error::{self, to_anyhow},
     flows::FlowValue,
@@ -953,10 +953,10 @@ async fn lock_modules<'c>(
                 }
 
                 if language == ScriptLang::Bun || language == ScriptLang::Bunnative {
-                    let anns = get_annotation_ts(&content);
-                    if anns.native_mode && language == ScriptLang::Bun {
+                    let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
+                    if anns.native && language == ScriptLang::Bun {
                         language = ScriptLang::Bunnative;
-                    } else if !anns.native_mode && language == ScriptLang::Bunnative {
+                    } else if !anns.native && language == ScriptLang::Bunnative {
                         language = ScriptLang::Bun;
                     };
                 }
@@ -1003,10 +1003,10 @@ async fn lock_modules<'c>(
 
 fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool {
     if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative {
-        let anns = get_annotation_ts(&content);
-        if anns.native_mode && language == &ScriptLang::Bun {
+        let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
+        if anns.native && language == &ScriptLang::Bun {
             return false;
-        } else if !anns.native_mode && language == &ScriptLang::Bunnative {
+        } else if !anns.native && language == &ScriptLang::Bunnative {
             return false;
         };
     }
@@ -1077,11 +1077,13 @@ async fn lock_modules_app(
                             match new_lock {
                                 Ok(new_lock) => {
                                     append_logs(&job.id, &job.workspace_id, logs, db).await;
-                                    let anns = get_annotation_ts(&content);
-                                    let nlang = if anns.native_mode && language == ScriptLang::Bun {
+                                    let anns =
+                                        windmill_common::worker::TypeScriptAnnotations::parse(
+                                            &content,
+                                        );
+                                    let nlang = if anns.native && language == ScriptLang::Bun {
                                         Some(ScriptLang::Bunnative)
-                                    } else if !anns.native_mode && language == ScriptLang::Bunnative
-                                    {
+                                    } else if !anns.native && language == ScriptLang::Bunnative {
                                         Some(ScriptLang::Bun)
                                     } else {
                                         None
@@ -1437,7 +1439,7 @@ async fn capture_dependency_job(
         }
         ScriptLang::Bun | ScriptLang::Bunnative => {
             let npm_mode = npm_mode.unwrap_or_else(|| {
-                windmill_common::worker::get_annotation_ts(job_raw_code).npm_mode
+                windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm
             });
             if !raw_deps {
                 let _ = write_file(job_dir, "main.ts", job_raw_code)?;