starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-04 08:01:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

More types and req parsing

Browse Source
This commit is contained in:
wendrul
2024-12-10 17:06:30 +01:00
parent f94fed5ee7
commit cd2791d09e
3 changed files with 162 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/parsers/windmill-parser-csharp/src/lib.rs
+101 -17
View File
@@ -32,7 +32,7 @@ pub fn parse_csharp_signature(code: &str) -> anyhow::Result<MainArgSignature> {
if let Some(param_list) = sig.child_by_field_name("parameters") {
for c in param_list.children(&mut param_list.walk()) {
if c.kind() == "parameter" {
let (otyp, typ, name) = parse_csharp_typ(c, code);
let (otyp, typ, name) = parse_csharp_typ(c, code)?;
args.push(Arg {
name,
otyp,
@@ -41,19 +41,8 @@ pub fn parse_csharp_signature(code: &str) -> anyhow::Result<MainArgSignature> {
has_default: false,
oidx: None,
});
for (i, w) in c.children(&mut c.walk()).enumerate() {
let s = w.utf8_text(code.as_bytes());
println!(
" {:?} - {:?} - {:?}",
w,
c.field_name_for_child((i) as u32),
s
);
}
}
}
} else {
println!("No one with parameter_list");
}
}
@@ -66,7 +55,46 @@ pub fn parse_csharp_signature(code: &str) -> anyhow::Result<MainArgSignature> {
})
}
fn parse_csharp_typ<'a>(param_node: Node<'a>, code: &str) -> (Option<String>, Typ, String) {
fn find_typ<'a>(typ_node: Node<'a>, code: &str) -> anyhow::Result<Typ> {
match typ_node.kind() {
"predefined_type" => {
match typ_node.utf8_text(code.as_bytes()) {
Ok("string") => Ok(Typ::Str(None)),
Ok("sbyte") | Ok("System.SByte") => Ok(Typ::Bytes),
Ok("byte") | Ok("System.Byte") => Ok(Typ::Bytes),
Ok("short") | Ok("System.Int16") => Ok(Typ::Int),
Ok("ushort") | Ok("System.UInt16") => Ok(Typ::Int),
Ok("int") | Ok("System.Int32") => Ok(Typ::Int),
Ok("uint") | Ok("System.UInt32") => Ok(Typ::Int),
Ok("long") | Ok("System.Int64") => Ok(Typ::Int),
Ok("ulong") | Ok("System.UInt64") => Ok(Typ::Int),
Ok("char") | Ok("System.Char") => Ok(Typ::Str(None)),
Ok("float") | Ok("System.Single") => Ok(Typ::Float),
Ok("double") | Ok("System.Double") => Ok(Typ::Float),
Ok("bool") | Ok("System.Boolean") => Ok(Typ::Bool),
Ok("decimal") | Ok("System.Decimal") => Ok(Typ::Float),
Ok("object") => Ok(Typ::Object(vec![])), // TODO: Complete the object type
Ok(s) => Err(anyhow!("Unknown type `{s}`")),
Err(e) => Err(anyhow!("Error getting type name: {}", e)),
}
}
"array_type" => {
let new_typ_node = typ_node
.child_by_field_name("type")
.ok_or(anyhow!("Failed to find inner type of array type"))?;
Ok(Typ::List(Box::new(find_typ(new_typ_node, code)?)))
}
"identifier" => Ok(Typ::Unknown),
"generic_name" => Ok(Typ::Unknown),
"pointer_type" => Ok(Typ::Int),
wc => Err(anyhow!("Unexpected node kind: {}", wc)),
}
}
fn parse_csharp_typ<'a>(
param_node: Node<'a>,
code: &str,
) -> anyhow::Result<(Option<String>, Typ, String)> {
let name = param_node
.child_by_field_name("name")
.and_then(|n| n.utf8_text(code.as_bytes()).ok())
@@ -76,9 +104,9 @@ fn parse_csharp_typ<'a>(param_node: Node<'a>, code: &str) -> (Option<String>, Ty
.and_then(|n| n.utf8_text(code.as_bytes()).ok())
.map(|s| s.to_string());
let typ = Typ::Str(None);
let typ = find_typ(otyp_node.unwrap(), code)?;
(otyp, typ, name.to_string())
Ok((otyp, typ, name.to_string()))
}
// Function to find the Main method's signature
@@ -108,6 +136,39 @@ fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> Option<Node<'a>>
return None;
}
pub fn parse_csharp_reqs(code: &str) -> (Vec<(String, Option<String>)>, Vec<usize>) {
let mut nuget_reqs = Vec::new();
let mut pkg_lines = Vec::new();
for (i, line) in code.split("\n").enumerate() {
if line.starts_with('#') {
if let Some(req) = parse_nuget_req(&line) {
pkg_lines.push(i);
nuget_reqs.push(req);
}
} else {
break; // Stop processing after the first non-comment line
}
}
(nuget_reqs, pkg_lines)
}
fn parse_nuget_req(line: &str) -> Option<(String, Option<String>)> {
// Check if the line starts with `#r "nuget:`
if let Some(start) = line.find("#r \"nuget:") {
// Extract the content after `#r "nuget:`
let start_idx = start + 10;
let end_idx = line[start_idx..].find('"')?;
let line = &line[start_idx..start_idx + end_idx];
let mut splitted = line.split(",");
if let Some(pkg) = splitted.next() {
return Some((pkg.trim().to_string(), splitted.next().map(|s| s.trim().to_string())));
}
}
None
}
#[cfg(test)]
mod test {
@@ -120,7 +181,7 @@ using System;
class LilProgram
{
public static string Main(string myString = "World", int myInt)
public static string Main(string myString = "World", int myInt, string[] jj)
{
Console.Writeline("Hello!!");
return "yeah";
@@ -129,7 +190,7 @@ class LilProgram
}"#;
let ret = parse_csharp_signature(code).unwrap();
assert_eq!(ret.args.len(), 2);
assert_eq!(ret.args.len(), 3);
assert_eq!(ret.args[0].name, "myString");
assert_eq!(ret.args[0].otyp, Some("string".to_string()));
@@ -138,5 +199,28 @@ class LilProgram
assert_eq!(ret.args[1].name, "myInt");
assert_eq!(ret.args[1].otyp, Some("int".to_string()));
assert_eq!(ret.args[1].typ, Typ::Int);
assert_eq!(ret.args[2].name, "jj");
assert_eq!(ret.args[2].otyp, Some("string[]".to_string()));
assert_eq!(ret.args[2].typ, Typ::List(Box::new(Typ::Str(None))));
}
#[test]
fn test_parse_csharp_reqs() {
let file_content = r#"#r "nuget: AutoMapper, 6.1.0"
#r "nuget: Newtonsoft.Json, 13.0.1"
#r "nuget: Serilog, 2.10.0"
# This is a comment
#r "nuget: Serilog, 2.10.0"
using System;
"#;
let requirements = parse_csharp_reqs(file_content).0;
assert_eq!(requirements.len(), 3);
assert_eq!(requirements[0], ("AutoMapper".to_string(), Some("6.1.0".to_string())));
assert_eq!(requirements[1], ("Newtonsoft.Json".to_string(), Some("13.0.1".to_string())));
assert_eq!(requirements[2], ("Serilog".to_string(), Some("2.10.0".to_string())));
}
}
backend/src/main.rs
+4 -5
View File
@@ -66,11 +66,10 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
PY311_CACHE_DIR, CSHARP_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, TMP_LOGS_DIR,
UV_CACHE_DIR,
RUST_CACHE_DIR, CSHARP_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
POWERSHELL_CACHE_DIR, PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR,
TMP_LOGS_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
backend/windmill-worker/src/csharp_executor.rs
+57 -20
View File
@@ -2,6 +2,7 @@ use anyhow::anyhow;
use serde_json::value::RawValue;
use std::{collections::HashMap, path::Path, process::Stdio};
use uuid::Uuid;
use windmill_parser_csharp::parse_csharp_reqs;
use windmill_parser_rust::parse_rust_deps_into_manifest;
use itertools::Itertools;
@@ -33,22 +34,39 @@ lazy_static::lazy_static! {
const CSHARP_OBJECT_STORE_PREFIX: &str = "csharpbin/";
fn gen_cs_proj(code: &str, job_dir: &str) -> anyhow::Result<()> {
fn gen_cs_proj(
code: &str,
job_dir: &str,
reqs: Vec<(String, Option<String>)>,
) -> anyhow::Result<()> {
let pkgs = reqs
.into_iter()
.map(|(pkg, vrsion_o)| {
let version = vrsion_o
.map(|v| format!("Version=\"{v}\""))
.unwrap_or("".to_string());
format!(" <PackageReference Include=\"{pkg}\" {version}/>")
})
.join("\n");
write_file(
job_dir,
"Main.csproj",
r#"<Project Sdk="Microsoft.NET.Sdk">
&format!(
r#"<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<StartupObject>WindmillScriptCSharpInternal.Wrapper</StartupObject>
</PropertyGroup>
<ItemGroup>
{pkgs}
</ItemGroup>
</Project>
"#,
"#
),
)?;
write_file(job_dir, "Script.cs", code)?;
@@ -66,7 +84,8 @@ fn gen_cs_proj(code: &str, job_dir: &str) -> anyhow::Result<()> {
.map(|x| {
Ok(format!(
" public {} {} {{ get; set; }}",
x.otyp.ok_or(anyhow!("Type not found for argument {}", x.name))?,
x.otyp
.ok_or(anyhow!("Type not found for argument {}", x.name))?,
&x.name,
))
})
@@ -173,17 +192,6 @@ async fn build_cs_proj(
.await?;
append_logs(job_id, w_id, "\n\n", db).await;
for entry in std::fs::read_dir(job_dir)? {
let entry = entry?;
let path = entry.path();
// Print file or directory name
if let Some(name) = path.file_name() {
// println!("{}", name.to_string_lossy());
println!("{path:?}");
}
}
let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR);
match save_cache(
@@ -202,6 +210,18 @@ async fn build_cs_proj(
}
}
fn remove_lines_from_text(contents: &str, indices_to_remove: Vec<usize>) -> String {
let mut result = Vec::new();
for (i, line) in contents.lines().enumerate() {
if !indices_to_remove.contains(&i) {
result.push(line);
}
}
result.join("\n")
}
pub async fn handle_csharp_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -219,8 +239,6 @@ pub async fn handle_csharp_job(
) -> Result<Box<RawValue>, Error> {
check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?;
let hash = calculate_hash(&format!(
"{}{}",
inner_content,
@@ -253,7 +271,26 @@ pub async fn handle_csharp_job(
let logs1 = format!("{cache_logs}\n\n--- DOTNET BUILD ---\n");
append_logs(&job.id, &job.workspace_id, logs1, db).await;
gen_cs_proj(inner_content, job_dir)?;
let (reqs, lines_to_remove) = parse_csharp_reqs(inner_content);
for req in &reqs {
append_logs(
&job.id,
&job.workspace_id,
format!(
"Requirement detected: {} {}\n",
req.0,
req.1.as_ref().unwrap_or(&"".to_string())
),
db,
)
.await;
}
let inner_content = remove_lines_from_text(inner_content, lines_to_remove);
let code = inner_content.as_str();
gen_cs_proj(code, job_dir, reqs)?;
build_cs_proj(
&job.id,