diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b9f21090c6..922d069483 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1396,6 +1396,16 @@ dependencies = [ "url", ] +[[package]] +name = "gosyn" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1584c4cbcb6f1d97481c19d1951c64f6e1256e5a03c28159ae45431c00511ee7" +dependencies = [ + "strum", + "unic-ucd-category", +] + [[package]] name = "h2" version = "0.3.16" @@ -2014,6 +2024,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "matchit" version = "0.7.0" @@ -3720,6 +3736,28 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "subtle" version = "2.4.1" @@ -4426,6 +4464,18 @@ dependencies = [ "unic-ucd-version", ] +[[package]] +name = "unic-ucd-category" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" +dependencies = [ + "matches", + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + [[package]] name = "unic-ucd-ident" version = "0.9.0" @@ -4974,6 +5024,7 @@ name = "windmill-parser-go" version = "1.86.0" dependencies = [ "anyhow", + "gosyn", "itertools", "phf", "unicode-general-category", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f9731e1f45..945f2e6717 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -28,11 +28,7 @@ name = "windmill" path = "./src/main.rs" [features] -enterprise = [ - "windmill-worker/enterprise", - "windmill-queue/enterprise", - "windmill-api/enterprise", -] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"] [dependencies] anyhow.workspace = true @@ -154,4 +150,5 @@ async-stripe = { version = "0.14", features = [ "checkout", ] } async_zip = { version = "0.0.11", features = ["full"] } -once_cell = "1.17.1" \ No newline at end of file +once_cell = "1.17.1" +gosyn = "0.2.2" diff --git a/backend/parsers/windmill-parser-go/Cargo.toml b/backend/parsers/windmill-parser-go/Cargo.toml index 729b08f1ee..6bc0be873b 100644 --- a/backend/parsers/windmill-parser-go/Cargo.toml +++ b/backend/parsers/windmill-parser-go/Cargo.toml @@ -15,3 +15,4 @@ phf.workspace = true unicode-general-category.workspace = true itertools.workspace = true anyhow.workspace = true +gosyn.workspace = true \ No newline at end of file diff --git a/backend/parsers/windmill-parser-go/src/lib.rs b/backend/parsers/windmill-parser-go/src/lib.rs index b1ba30e82e..203d087191 100644 --- a/backend/parsers/windmill-parser-go/src/lib.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -1,29 +1,27 @@ #![allow(non_snake_case)] // TODO: switch to parse_* function naming -mod parser_go_ast; -mod parser_go_scanner; -mod parser_go_token; - +use gosyn::{ + ast::{Declaration, Expression, Field, Ident, StructType}, + parse_source, +}; use itertools::Itertools; -use parser_go_ast::{Decl, Expr}; -use parser_go_ast::{FieldList, Ident, StructType}; -use parser_go_token::{Position, Token}; -use std::fmt; -use windmill_common::error::to_anyhow; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; pub fn parse_go_sig(code: &str) -> windmill_common::error::Result { let filtered_code = filter_non_main(code); - let file = parse_file("main.go", &filtered_code).map_err(to_anyhow)?; - if let Some(Decl::FuncDecl(func)) = file.decls.first() { + let file = parse_source(&filtered_code).map_err(|x| anyhow::anyhow!(x.to_string()))?; + if let Some(func) = file.decl.iter().find_map(|x| match x { + Declaration::Function(func) if &func.name.name == "main" => Some(func), + _ => None, + }) { let args = func - .type_ + .typ .params .list .iter() .map(|param| { - let (otyp, typ) = get_type(param); + let (otyp, typ) = parse_go_typ(¶m.typ); Arg { name: get_name(param), otyp, typ, default: None, has_default: false } }) .collect_vec(); @@ -35,28 +33,37 @@ pub fn parse_go_sig(code: &str) -> windmill_common::error::Result (Option, Typ) { - let (otyp, typ) = ¶m - .type_ - .as_ref() - .map(|typ| parse_go_typ(typ)) - .unwrap_or_else(|| (None, Typ::Unknown)); - (otyp.clone(), typ.clone()) +pub fn parse_go_imports(code: &str) -> windmill_common::error::Result> { + let file = + parse_source(filter_non_imports(code)).map_err(|x| anyhow::anyhow!(x.to_string()))?; + let mut imports: Vec = file + .imports + .iter() + .filter_map(|x| { + if x.path.value.contains("/") { + Some(x.path.value.clone()) + } else { + None + } + }) + .collect(); + imports.sort(); + Ok(imports) } -fn get_name(param: &parser_go_ast::Field) -> String { +fn get_name(param: &Field) -> String { param - .names - .as_ref() - .and_then(|x| x.first().map(|y| y.name.to_string())) + .name + .first() + .map(|y| y.name.to_string()) .unwrap_or_else(|| "".to_string()) } -fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option, Typ) { +fn parse_go_typ(typ: &Expression) -> (Option, Typ) { match typ { - Expr::Ident(Ident { name, .. }) => ( - Some((*name).to_string()), - match *name { + Expression::Ident(Ident { name, .. }) => ( + Some((name).to_string()), + match name.as_str() { "int" => Typ::Int, "int16" => Typ::Int, "int32" => Typ::Int, @@ -66,15 +73,22 @@ fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option, Typ) { _ => Typ::Unknown, }, ), - Expr::ArrayType(array_type) => { - let (inner_otyp, inner_typ) = parse_go_typ(&*array_type.elt); + Expression::TypeSlice(slice_type) => { + let (inner_otyp, inner_typ) = parse_go_typ(&*slice_type.typ); + ( + inner_otyp.map(|x| format!("[]{}", x)), + Typ::List(Box::new(inner_typ)), + ) + } + Expression::TypeArray(array_type) => { + let (inner_otyp, inner_typ) = parse_go_typ(&*array_type.typ); ( inner_otyp.map(|x| format!("[]{x}")), Typ::List(Box::new(inner_typ)), ) } - Expr::StructType(StructType { fields: Some(FieldList { list, .. }), .. }) => { - let (otyps, typs): (Vec, Vec) = list + Expression::TypeStruct(StructType { fields, .. }) => { + let (otyps, typs): (Vec, Vec) = fields .iter() .map(|field| { let json_tag = field @@ -83,7 +97,7 @@ fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option, Typ) { .and_then(|x| x.value.strip_prefix("`json:\"")) .and_then(|x| x.strip_suffix("\"`")) .and_then(|x| x.split(',').last().map(|x| x.to_string())); - let (otyp, typ) = get_type(field); + let (otyp, typ) = parse_go_typ(&field.typ); let name = get_name(field); let key = json_tag.unwrap_or_else(|| name.to_string()); ( @@ -102,8 +116,8 @@ fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option, Typ) { Typ::Object(typs), ) } - Expr::InterfaceType(_) => (Some("interface{}".to_string()), Typ::Object(vec![])), - Expr::MapType(_) => ( + Expression::TypeInterface(_) => (Some("interface{}".to_string()), Typ::Object(vec![])), + Expression::TypeMap(_) => ( Some("map[string]interface{}".to_string()), Typ::Object(vec![]), ), @@ -202,10 +216,50 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam } } +#[test] +fn test_parse_go_import() -> anyhow::Result<()> { + let code = r#" +package inner + +import ( + "fmt" + "rsc.io/quote" + wmill "github.com/windmill-labs/windmill-go-client" +) + +// the main must return (interface{}, error) + +func main(x string, nested struct { + Foo string `json:"foo"` +}) (interface{}, error) { + fmt.Println("Hello, World") + fmt.Println(nested.Foo) + fmt.Println(quote.Opt()) + v, _ := wmill.GetVariable("f/examples/secret") + return v, nil +} +"#; + + assert_eq!( + parse_go_imports(code)?, + vec![ + "\"github.com/windmill-labs/windmill-go-client\"", + "\"rsc.io/quote\"" + ] + ); + + Ok(()) +} + +fn filter_non_imports(code: &str) -> String { + code.split_once("func ") + .map(|(x, _)| x.to_string()) + .unwrap_or_else(|| code.to_string()) +} fn filter_non_main(code: &str) -> String { const FUNC_MAIN: &str = "func main("; - let mut filtered_code = String::new(); + let mut filtered_code = "package main;\n".to_string(); let mut code_iter = code.split("\n"); let mut remaining: String = String::new(); while let Some(line) = code_iter.next() { @@ -237,1117 +291,3 @@ fn filter_non_main(code: &str) -> String { filtered_code.push_str("{}"); return filtered_code; } - -#[derive(Debug)] -pub enum ParserError { - ScannerError(parser_go_scanner::ScannerError), - UnexpectedEndOfFile, - UnexpectedToken, - UnexpectedTokenAt { at: String, token: Token, literal: String }, -} - -impl std::error::Error for ParserError {} - -impl From for ParserError { - fn from(e: parser_go_scanner::ScannerError) -> Self { - Self::ScannerError(e) - } -} - -impl fmt::Display for ParserError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "parser error: {:?}", self) - } -} - -pub type Result = std::result::Result; - -trait ResultExt { - fn required(self) -> Result; -} - -impl ResultExt for Result> { - fn required(self) -> Result { - self.and_then(|node| node.map_or(Err(ParserError::UnexpectedToken), |node| Ok(node))) - } -} - -pub fn parse_file<'a>(filename: &'a str, buffer: &'a str) -> Result> { - let parser_go_scanner = parser_go_scanner::Scanner::new(filename, buffer); - let mut parser = Parser::new(parser_go_scanner); - parser.next()?; - parser.SourceFile().required().map_err(|err| match err { - ParserError::UnexpectedToken => ParserError::UnexpectedTokenAt { - at: parser.current_step.0.to_string(), - token: parser.current_step.1, - literal: parser.current_step.2.to_owned(), - }, - err => err, - }) -} - -struct Parser<'parser_go_scanner> { - steps: parser_go_scanner::IntoIter<'parser_go_scanner>, - current_step: parser_go_scanner::Step<'parser_go_scanner>, - expr_level: isize, -} - -impl<'parser_go_scanner> Parser<'parser_go_scanner> { - pub fn new(parser_go_scanner: parser_go_scanner::Scanner<'parser_go_scanner>) -> Self { - Self { - steps: parser_go_scanner.into_iter(), - current_step: (Position::default(), Token::EOF, ""), - expr_level: -1, - } - } - - // SourceFile = PackageClause ";" { ImportDecl ";" } { TopLevelDecl ";" } . - fn SourceFile(&mut self) -> Result>> { - let mut out = parser_go_ast::File { decls: vec![] }; - - while let Some(top_level_decl) = self.TopLevelDecl()? { - self.token(Token::SEMICOLON).required()?; - out.decls.push(top_level_decl); - } - - self.token(Token::EOF).required()?; - - Ok(Some(out)) - } - - // TopLevelDecl = Declaration | FunctionDecl | MethodDecl . - fn TopLevelDecl(&mut self) -> Result>> { - use Token::*; - Ok(match self.current_step.1 { - FUNC => Some(parser_go_ast::Decl::FuncDecl( - self.FunctionDecl_or_MethodDecl().required()?, - )), - _ => None, - }) - } - - // IdentifierList = identifier { "," identifier } . - fn IdentifierList(&mut self) -> Result>>> { - let mut out = match self.identifier()? { - Some(v) => vec![v], - None => return Ok(None), - }; - - while self.token(Token::COMMA)?.is_some() { - out.push(self.identifier().required()?); - } - - Ok(Some(out)) - } - - // ExpressionList = Expression { "," Expression } . - fn ExpressionList(&mut self) -> Result>>> { - let mut out = match self.Expression()? { - Some(v) => vec![v], - None => return Ok(None), - }; - - while self.token(Token::COMMA)?.is_some() { - out.push(self.Expression().required()?); - } - - Ok(Some(out)) - } - - // Expression = UnaryExpr | Expression binary_op Expression . - fn Expression(&mut self) -> Result>> { - let unary_expr = match self.UnaryExpr()? { - Some(v) => v, - None => return Ok(None), - }; - - self.expression(unary_expr, Token::lowest_precedence()) - } - - // https://en.wikipedia.org/wiki/Operator-precedence_parser - fn expression( - &mut self, - mut lhs: parser_go_ast::Expr<'parser_go_scanner>, - min_precedence: u8, - ) -> Result>> { - while let Some(op) = self.get_binary_op(min_precedence)? { - self.next()?; - - let mut rhs = self.UnaryExpr().required()?; - while self.get_binary_op(op.1.precedence() + 1)?.is_some() { - rhs = self.expression(rhs, op.1.precedence() + 1).required()?; - } - - lhs = parser_go_ast::Expr::BinaryExpr(parser_go_ast::BinaryExpr { - x: Box::new(lhs), - op_pos: op.0, - op: op.1, - y: Box::new(rhs), - }); - } - - Ok(Some(lhs)) - } - - // UnaryExpr = PrimaryExpr | unary_op UnaryExpr . - fn UnaryExpr(&mut self) -> Result>> { - if let Some(op) = self.unary_op()? { - let x = Box::new(self.UnaryExpr().required()?); - let expr = if op.1 == Token::MUL { - parser_go_ast::Expr::StarExpr(parser_go_ast::StarExpr { star: op.0, x }) - } else { - parser_go_ast::Expr::UnaryExpr(parser_go_ast::UnaryExpr { - op: op.1, - op_pos: op.0, - x, - }) - }; - return Ok(Some(expr)); - } - - self.PrimaryExpr() - } - - // PrimaryExpr = - // Operand | - // Conversion | - // MethodExpr | - // PrimaryExpr Selector | - // PrimaryExpr Index | - // PrimaryExpr Slice | - // PrimaryExpr TypeAssertion | - // PrimaryExpr Arguments . - fn PrimaryExpr(&mut self) -> Result>> { - let mut primary_expr = match self.Operand()? { - Some(v) => v, - None => return Ok(None), - }; - - loop { - match self.current_step.1 { - Token::PERIOD => { - primary_expr = self.Selector_or_TypeAssertion(primary_expr).required()?; - } - Token::LBRACK => { - primary_expr = self.Index_or_Slice(primary_expr).required()?; - } - Token::LPAREN => { - primary_expr = self.Arguments(primary_expr).required()?; - } - Token::LBRACE if self.expr_level >= 0 => { - unimplemented!("composite literal"); - } - _ => break, - } - } - - Ok(Some(primary_expr)) - } - - // Selector = "." identifier . - // TypeAssertion = "." "(" Type ")" . - fn Selector_or_TypeAssertion( - &mut self, - x: parser_go_ast::Expr<'parser_go_scanner>, - ) -> Result>> { - if self.token(Token::PERIOD)?.is_none() { - return Ok(None); - } - - if let Some(lparen) = self.token(Token::LPAREN)? { - let type_ = self.Type().required()?; - let rparen = self.token(Token::RPAREN).required()?; - return Ok(Some(parser_go_ast::Expr::TypeAssertExpr( - parser_go_ast::TypeAssertExpr { - x: Box::new(x), - lparen: lparen.0, - type_: Box::new(type_), - rparen: rparen.0, - }, - ))); - } - - Ok(Some(parser_go_ast::Expr::SelectorExpr( - parser_go_ast::SelectorExpr { x: Box::new(x), sel: self.identifier().required()? }, - ))) - } - - // Index = "[" Expression "]" . - // Slice = "[" [ Expression ] ":" [ Expression ] "]" | - // "[" [ Expression ] ":" Expression ":" Expression "]" . - fn Index_or_Slice( - &mut self, - x: parser_go_ast::Expr<'parser_go_scanner>, - ) -> Result>> { - let lbrack = match self.token(Token::LBRACK)? { - Some(v) => v, - None => return Ok(None), - }; - - let low = if let Some(low) = self.Expression()? { - if let Some(rbrack) = self.token(Token::RBRACK)? { - return Ok(Some(parser_go_ast::Expr::IndexExpr( - parser_go_ast::IndexExpr { - x: Box::new(x), - lbrack: lbrack.0, - index: Box::new(low), - rbrack: rbrack.0, - }, - ))); - } - Some(low) - } else { - None - }; - - self.token(Token::COLON).required()?; - - let high = if let Some(high) = self.Expression()? { - if self.token(Token::COLON)?.is_some() { - let max = self.Expression().required()?; - let rbrack = self.token(Token::RBRACK).required()?; - return Ok(Some(parser_go_ast::Expr::SliceExpr( - parser_go_ast::SliceExpr { - x: Box::new(x), - lbrack: lbrack.0, - low: low.map(Box::new), - high: Some(Box::new(high)), - max: Some(Box::new(max)), - slice3: true, - rbrack: rbrack.0, - }, - ))); - } - Some(high) - } else { - None - }; - let rbrack = self.token(Token::RBRACK).required()?; - - Ok(Some(parser_go_ast::Expr::SliceExpr( - parser_go_ast::SliceExpr { - x: Box::new(x), - lbrack: lbrack.0, - low: low.map(Box::new), - high: high.map(Box::new), - max: None, - slice3: false, - rbrack: rbrack.0, - }, - ))) - } - - // Arguments = "(" [ ( ExpressionList | Type [ "," ExpressionList ] ) [ "..." ] [ "," ] ] ")" . - fn Arguments( - &mut self, - x: parser_go_ast::Expr<'parser_go_scanner>, - ) -> Result>> { - let lparen = match self.token(Token::LPAREN)? { - Some(v) => v, - None => return Ok(None), - }; - - let mut args = if let Some(exprs) = self.ExpressionList()? { - exprs - } else if let Some(type_) = self.Type()? { - vec![type_] - } else { - vec![] - }; - - if self.token(Token::COMMA)?.is_some() { - let mut exprs = self.ExpressionList().required()?; - args.append(&mut exprs); - } - - let ellipsis = if !args.is_empty() { - let ellipsis = self.token(Token::ELLIPSIS)?; - self.token(Token::COMMA)?; - ellipsis - } else { - None - }; - - let rparen = self.token(Token::RPAREN).required()?; - - Ok(Some(parser_go_ast::Expr::CallExpr( - parser_go_ast::CallExpr { - fun: Box::new(x), - lparen: lparen.0, - args: Some(args), - ellipsis: ellipsis.map(|(pos, _, _)| pos), - rparen: rparen.0, - }, - ))) - } - - // Operand = Literal | OperandName | "(" Expression ")" . - // Literal = BasicLit | CompositeLit | FunctionLit . - // OperandName = identifier | QualifiedIdent . - fn Operand(&mut self) -> Result>> { - use Token::*; - Ok(match self.current_step.1 { - IDENT => Some(parser_go_ast::Expr::Ident(self.identifier().required()?)), - INT | FLOAT | IMAG | CHAR | STRING => { - Some(parser_go_ast::Expr::BasicLit(self.BasicLit().required()?)) - } - LPAREN => { - let lparen = self.token(Token::LPAREN).required()?; - let expr = self.Expression().required()?; - let rparen = self.token(Token::RPAREN).required()?; - return Ok(Some(parser_go_ast::Expr::ParenExpr( - parser_go_ast::ParenExpr { - lparen: lparen.0, - x: Box::new(expr), - rparen: rparen.0, - }, - ))); - } - FUNC => Some(parser_go_ast::Expr::FuncLit(self.FunctionLit().required()?)), - _ => self.CompositeLit()?.map(parser_go_ast::Expr::CompositeLit), - }) - } - - // CompositeLit = LiteralType LiteralValue . - // LiteralValue = "{" [ ElementList [ "," ] ] "}" . - // ElementList = KeyedElement { "," KeyedElement } . - fn CompositeLit(&mut self) -> Result>> { - let type_ = match self.LiteralType()? { - Some(v) => v, - None => return Ok(None), - }; - - let lbrace = self.token(Token::LBRACE).required()?; - - let mut elts = self.KeyedElement()?.map(|elt| vec![elt]); - if let Some(elts) = elts.as_mut() { - while self.token(Token::COMMA)?.is_some() { - if let Some(k) = self.KeyedElement()? { - elts.push(k); - } else { - break; - } - } - } - - let rbrace = self.token(Token::RBRACE).required()?; - - Ok(Some(parser_go_ast::CompositeLit { - type_: Box::new(type_), - lbrace: lbrace.0, - elts, - rbrace: rbrace.0, - incomplete: false, - })) - } - - // LiteralType = StructType | ArrayType | "[" "..." "]" ElementType | - // SliceType | MapType | TypeName . - fn LiteralType(&mut self) -> Result>> { - Ok(match self.current_step.1 { - Token::STRUCT => Some(parser_go_ast::Expr::StructType( - self.StructType().required()?, - )), - Token::LBRACK => Some(parser_go_ast::Expr::ArrayType( - self.ArrayType_or_SliceType::().required()?, - )), - Token::MAP => Some(parser_go_ast::Expr::MapType(self.MapType().required()?)), - Token::IDENT => Some(self.TypeName().required()?), - _ => None, - }) - } - - // KeyedElement = [ Key ":" ] Element . - // Key = FieldName | Expression | LiteralValue . - // FieldName = identifier . - // Element = Expression | LiteralValue . - fn KeyedElement(&mut self) -> Result>> { - let key = match self.Expression()? { - Some(v) => v, - None => return Ok(None), - }; - - if let Some(colon) = self.token(Token::COLON)? { - let value = self.Expression().required()?; - return Ok(Some(parser_go_ast::Expr::KeyValueExpr( - parser_go_ast::KeyValueExpr { - key: Box::new(key), - colon: colon.0, - value: Box::new(value), - }, - ))); - } - - Ok(Some(key)) - } - - // FunctionLit = "func" Signature FunctionBody . - fn FunctionLit(&mut self) -> Result>> { - let func = match self.token(Token::FUNC)? { - Some(v) => v, - None => return Ok(None), - }; - let type_ = self.Signature(Some(func.0)).required()?; - let body = self.FunctionBody().required()?; - - Ok(Some(parser_go_ast::FuncLit { type_, body })) - } - - // BasicLit = int_lit | float_lit | imaginary_lit | rune_lit | string_lit . - fn BasicLit(&mut self) -> Result>> { - Ok(match self.current_step.1 { - Token::INT => Some(self.int_lit().required()?), - Token::FLOAT => Some(self.float_lit().required()?), - Token::IMAG => Some(self.imaginary_lit().required()?), - Token::CHAR => Some(self.rune_lit().required()?), - Token::STRING => Some(self.string_lit().required()?), - _ => None, - }) - } - - // Type = TypeName | TypeLit | "(" Type ")" . - fn Type(&mut self) -> Result>> { - if self.token(Token::LPAREN)?.is_some() { - let type_ = self.Type().required()?; - self.token(Token::RPAREN).required()?; - return Ok(Some(type_)); - } - - if let Some(type_name) = self.TypeName()? { - return Ok(Some(type_name)); - } - - if let Some(type_lit) = self.TypeLit()? { - return Ok(Some(type_lit)); - } - - Ok(None) - } - - // TypeName = identifier | QualifiedIdent . - fn TypeName(&mut self) -> Result>> { - self.identifier_or_QualifiedIdent() - } - - // TypeLit = ArrayType | StructType | PointerType | FunctionType | InterfaceType | - // SliceType | MapType | ChannelType . - fn TypeLit(&mut self) -> Result>> { - Ok(match self.current_step.1 { - Token::LBRACK => Some(parser_go_ast::Expr::ArrayType( - self.ArrayType_or_SliceType::().required()?, - )), - Token::STRUCT => Some(parser_go_ast::Expr::StructType( - self.StructType().required()?, - )), - Token::MUL => Some(parser_go_ast::Expr::StarExpr( - self.PointerType().required()?, - )), - // TODO: FunctionType - Token::INTERFACE => Some(parser_go_ast::Expr::InterfaceType( - self.InterfaceType().required()?, - )), - Token::MAP => Some(parser_go_ast::Expr::MapType(self.MapType().required()?)), - Token::CHAN => Some(parser_go_ast::Expr::ChanType( - self.ChannelType().required()?, - )), - _ => None, - }) - } - - // ArrayType = "[" ArrayLength "]" ElementType . - // ArrayLength = Expression . - // SliceType = "[" "]" ElementType . - fn ArrayType_or_SliceType( - &mut self, - ) -> Result>> { - let lbrack = match self.token(Token::LBRACK)? { - Some(v) => v, - None => return Ok(None), - }; - - let len = if ELLIPSIS { - if let Some(ellipsis) = self.token(Token::ELLIPSIS)? { - Some(parser_go_ast::Expr::Ellipsis(parser_go_ast::Ellipsis { - ellipsis: ellipsis.0, - elt: None, - })) - } else { - self.Expression()? - } - } else { - self.Expression()? - }; - - self.token(Token::RBRACK).required()?; - - let element_type = self.ElementType().required()?; - - Ok(Some(parser_go_ast::ArrayType { - lbrack: lbrack.0, - len: len.map(Box::new), - elt: Box::new(element_type), - })) - } - - // MapType = "map" "[" KeyType "]" ElementType . - fn MapType(&mut self) -> Result>> { - let map = match self.token(Token::MAP)? { - Some(v) => v, - None => return Ok(None), - }; - self.token(Token::LBRACK).required()?; - let key_type = self.KeyType().required()?; - self.token(Token::RBRACK).required()?; - let element_type = self.ElementType().required()?; - - Ok(Some(parser_go_ast::MapType { - map: map.0, - key: Box::new(key_type), - value: Box::new(element_type), - })) - } - - // KeyType = Type . - fn KeyType(&mut self) -> Result>> { - self.Type() - } - - // ChannelType = ( "chan" | "chan" "<-" | "<-" "chan" ) ElementType . - fn ChannelType(&mut self) -> Result>> { - if let Some(chan) = self.token(Token::CHAN)? { - if let Some(arrow) = self.token(Token::ARROW)? { - let value = Box::new(self.ElementType().required()?); - return Ok(Some(parser_go_ast::ChanType { - begin: chan.0, - arrow: Some(arrow.0), - dir: parser_go_ast::ChanDir::SEND as u8, - value, - })); - } - - let value = Box::new(self.ElementType().required()?); - return Ok(Some(parser_go_ast::ChanType { - begin: chan.0, - arrow: None, - dir: parser_go_ast::ChanDir::SEND as u8 | parser_go_ast::ChanDir::RECV as u8, - value, - })); - } - - if let Some(arrow) = self.token(Token::ARROW)? { - self.token(Token::CHAN).required()?; - let value = Box::new(self.ElementType().required()?); - return Ok(Some(parser_go_ast::ChanType { - begin: arrow.0, - arrow: None, - dir: parser_go_ast::ChanDir::RECV as u8, - value, - })); - } - - Ok(None) - } - - // ElementType = Type . - fn ElementType(&mut self) -> Result>> { - self.Type() - } - - // PointerType = "*" BaseType . - fn PointerType(&mut self) -> Result>> { - let star = match self.token(Token::MUL)? { - Some(v) => v, - None => return Ok(None), - }; - - let x = Box::new(self.BaseType().required()?); - Ok(Some(parser_go_ast::StarExpr { star: star.0, x })) - } - - // BaseType = Type . - fn BaseType(&mut self) -> Result>> { - self.Type() - } - - // InterfaceType = "interface" "{" { ( MethodSpec | InterfaceTypeName ) ";" } "}" . - // MethodSpec = MethodName Signature . - fn InterfaceType( - &mut self, - ) -> Result>> { - let interface = match self.token(Token::INTERFACE)? { - Some(v) => v, - None => return Ok(None), - }; - - let lbrace = self.token(Token::LBRACE).required()?; - - let mut fields = vec![]; - loop { - if let Some(method_spec) = self.MethodName()? { - if let Some(signature) = self.Signature(None)? { - self.token(Token::SEMICOLON).required()?; - fields.push(parser_go_ast::Field { - doc: None, - names: Some(vec![method_spec]), - type_: Some(parser_go_ast::Expr::FuncType(signature)), - tag: None, - comment: None, - }); - continue; - } - - fields.push(parser_go_ast::Field { - doc: None, - names: None, - type_: Some(parser_go_ast::Expr::Ident(method_spec)), - tag: None, - comment: None, - }); - if self.token(Token::SEMICOLON)?.is_none() { - break; - } - continue; - }; - - if let Some(interface_type_name) = self.InterfaceTypeName()? { - fields.push(parser_go_ast::Field { - doc: None, - names: None, - type_: Some(interface_type_name), - tag: None, - comment: None, - }); - if self.token(Token::SEMICOLON)?.is_none() { - break; - } - continue; - } - - break; - } - - let rbrace = self.token(Token::RBRACE).required()?; - - Ok(Some(parser_go_ast::InterfaceType { - interface: interface.0, - methods: Some(parser_go_ast::FieldList { - opening: Some(lbrace.0), - list: fields, - closing: Some(rbrace.0), - }), - incomplete: false, - })) - } - - // MethodName = identifier . - fn MethodName(&mut self) -> Result>> { - self.identifier() - } - - // InterfaceTypeName = TypeName . - fn InterfaceTypeName(&mut self) -> Result>> { - self.TypeName() - } - - // StructType = "struct" "{" { FieldDecl ";" } "}" . - fn StructType(&mut self) -> Result>> { - let struct_ = match self.token(Token::STRUCT)? { - Some(v) => v, - None => return Ok(None), - }; - - let lbrace = self.token(Token::LBRACE).required()?; - - let mut fields = vec![]; - while let Some(field_decl) = self.FieldDecl()? { - fields.push(field_decl); - if self.token(Token::SEMICOLON)?.is_none() { - break; - } - } - - let rbrace = self.token(Token::RBRACE).required()?; - - Ok(Some(parser_go_ast::StructType { - struct_: struct_.0, - fields: Some(parser_go_ast::FieldList { - opening: Some(lbrace.0), - list: fields, - closing: Some(rbrace.0), - }), - incomplete: false, - })) - } - - // FieldDecl = (IdentifierList Type | EmbeddedField) [ Tag ] . - // EmbeddedField = [ "*" ] TypeName . - fn FieldDecl(&mut self) -> Result>> { - if let Some(star) = self.token(Token::MUL)? { - let type_name = Box::new(self.TypeName().required()?); - let tag = self.Tag()?; - return Ok(Some(parser_go_ast::Field { - doc: None, - type_: Some(parser_go_ast::Expr::StarExpr(parser_go_ast::StarExpr { - star: star.0, - x: type_name, - })), - names: None, - tag, - comment: None, - })); - }; - - if let Some(names) = self.IdentifierList()? { - if let Some(type_) = self.Type()? { - let tag = self.Tag()?; - return Ok(Some(parser_go_ast::Field { - doc: None, - names: Some(names), - type_: Some(type_), - tag, - comment: None, - })); - } - - if names.len() == 1 { - let name = names.into_iter().next().unwrap(); - let tag = self.Tag()?; - return Ok(Some(parser_go_ast::Field { - doc: None, - type_: Some(parser_go_ast::Expr::Ident(name)), - names: None, - tag, - comment: None, - })); - } - - return Err(ParserError::UnexpectedToken); - } - - if let Some(type_) = self.TypeName()? { - let tag = self.Tag()?; - return Ok(Some(parser_go_ast::Field { - doc: None, - type_: Some(type_), - names: None, - tag, - comment: None, - })); - } - - Ok(None) - } - - // Tag = string_lit . - fn Tag(&mut self) -> Result>> { - self.string_lit() - } - - // Signature = Parameters [ Result ] . - fn Signature( - &mut self, - func: Option>, - ) -> Result>> { - let params = match self.Parameters()? { - Some(v) => v, - None => return Ok(None), - }; - let results = self.Result()?; - - Ok(Some(parser_go_ast::FuncType { func, params, results })) - } - - // Result = Parameters | Type . - fn Result(&mut self) -> Result>> { - if let Some(parameters) = self.Parameters()? { - Ok(Some(parameters)) - } else if let Some(type_) = self.Type()? { - Ok(Some(parser_go_ast::FieldList { - opening: None, - list: vec![parser_go_ast::Field { - doc: None, - names: None, - tag: None, - type_: Some(type_), - comment: None, - }], - closing: None, - })) - } else { - Ok(None) - } - } - - // Parameters = "(" [ ParameterList [ "," ] ] ")" . - fn Parameters(&mut self) -> Result>> { - let lparen = match self.token(Token::LPAREN)? { - Some(v) => v, - None => return Ok(None), - }; - let list = self - .ParameterList()? - .map(|list| { - let _ = self.token(Token::COMMA); - list - }) - .unwrap_or_default(); - let rparen = self.token(Token::RPAREN).required()?; - - Ok(Some(parser_go_ast::FieldList { - opening: Some(lparen.0), - list, - closing: Some(rparen.0), - })) - } - - // ParameterList = ParameterDecl { "," ParameterDecl } . - // ParameterDecl = [ IdentifierList ] [ "..." ] Type . - fn ParameterList(&mut self) -> Result>>> { - let idents = match self.IdentifierList()? { - Some(v) => v, - None => return Ok(None), - }; - let type_ = self.Type()?; - - // If no type can be found, then the idents are types, e.g.: (bool, bool) - if type_.is_none() { - return Ok(Some( - idents - .into_iter() - .map(|ident| parser_go_ast::Field { - doc: None, - names: None, - type_: Some(parser_go_ast::Expr::Ident(ident)), - tag: None, - comment: None, - }) - .collect(), - )); - } - - // If a type can be found, then we expect idents + types: (a, b bool, c bool, d bool) - - let mut fields = vec![parser_go_ast::Field { - comment: None, - type_, - tag: None, - names: Some(idents), - doc: None, - }]; - - while self.token(Token::COMMA)?.is_some() { - let idents = self.IdentifierList().required()?; - let ellipsis = self.token(Token::ELLIPSIS)?; - let type_ = self.Type().required()?; - - if let Some(ellipsis) = ellipsis { - fields.push(parser_go_ast::Field { - comment: None, - type_: Some(parser_go_ast::Expr::Ellipsis(parser_go_ast::Ellipsis { - ellipsis: ellipsis.0, - elt: Some(Box::new(type_)), - })), - tag: None, - names: Some(idents), - doc: None, - }); - return Ok(Some(fields)); - } - - fields.push(parser_go_ast::Field { - comment: None, - type_: Some(type_), - tag: None, - names: Some(idents), - doc: None, - }); - } - - Ok(Some(fields)) - } - - // FunctionBody = Block . - fn FunctionBody(&mut self) -> Result>> { - self.Block() - } - - // Block = "{" StatementList "}" . - // StatementList = { Statement ";" } . - fn Block(&mut self) -> Result>> { - let lbrace = match self.token(Token::LBRACE)? { - Some(v) => v, - None => return Ok(None), - }; - - let list = vec![]; - - let rbrace = self.token(Token::RBRACE).required()?; - - Ok(Some(parser_go_ast::BlockStmt { - lbrace: lbrace.0, - list, - rbrace: rbrace.0, - })) - } - - // Receiver = Parameters . - fn Receiver(&mut self) -> Result>> { - self.Parameters() - } - - // identifier | QualifiedIdent - // QualifiedIdent = PackageName "." identifier . - // PackageName = identifier . - fn identifier_or_QualifiedIdent( - &mut self, - ) -> Result>> { - let ident = match self.identifier()? { - Some(v) => v, - None => return Ok(None), - }; - - if self.token(Token::PERIOD)?.is_some() { - let sel = self.identifier().required()?; - return Ok(Some(parser_go_ast::Expr::SelectorExpr( - parser_go_ast::SelectorExpr { x: Box::new(parser_go_ast::Expr::Ident(ident)), sel }, - ))); - } - - Ok(Some(parser_go_ast::Expr::Ident(ident))) - } - - // FunctionDecl | MethodDecl - // FunctionDecl = "func" FunctionName Signature [ FunctionBody ] . - // MethodDecl = "func" Receiver MethodName Signature [ FunctionBody ] . - // FunctionName = identifier . - // MethodName = identifier . - fn FunctionDecl_or_MethodDecl( - &mut self, - ) -> Result>> { - let func = match self.token(Token::FUNC)? { - Some(v) => v, - None => return Ok(None), - }; - let recv = self.Receiver()?; - let name = self.identifier().required()?; - let type_ = self.Signature(Some(func.0)).required()?; - let body = self.FunctionBody()?; - - Ok(Some(parser_go_ast::FuncDecl { - doc: None, - recv, - name, - type_, - body, - })) - } - - // unary_op = "+" | "-" | "!" | "^" | "*" | "&" | "<-" . - fn unary_op(&mut self) -> Result>> { - use Token::*; - Ok(match self.current_step { - step @ (_, ADD | SUB | NOT | MUL | XOR | AND | ARROW, _) => { - self.next()?; - Some(step) - } - _ => None, - }) - } - - // binary_op = "||" | "&&" | rel_op | add_op | mul_op . - // rel_op = "==" | "!=" | "<" | "<=" | ">" | ">=" . - // add_op = "+" | "-" | "|" | "^" . - // mul_op = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" . - fn get_binary_op( - &mut self, - min_precedence: u8, - ) -> Result>> { - use Token::*; - Ok(match self.current_step { - step @ (_, - /* binary_op */ - LOR | LAND | - /* rel_op */ - EQL | NEQ | LSS | LEQ | GTR | GEQ | - /* add_op */ - ADD | SUB | OR | XOR | - /* mul_op */ - MUL | QUO | REM | SHL | SHR | AND | AND_NOT - , _) if step.1.precedence() >= min_precedence => { - Some(step) - } - _ => None, - }) - } - - fn identifier(&mut self) -> Result>> { - self.token(Token::IDENT)? - .map_or(Ok(None), |(name_pos, _, name)| { - Ok(Some(parser_go_ast::Ident { name_pos, name, obj: None })) - }) - } - - fn int_lit(&mut self) -> Result>> { - self.token(Token::INT)? - .map_or(Ok(None), |(value_pos, kind, value)| { - Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) - }) - } - - fn float_lit(&mut self) -> Result>> { - self.token(Token::FLOAT)? - .map_or(Ok(None), |(value_pos, kind, value)| { - Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) - }) - } - - fn imaginary_lit(&mut self) -> Result>> { - self.token(Token::IMAG)? - .map_or(Ok(None), |(value_pos, kind, value)| { - Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) - }) - } - - fn rune_lit(&mut self) -> Result>> { - self.token(Token::CHAR)? - .map_or(Ok(None), |(value_pos, kind, value)| { - Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) - }) - } - - fn string_lit(&mut self) -> Result>> { - self.token(Token::STRING)? - .map_or(Ok(None), |(value_pos, kind, value)| { - Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) - }) - } - - /// Returns the current step and advances to the next one, but only if it matches the expected - /// token. [`Parser::next`] is automatically called for you. - fn token( - &mut self, - expected: Token, - ) -> Result>> { - Ok(match self.current_step { - step @ (_, tok, _) if tok == expected => { - if expected != Token::EOF { - self.next()?; - } - Some(step) - } - _ => None, - }) - } - - /// Advances to the next token. Skips all the comment tokens. - fn next(&mut self) -> Result<()> { - if let Some(step) = self - .steps - .find(|step| !matches!(step, Ok((_, Token::COMMENT, _)))) - { - self.current_step = step?; - return Ok(()); - } - Err(ParserError::UnexpectedEndOfFile) - } -} diff --git a/backend/parsers/windmill-parser-go/src/parser_go_ast.rs b/backend/parsers/windmill-parser-go/src/parser_go_ast.rs deleted file mode 100644 index cf813f4dac..0000000000 --- a/backend/parsers/windmill-parser-go/src/parser_go_ast.rs +++ /dev/null @@ -1,347 +0,0 @@ -#![allow(clippy::large_enum_variant)] // TODO: we allow large enum variant for now, let's profile properly to see if we want to box. - -use crate::parser_go_token::{Position, Token}; -use std::collections::BTreeMap; - -// https://pkg.go.dev/go/ast#CommentGroup -#[derive(Debug)] -pub struct CommentGroup { - // List []*Comment // len(List) > 0 -} - -// https://pkg.go.dev/go/ast#FieldList -#[derive(Debug)] -pub struct FieldList<'a> { - pub opening: Option>, // position of opening parenthesis/brace, if any - pub list: Vec>, // field list; or nil - pub closing: Option>, // position of closing parenthesis/brace, if any -} - -// https://pkg.go.dev/go/ast#Field -#[derive(Debug)] -pub struct Field<'a> { - pub doc: Option, // associated documentation; or nil - pub names: Option>>, // field/method/(type) parameter names, or type "type"; or nil - pub type_: Option>, // field/method/parameter type, type list type; or nil - pub tag: Option>, // field tag; or nil - pub comment: Option, // line comments; or nil -} - -// https://pkg.go.dev/go/ast#File -#[derive(Debug)] -pub struct File<'a> { - // package name - pub decls: Vec>, // top-level declarations; or nil // list of all comments in the source file -} - -// https://pkg.go.dev/go/ast#FuncDecl -#[derive(Debug)] -pub struct FuncDecl<'a> { - pub doc: Option, // associated documentation; or nil - pub recv: Option>, // receiver (methods); or nil (functions) - pub name: Ident<'a>, // function/method name - pub type_: FuncType<'a>, // function signature: type and value parameters, results, and position of "func" keyword - pub body: Option>, // function body; or nil for external (non-Go) function -} - -// https://pkg.go.dev/go/ast#BlockStmt -#[derive(Debug)] -pub struct BlockStmt<'a> { - pub lbrace: Position<'a>, // position of "{" - pub list: Vec, - pub rbrace: Position<'a>, // position of "}", if any (may be absent due to syntax error) -} - -// https://pkg.go.dev/go/ast#FuncType -#[derive(Debug)] -pub struct FuncType<'a> { - pub func: Option>, // position of "func" keyword (token.NoPos if there is no "func") - pub params: FieldList<'a>, // (incoming) parameters; non-nil - pub results: Option>, // (outgoing) results; or nil -} - -// https://pkg.go.dev/go/ast#Ident -#[derive(Debug)] -pub struct Ident<'a> { - pub name_pos: Position<'a>, // identifier position - pub name: &'a str, // identifier name - pub obj: Option>>, // denoted object; or nil -} - -// https://pkg.go.dev/go/ast#ValueSpec -#[derive(Debug)] -pub struct ValueSpec<'a> { - pub doc: Option, // associated documentation; or nil - pub names: Vec>, // value names (len(Names) > 0) - pub type_: Option>, // value type; or nil - pub values: Option>>, // initial values; or nil - pub comment: Option, // line comments; or nil -} - -// https://pkg.go.dev/go/ast#BasicLit -#[derive(Debug)] -pub struct BasicLit<'a> { - pub value_pos: Position<'a>, // literal position - pub kind: Token, // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING - pub value: &'a str, // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` -} - -// https://pkg.go.dev/go/ast#Object -#[derive(Debug)] -pub struct Object<'a> { - pub kind: ObjKind, - pub name: &'a str, // declared name - pub decl: Option, // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil - pub data: Option, // object-specific data; or nil - pub type_: Option<()>, // placeholder for type information; may be nil -} - -// https://pkg.go.dev/go/ast#Ellipsis -#[derive(Debug)] -pub struct Ellipsis<'a> { - pub ellipsis: Position<'a>, // position of "..." - pub elt: Option>>, // ellipsis element type (parameter lists only); or nil -} - -// https://pkg.go.dev/go/ast#Ellipsis -#[derive(Debug)] -pub struct TypeAssertExpr<'a> { - pub x: Box>, // expression - pub lparen: Position<'a>, // position of "(" - pub type_: Box>, // asserted type; nil means type switch X.(type) - pub rparen: Position<'a>, // position of ")" -} - -// https://pkg.go.dev/go/ast#SliceExpr -#[derive(Debug)] -pub struct SliceExpr<'a> { - pub x: Box>, // expression - pub lbrack: Position<'a>, // position of "[" - pub low: Option>>, // begin of slice range; or nil - pub high: Option>>, // end of slice range; or nil - pub max: Option>>, // maximum capacity of slice; or nil - pub slice3: bool, // true if 3-index slice (2 colons present) - pub rbrack: Position<'a>, // position of "]" -} - -// https://pkg.go.dev/go/ast#ObjKind -#[derive(Debug)] -pub enum ObjKind {} - -#[derive(Debug)] -pub enum ObjDecl {} - -// https://pkg.go.dev/go/ast#Decl -#[derive(Debug)] -pub enum Decl<'a> { - FuncDecl(FuncDecl<'a>), -} - -// https://pkg.go.dev/go/ast#Scope -#[derive(Debug)] -pub struct Scope<'a> { - pub outer: Option>>, - pub objects: BTreeMap<&'a str, Object<'a>>, -} - -// https://pkg.go.dev/go/ast#GenDecl -#[derive(Debug)] -pub struct GenDecl<'a> { - pub doc: Option, // associated documentation; or nil - pub tok_pos: Position<'a>, // position of Tok - pub tok: Token, // IMPORT, CONST, TYPE, or VAR - pub lparen: Option>, // position of '(', if any - pub specs: Vec, - pub rparen: Option>, // position of ')', if any -} - -// https://pkg.go.dev/go/ast#AssignStmt -#[derive(Debug)] -pub struct AssignStmt<'a> { - pub lhs: Vec>, - pub tok_pos: Position<'a>, // position of Tok - pub tok: Token, // assignment token, DEFINE - pub rhs: Vec>, -} - -// https://pkg.go.dev/go/ast#BinaryExpr -#[derive(Debug)] -pub struct BinaryExpr<'a> { - pub x: Box>, // left operand - pub op_pos: Position<'a>, // position of Op - pub op: Token, // operator - pub y: Box>, // right operand -} - -// https://pkg.go.dev/go/ast#ReturnStmt -#[derive(Debug)] -pub struct ReturnStmt<'a> { - pub return_: Position<'a>, // position of "return" keyword - pub results: Vec>, // result expressions; or nil -} - -// https://pkg.go.dev/go/ast#TypeSpec -#[derive(Debug)] -pub struct TypeSpec<'a> { - pub doc: Option, // associated documentation; or nil - pub name: Option>, // type name - pub assign: Option>, // position of '=', if any - pub type_: Expr<'a>, // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes - pub comment: Option, // line comments; or nil -} - -// https://pkg.go.dev/go/ast#StructType -#[derive(Debug)] -pub struct StructType<'a> { - pub struct_: Position<'a>, // position of "struct" keyword - pub fields: Option>, // list of field declarations - pub incomplete: bool, // true if (source) fields are missing in the Fields list -} - -// https://pkg.go.dev/go/ast#StarExpr -#[derive(Debug)] -pub struct StarExpr<'a> { - pub star: Position<'a>, // position of "*" - pub x: Box>, // operand -} - -// https://pkg.go.dev/go/ast#InterfaceType -#[derive(Debug)] -pub struct InterfaceType<'a> { - pub interface: Position<'a>, // position of "interface" keyword - pub methods: Option>, // list of embedded interfaces, methods, or types - pub incomplete: bool, // true if (source) methods or types are missing in the Methods list -} - -// https://pkg.go.dev/go/ast#UnaryExpr -#[derive(Debug)] -pub struct UnaryExpr<'a> { - pub op_pos: Position<'a>, // position of Op - pub op: Token, // operator - pub x: Box>, // operand -} - -// https://pkg.go.dev/go/ast#CallExpr -#[derive(Debug)] -pub struct CallExpr<'a> { - pub fun: Box>, // function expression - pub lparen: Position<'a>, // position of "(" - pub args: Option>>, // function arguments; or nil - pub ellipsis: Option>, // position of "..." (token.NoPos if there is no "...") - pub rparen: Position<'a>, // position of ")" -} - -// https://pkg.go.dev/go/ast#SelectorExpr -#[derive(Debug)] -pub struct SelectorExpr<'a> { - pub x: Box>, // expression - pub sel: Ident<'a>, // field selector -} - -// https://pkg.go.dev/go/ast#ParenExpr -#[derive(Debug)] -pub struct ParenExpr<'a> { - pub lparen: Position<'a>, // position of "(" - pub x: Box>, // parenthesized expression - pub rparen: Position<'a>, // position of ")" -} - -// https://pkg.go.dev/go/ast#FuncLit -#[derive(Debug)] -pub struct FuncLit<'a> { - pub type_: FuncType<'a>, // function type - pub body: BlockStmt<'a>, // function body -} - -// https://pkg.go.dev/go/ast#ChanType -#[derive(Debug)] -pub struct ChanType<'a> { - pub begin: Position<'a>, // position of "chan" keyword or "<-" (whichever comes first) - pub arrow: Option>, // position of "<-" (token.NoPos if there is no "<-") - pub dir: u8, // channel direction - pub value: Box>, // value type -} - -// htt/opt/visual-studio-code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.htmlps://pkg.go.dev/go/ast#IndexExpr -#[derive(Debug)] -pub struct IndexExpr<'a> { - pub x: Box>, // expression - pub lbrack: Position<'a>, // position of "[" - pub index: Box>, // index expression - pub rbrack: Position<'a>, // position of "]" -} - -// https://pkg.go.dev/go/ast#MapType -#[derive(Debug)] -pub struct MapType<'a> { - pub map: Position<'a>, - pub key: Box>, - pub value: Box>, -} - -// https://pkg.go.dev/go/ast#CompositeLit -#[derive(Debug)] -pub struct CompositeLit<'a> { - pub type_: Box>, // literal type; or nil - pub lbrace: Position<'a>, // position of "{" - pub elts: Option>>, // list of composite elements; or nil - pub rbrace: Position<'a>, // position of "}" - pub incomplete: bool, // true if (source) expressions are missing in the Elts list -} - -// https://pkg.go.dev/go/ast#KeyValueExpr -#[derive(Debug)] -pub struct KeyValueExpr<'a> { - pub key: Box>, - pub colon: Position<'a>, // position of ":" - pub value: Box>, -} - -// https://pkg.go.dev/go/ast#ArrayType -#[derive(Debug)] -pub struct ArrayType<'a> { - pub lbrack: Position<'a>, // position of "[" - pub len: Option>>, // Ellipsis node for [...]T array types, nil for slice types - pub elt: Box>, // element type -} - -// https://pkg.go.dev/go/ast#ChanDir -#[derive(Debug)] -pub enum ChanDir { - SEND = 1 << 0, - RECV = 1 << 1, -} - -// https://pkg.go.dev/go/ast#Spec -#[derive(Debug)] -pub enum Spec {} - -// https://pkg.go.dev/go/ast#Expr -#[derive(Debug)] -pub enum Expr<'a> { - ArrayType(ArrayType<'a>), - BasicLit(BasicLit<'a>), - BinaryExpr(BinaryExpr<'a>), - CallExpr(CallExpr<'a>), - ChanType(ChanType<'a>), - CompositeLit(CompositeLit<'a>), - Ellipsis(Ellipsis<'a>), - FuncLit(FuncLit<'a>), - FuncType(FuncType<'a>), - Ident(Ident<'a>), - IndexExpr(IndexExpr<'a>), - InterfaceType(InterfaceType<'a>), - KeyValueExpr(KeyValueExpr<'a>), - MapType(MapType<'a>), - ParenExpr(ParenExpr<'a>), - SelectorExpr(SelectorExpr<'a>), - SliceExpr(SliceExpr<'a>), - StarExpr(StarExpr<'a>), - StructType(StructType<'a>), - TypeAssertExpr(TypeAssertExpr<'a>), - UnaryExpr(UnaryExpr<'a>), -} - -// https://pkg.go.dev/go/ast#Stmt -#[derive(Debug)] -pub enum Stmt {} diff --git a/backend/parsers/windmill-parser-go/src/parser_go_scanner.rs b/backend/parsers/windmill-parser-go/src/parser_go_scanner.rs deleted file mode 100644 index 840539fce6..0000000000 --- a/backend/parsers/windmill-parser-go/src/parser_go_scanner.rs +++ /dev/null @@ -1,948 +0,0 @@ -// https://golang.org/ref/spec#Lexical_elements - -use crate::parser_go_token::{Position, Token}; -use phf::{phf_map, Map}; -use std::fmt; -use unicode_general_category::{get_general_category, GeneralCategory}; - -pub type Step<'a> = (Position<'a>, Token, &'a str); - -#[derive(Debug)] -pub enum ScannerError { - HexadecimalNotFound, - OctalNotFound, - UnterminatedComment, - UnterminatedEscapedChar, - UnterminatedRune, - UnterminatedString, - InvalidDirective, -} - -impl std::error::Error for ScannerError {} - -impl fmt::Display for ScannerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "scanner error: {:?}", self) - } -} - -pub type Result = std::result::Result; - -#[derive(Debug)] -pub struct Scanner<'a> { - directory: &'a str, - file: &'a str, - buffer: &'a str, - // - chars: std::iter::Peekable>, - current_char: Option, - current_char_len: usize, - // - offset: usize, - line: usize, - column: usize, - start_offset: usize, - start_line: usize, - start_column: usize, - // - hide_column: bool, - insert_semi: bool, - pending_line_info: Option>, -} - -type LineInfo<'a> = (Option<&'a str>, usize, Option, bool); - -impl<'a> Scanner<'a> { - pub fn new(filename: &'a str, buffer: &'a str) -> Self { - let (directory, file) = filename.rsplit_once('/').unwrap_or(("", filename)); - let mut s = Scanner { - directory, - file, - buffer, - // - chars: buffer.chars().peekable(), - current_char: None, - current_char_len: 0, - // - offset: 0, - line: 1, - column: 1, - start_offset: 0, - start_line: 1, - start_column: 1, - // - hide_column: false, - insert_semi: false, - pending_line_info: None, - }; - s.next(); // read the first character - s - } - - #[allow(clippy::cognitive_complexity)] // Allow complex scan function - pub fn scan(&mut self) -> Result> { - let insert_semi = self.insert_semi; - self.insert_semi = false; - - while let Some(c) = self.current_char { - self.reset_start(); - - match c { - ' ' | '\t' | '\r' => { - self.next(); - } - - '\n' => { - self.next(); - if insert_semi { - return Ok((self.position(), Token::SEMICOLON, "\n")); - } - } - - _ => break, - } - } - - if let Some(c) = self.current_char { - match c { - '+' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::ADD_ASSIGN, "")); - } - Some('+') => { - self.insert_semi = true; - self.next(); - return Ok((self.position(), Token::INC, "")); - } - _ => return Ok((self.position(), Token::ADD, "")), - } - } - - '-' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::SUB_ASSIGN, "")); - } - Some('-') => { - self.insert_semi = true; - self.next(); - return Ok((self.position(), Token::DEC, "")); - } - _ => return Ok((self.position(), Token::SUB, "")), - } - } - - '*' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::MUL_ASSIGN, "")); - } - _ => return Ok((self.position(), Token::MUL, "")), - } - } - - '/' => match self.peek() { - Some('=') => { - self.next(); - self.next(); - return Ok((self.position(), Token::QUO_ASSIGN, "")); - } - Some('/') => { - if insert_semi { - return Ok((self.position(), Token::SEMICOLON, "\n")); - } - return self.scan_line_comment(); - } - Some('*') => { - if insert_semi && self.find_line_end() { - return Ok((self.position(), Token::SEMICOLON, "\n")); - } - return self.scan_general_comment(); - } - _ => { - self.next(); - return Ok((self.position(), Token::QUO, "")); - } - }, - - '%' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::REM_ASSIGN, "")); - } - _ => return Ok((self.position(), Token::REM, "")), - } - } - - '&' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::AND_ASSIGN, "")); - } - Some('&') => { - self.next(); - return Ok((self.position(), Token::LAND, "")); - } - Some('^') => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::AND_NOT_ASSIGN, "")); - } - _ => return Ok((self.position(), Token::AND_NOT, "")), - } - } - _ => return Ok((self.position(), Token::AND, "")), - } - } - - '|' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::OR_ASSIGN, "")); - } - Some('|') => { - self.next(); - return Ok((self.position(), Token::LOR, "")); - } - _ => return Ok((self.position(), Token::OR, "")), - } - } - - '^' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::XOR_ASSIGN, "")); - } - _ => return Ok((self.position(), Token::XOR, "")), - } - } - - '<' => { - self.next(); - match self.current_char { - Some('<') => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::SHL_ASSIGN, "")); - } - _ => return Ok((self.position(), Token::SHL, "")), - } - } - Some('=') => { - self.next(); - return Ok((self.position(), Token::LEQ, "")); - } - Some('-') => { - self.next(); - return Ok((self.position(), Token::ARROW, "")); - } - _ => return Ok((self.position(), Token::LSS, "")), - } - } - - '>' => { - self.next(); - match self.current_char { - Some('>') => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::SHR_ASSIGN, "")); - } - _ => { - return Ok((self.position(), Token::SHR, "")); - } - } - } - Some('=') => { - self.next(); - return Ok((self.position(), Token::GEQ, "")); - } - _ => return Ok((self.position(), Token::GTR, "")), - } - } - - ':' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::DEFINE, "")); - } - _ => return Ok((self.position(), Token::COLON, "")), - } - } - - '!' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::NEQ, "")); - } - _ => return Ok((self.position(), Token::NOT, "")), - } - } - - ',' => { - self.next(); - return Ok((self.position(), Token::COMMA, "")); - } - - '(' => { - self.next(); - return Ok((self.position(), Token::LPAREN, "")); - } - - ')' => { - self.insert_semi = true; - self.next(); - return Ok((self.position(), Token::RPAREN, "")); - } - - '[' => { - self.next(); - return Ok((self.position(), Token::LBRACK, "")); - } - - ']' => { - self.insert_semi = true; - self.next(); - return Ok((self.position(), Token::RBRACK, "")); - } - - '{' => { - self.next(); - return Ok((self.position(), Token::LBRACE, "")); - } - - '}' => { - self.insert_semi = true; - self.next(); - return Ok((self.position(), Token::RBRACE, "")); - } - - ';' => { - self.next(); - return Ok((self.position(), Token::SEMICOLON, ";")); - } - - '.' => { - self.next(); - match self.current_char { - Some('0'..='9') => return self.scan_int_or_float_or_imag(true), - Some('.') => match self.peek() { - Some('.') => { - self.next(); - self.next(); - return Ok((self.position(), Token::ELLIPSIS, "")); - } - _ => return Ok((self.position(), Token::PERIOD, "")), - }, - _ => return Ok((self.position(), Token::PERIOD, "")), - } - } - - '=' => { - self.next(); - match self.current_char { - Some('=') => { - self.next(); - return Ok((self.position(), Token::EQL, "")); - } - _ => return Ok((self.position(), Token::ASSIGN, "")), - } - } - - '0'..='9' => return self.scan_int_or_float_or_imag(false), - '\'' => return self.scan_rune(), - '"' => return self.scan_interpreted_string(), - '`' => return self.scan_raw_string(), - _ => return self.scan_pkg_or_keyword_or_ident(), - }; - } - - self.reset_start(); - if insert_semi { - Ok((self.position(), Token::SEMICOLON, "\n")) - } else { - Ok((self.position(), Token::EOF, "")) - } - } - - // https://golang.org/ref/spec#Keywords - // https://golang.org/ref/spec#Identifiers - fn scan_pkg_or_keyword_or_ident(&mut self) -> Result> { - self.next(); - - while let Some(c) = self.current_char { - if !(is_letter(c) || is_unicode_digit(c)) { - break; - } - self.next() - } - - let pos = self.position(); - let literal = self.literal(); - - if literal.len() > 1 { - if let Some(&token) = KEYWORDS.get(literal) { - self.insert_semi = matches!( - token, - Token::BREAK | Token::CONTINUE | Token::FALLTHROUGH | Token::RETURN - ); - return Ok((pos, token, literal)); - } - } - - self.insert_semi = true; - Ok((pos, Token::IDENT, literal)) - } - - // https://golang.org/ref/spec#Integer_literals - // https://golang.org/ref/spec#Floating-point_literals - // https://golang.org/ref/spec#Imaginary_literals - fn scan_int_or_float_or_imag(&mut self, preceding_dot: bool) -> Result> { - self.insert_semi = true; - - let mut token = Token::INT; - let mut digits = "_0123456789"; - let mut exp = "eE"; - - if !preceding_dot { - if matches!(self.current_char, Some('0')) { - self.next(); - match self.current_char { - Some('b' | 'B') => { - digits = "_01"; - exp = ""; - self.next(); - } - Some('o' | 'O') => { - digits = "_01234567"; - exp = ""; - self.next(); - } - Some('x' | 'X') => { - digits = "_0123456789abcdefABCDEF"; - exp = "pP"; - self.next(); - } - _ => {} - }; - } - - while let Some(c) = self.current_char { - if !digits.contains(c) { - break; - } - self.next(); - } - } - - if preceding_dot || matches!(self.current_char, Some('.')) { - token = Token::FLOAT; - self.next(); - while let Some(c) = self.current_char { - if !digits.contains(c) { - break; - } - self.next(); - } - } - - if !exp.is_empty() { - if let Some(c) = self.current_char { - if exp.contains(c) { - token = Token::FLOAT; - self.next(); - if matches!(self.current_char, Some('-' | '+')) { - self.next(); - } - while let Some(c) = self.current_char { - if !matches!(c, '_' | '0'..='9') { - break; - } - self.next(); - } - } - } - } - - if matches!(self.current_char, Some('i')) { - token = Token::IMAG; - self.next(); - } - - Ok((self.position(), token, self.literal())) - } - - // https://golang.org/ref/spec#Rune_literals - fn scan_rune(&mut self) -> Result> { - self.insert_semi = true; - self.next(); - - match self.current_char { - Some('\\') => self.require_escaped_char::<'\''>()?, - Some(_) => self.next(), - _ => return Err(ScannerError::UnterminatedRune), - } - - if matches!(self.current_char, Some('\'')) { - self.next(); - return Ok((self.position(), Token::CHAR, self.literal())); - } - - Err(ScannerError::UnterminatedRune) - } - - // https://golang.org/ref/spec#String_literals - fn scan_interpreted_string(&mut self) -> Result> { - self.insert_semi = true; - self.next(); - - while let Some(c) = self.current_char { - match c { - '"' => { - self.next(); - return Ok((self.position(), Token::STRING, self.literal())); - } - '\\' => self.require_escaped_char::<'"'>()?, - _ => self.next(), - } - } - - Err(ScannerError::UnterminatedString) - } - - // https://golang.org/ref/spec#String_literals - fn scan_raw_string(&mut self) -> Result> { - self.insert_semi = true; - self.next(); - - while let Some(c) = self.current_char { - match c { - '`' => { - self.next(); - return Ok((self.position(), Token::STRING, self.literal())); - } - _ => self.next(), - } - } - - Err(ScannerError::UnterminatedString) - } - - // https://golang.org/ref/spec#Comments - fn scan_general_comment(&mut self) -> Result> { - self.next(); - self.next(); - - while let Some(c) = self.current_char { - match c { - '*' => { - self.next(); - if matches!(self.current_char, Some('/')) { - self.next(); - - let pos = self.position(); - let lit = self.literal(); - - // look for compiler directives - self.directive(&lit["/*".len()..lit.len() - "*/".len()], true)?; - - return Ok((pos, Token::COMMENT, lit)); - } - } - _ => self.next(), - } - } - - Err(ScannerError::UnterminatedComment) - } - - // https://golang.org/ref/spec#Comments - fn scan_line_comment(&mut self) -> Result> { - self.next(); - self.next(); - - while let Some(c) = self.current_char { - if is_newline(c) { - break; - } - self.next(); - } - - let pos = self.position(); - let lit = self.literal(); - - // look for compiler directives (at the beginning of line) - if self.start_column == 1 { - self.directive(lit["//".len()..].trim_end(), false)?; - } - - Ok((pos, Token::COMMENT, self.literal())) - } - - // https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives - fn directive(&mut self, input: &'a str, immediate: bool) -> Result<()> { - if let Some(line_directive) = input.strip_prefix("line ") { - self.pending_line_info = self.parse_line_directive(line_directive)?; - if immediate { - self.consume_pending_line_info(); - } - } - Ok(()) - } - - fn parse_line_directive(&mut self, line_directive: &'a str) -> Result>> { - if let Some((file, line)) = line_directive.rsplit_once(':') { - let line = line.parse().map_err(|_| ScannerError::InvalidDirective)?; - - if let Some((file, l)) = file.rsplit_once(':') { - if let Ok(l) = l.parse() { - //line :line:col - //line filename:line:col - /*line :line:col*/ - /*line filename:line:col*/ - let file = if !file.is_empty() { Some(file) } else { None }; - let col = Some(line); - let line = l; - let hide_column = false; - return Ok(Some((file, line, col, hide_column))); - } - } - - //line :line - //line filename:line - /*line :line*/ - /*line filename:line*/ - Ok(Some((Some(file), line, None, true))) - } else { - Ok(None) - } - } - - const fn find_line_end(&self) -> bool { - let buffer = self.buffer.as_bytes(); - let mut in_comment = true; - - let mut i = self.offset; - let max = self.buffer.len(); - while i < max { - let c = buffer[i] as char; - - if i < max - 1 { - let n = buffer[i + 1] as char; - - if !in_comment && c == '/' && n == '/' { - return true; - } - - if c == '/' && n == '*' { - i += 2; - in_comment = true; - continue; - } - - if c == '*' && n == '/' { - i += 2; - in_comment = false; - continue; - } - } - - if is_newline(c) { - return true; - } - - if !in_comment && !matches!(c, ' ' | '\t' | '\r') { - return false; - } - - i += 1; - } - - !in_comment - } - - fn consume_pending_line_info(&mut self) { - if let Some(line_info) = self.pending_line_info.take() { - if let Some(file) = line_info.0 { - self.file = file; - } - - self.line = line_info.1; - - if let Some(column) = line_info.2 { - self.column = column; - } - - self.hide_column = line_info.3; - } - } - - fn peek(&mut self) -> Option { - self.chars.peek().copied() - } - - fn next(&mut self) { - self.offset += self.current_char_len; - self.column += self.current_char_len; - let last_char = self.current_char; - - self.current_char = self.chars.next(); - if let Some(c) = self.current_char { - self.current_char_len = c.len_utf8(); - if matches!(last_char, Some('\n')) { - self.line += 1; - self.column = 1; - self.consume_pending_line_info(); - } - } else { - self.current_char_len = 0 - } - } - - const fn position(&self) -> Position<'a> { - Position { - directory: self.directory, - file: self.file, - offset: self.start_offset, - line: self.start_line, - column: if self.hide_column { - 0 - } else { - self.start_column - }, - } - } - - fn reset_start(&mut self) { - self.start_offset = self.offset; - self.start_line = self.line; - self.start_column = self.column; - } - - fn literal(&self) -> &'a str { - &self.buffer[self.start_offset..self.offset] - } - - fn require_escaped_char(&mut self) -> Result<()> { - self.next(); - - let c = self - .current_char - .ok_or(ScannerError::UnterminatedEscapedChar)?; - - // TODO: move this to the match when const generics can be referenced in patterns - if c == DELIM { - self.next(); - return Ok(()); - } - - match c { - 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | '\\' => self.next(), - 'x' => { - self.next(); - self.require_hex_digits::<2>()? - } - 'u' => { - self.next(); - self.require_hex_digits::<4>()?; - } - 'U' => { - self.next(); - self.require_hex_digits::<8>()?; - } - '0'..='7' => self.require_octal_digits::<3>()?, - _ => return Err(ScannerError::UnterminatedEscapedChar), - } - - Ok(()) - } - - fn require_octal_digits(&mut self) -> Result<()> { - for _ in 0..COUNT { - let c = self.current_char.ok_or(ScannerError::OctalNotFound)?; - - if !is_octal_digit(c) { - return Err(ScannerError::OctalNotFound); - } - - self.next(); - } - - Ok(()) - } - - fn require_hex_digits(&mut self) -> Result<()> { - for _ in 0..COUNT { - let c = self.current_char.ok_or(ScannerError::HexadecimalNotFound)?; - - if !is_hex_digit(c) { - return Err(ScannerError::HexadecimalNotFound); - } - - self.next(); - } - - Ok(()) - } -} - -impl<'a> IntoIterator for Scanner<'a> { - type Item = Result>; - type IntoIter = IntoIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - Self::IntoIter::new(self) - } -} - -pub struct IntoIter<'a> { - scanner: Scanner<'a>, - done: bool, -} - -impl<'a> IntoIter<'a> { - const fn new(scanner: Scanner<'a>) -> Self { - Self { scanner, done: false } - } -} - -impl<'a> Iterator for IntoIter<'a> { - type Item = Result>; - - fn next(&mut self) -> Option { - if self.done { - return None; - } - - match self.scanner.scan() { - Ok((pos, tok, lit)) => { - if tok == Token::EOF { - self.done = true; - } - Some(Ok((pos, tok, lit))) - } - Err(err) => { - self.done = true; - Some(Err(err)) - } - } - } -} - -// https://golang.org/ref/spec#Letters_and_digits - -fn is_letter(c: char) -> bool { - c == '_' || is_unicode_letter(c) -} - -//const fn is_decimal_digit(c: char) -> bool { -//matches!(c, '0'..='9') -//} - -//const fn is_binary_digit(c: char) -> bool { -//matches!(c, '0'..='1') -//} - -const fn is_octal_digit(c: char) -> bool { - matches!(c, '0'..='7') -} - -const fn is_hex_digit(c: char) -> bool { - matches!(c, '0'..='9' | 'A'..='F' | 'a'..='f') -} - -// https://golang.org/ref/spec#Characters - -const fn is_newline(c: char) -> bool { - c == '\n' -} - -//const fn is_unicode_char(c: char) -> bool { -//c != '\n' -//} - -fn is_unicode_letter(c: char) -> bool { - matches!( - get_general_category(c), - GeneralCategory::UppercaseLetter - | GeneralCategory::LowercaseLetter - | GeneralCategory::TitlecaseLetter - | GeneralCategory::ModifierLetter - | GeneralCategory::OtherLetter - ) -} - -fn is_unicode_digit(c: char) -> bool { - get_general_category(c) == GeneralCategory::DecimalNumber -} - -// https://golang.org/ref/spec#Keywords - -static KEYWORDS: Map<&'static str, Token> = phf_map! { - "break" => Token::BREAK, - "case" => Token::CASE, - "chan" => Token::CHAN, - "const" => Token::CONST, - "continue" => Token::CONTINUE, - - "default" => Token::DEFAULT, - "defer" => Token::DEFER, - "else" => Token::ELSE, - "fallthrough" => Token::FALLTHROUGH, - "for" => Token::FOR, - - "func" => Token::FUNC, - "go" => Token::GO, - "goto" => Token::GOTO, - "if" => Token::IF, - "import" => Token::IMPORT, - - "interface" => Token::INTERFACE, - "map" => Token::MAP, - "package" => Token::PACKAGE, - "range" => Token::RANGE, - "return" => Token::RETURN, - - "select" => Token::SELECT, - "struct" => Token::STRUCT, - "switch" => Token::SWITCH, - "type" => Token::TYPE, - "var" => Token::VAR, -}; - -#[cfg(test)] -mod tests { - use super::Scanner; - - #[test] // fuzz - fn it_should_return_an_error_on_missing_line_number() { - let input = "/*line :*/"; - let mut out: Vec<_> = Scanner::new(file!(), input).into_iter().collect(); - assert!(out.pop().unwrap().is_err()); - } -} diff --git a/backend/parsers/windmill-parser-go/src/parser_go_token.rs b/backend/parsers/windmill-parser-go/src/parser_go_token.rs deleted file mode 100644 index 5d63605374..0000000000 --- a/backend/parsers/windmill-parser-go/src/parser_go_token.rs +++ /dev/null @@ -1,273 +0,0 @@ -// https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/go/token/token.go - -#![allow(non_camel_case_types)] // For consistency with the Go tokens - -use std::fmt; - -#[derive(Clone, Copy, Debug, Default)] -pub struct Position<'a> { - pub directory: &'a str, - pub file: &'a str, - pub offset: usize, - pub line: usize, - pub column: usize, -} - -impl<'a> fmt::Display for Position<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.file.is_empty() { - write!(f, ":{}:{}", self.line, self.column) - } else if self.file.starts_with('/') { - write!(f, "{}:{}:{}", self.file, self.line, self.column) - } else { - write!( - f, - "{}/{}:{}:{}", - self.directory, self.file, self.line, self.column - ) - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum Token { - EOF, - COMMENT, - - IDENT, // main - INT, // 12345 - FLOAT, // 123.45 - IMAG, // 123.45i - CHAR, // 'a' - STRING, // "abc" - - ADD, // + - SUB, // - - MUL, // * - QUO, // / - REM, // % - - AND, // & - OR, // | - XOR, // ^ - SHL, // << - SHR, // >> - AND_NOT, // &^ - - ADD_ASSIGN, // += - SUB_ASSIGN, // -= - MUL_ASSIGN, // *= - QUO_ASSIGN, // /= - REM_ASSIGN, // %= - - AND_ASSIGN, // &= - OR_ASSIGN, // |= - XOR_ASSIGN, // ^= - SHL_ASSIGN, // <<= - SHR_ASSIGN, // >>= - AND_NOT_ASSIGN, // &^= - - LAND, // && - LOR, // || - ARROW, // <- - INC, // ++ - DEC, // -- - - EQL, // == - LSS, // < - GTR, // > - ASSIGN, // = - NOT, // ! - - NEQ, // != - LEQ, // <= - GEQ, // >= - DEFINE, // := - ELLIPSIS, // ... - - LPAREN, // ( - LBRACK, // [ - LBRACE, // { - COMMA, // , - PERIOD, // . - - RPAREN, // ) - RBRACK, // ] - RBRACE, // } - SEMICOLON, // ; - COLON, // : - - BREAK, - CASE, - CHAN, - CONST, - CONTINUE, - - DEFAULT, - DEFER, - ELSE, - FALLTHROUGH, - FOR, - - FUNC, - GO, - GOTO, - IF, - IMPORT, - - INTERFACE, - MAP, - PACKAGE, - RANGE, - RETURN, - - SELECT, - STRUCT, - SWITCH, - TYPE, - VAR, -} - -impl Token { - pub const fn is_assign_op(&self) -> bool { - use Token::*; - matches!( - self, - ADD_ASSIGN - | SUB_ASSIGN - | MUL_ASSIGN - | QUO_ASSIGN - | REM_ASSIGN - | AND_ASSIGN - | OR_ASSIGN - | XOR_ASSIGN - | SHL_ASSIGN - | SHR_ASSIGN - | AND_NOT_ASSIGN - ) - } - - // https://go.dev/ref/spec#Operator_precedence - pub fn precedence(&self) -> u8 { - use Token::*; - match self { - MUL | QUO | REM | SHL | SHR | AND | AND_NOT => 5, - ADD | SUB | OR | XOR => 4, - EQL | NEQ | LSS | LEQ | GTR | GEQ => 3, - LAND => 2, - LOR => 1, - _ => unreachable!( - "precedence() is only supported for binary operators, called with: {:?}", - self - ), - } - } - - pub const fn lowest_precedence() -> u8 { - 0 - } -} - -impl From<&Token> for &'static str { - fn from(token: &Token) -> Self { - use Token::*; - - match token { - EOF => "EOF", - COMMENT => "COMMENT", - - IDENT => "IDENT", - INT => "INT", - FLOAT => "FLOAT", - IMAG => "IMAG", - CHAR => "CHAR", - STRING => "STRING", - - ADD => "+", - SUB => "-", - MUL => "*", - QUO => "/", - REM => "%", - - AND => "&", - OR => "|", - XOR => "^", - SHL => "<<", - SHR => ">>", - AND_NOT => "&^", - - ADD_ASSIGN => "+=", - SUB_ASSIGN => "-=", - MUL_ASSIGN => "*=", - QUO_ASSIGN => "/=", - REM_ASSIGN => "%=", - - AND_ASSIGN => "&=", - OR_ASSIGN => "|=", - XOR_ASSIGN => "^=", - SHL_ASSIGN => "<<=", - SHR_ASSIGN => ">>=", - AND_NOT_ASSIGN => "&^=", - - LAND => "&&", - LOR => "||", - ARROW => "<-", - INC => "++", - DEC => "--", - - EQL => "==", - LSS => "<", - GTR => ">", - ASSIGN => "=", - NOT => "!", - - NEQ => "!=", - LEQ => "<=", - GEQ => ">=", - DEFINE => ":=", - ELLIPSIS => "...", - - LPAREN => "(", - LBRACK => "[", - LBRACE => "{", - COMMA => ",", - PERIOD => ".", - - RPAREN => ")", - RBRACK => "]", - RBRACE => "}", - SEMICOLON => ";", - COLON => ":", - - BREAK => "break", - CASE => "case", - CHAN => "chan", - CONST => "const", - CONTINUE => "continue", - - DEFAULT => "default", - DEFER => "defer", - ELSE => "else", - FALLTHROUGH => "fallthrough", - FOR => "for", - - FUNC => "func", - GO => "go", - GOTO => "goto", - IF => "if", - IMPORT => "import", - - INTERFACE => "interface", - MAP => "map", - PACKAGE => "package", - RANGE => "range", - RETURN => "return", - - SELECT => "select", - STRUCT => "struct", - SWITCH => "switch", - TYPE => "type", - VAR => "var", - } - } -} diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 9131cb35fa..05970cc991 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -218,7 +218,7 @@ pub fn parse_python_imports(code: &str) -> error::Result> { let ast = parser::parse_program(code, "main.py").map_err(|e| { error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) })?; - let imports = ast + let mut imports: Vec = ast .into_iter() .filter_map(|x| match x { Located { node, .. } => match node { @@ -250,7 +250,7 @@ pub fn parse_python_imports(code: &str) -> error::Result> { .filter(|x| !STDIMPORTS.contains(&x.as_str())) .unique() .collect(); - + imports.sort(); Ok(imports) } } @@ -460,7 +460,7 @@ def main(): "; let r = parse_python_imports(code)?; // println!("{}", serde_json::to_string(&r)?); - assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib", "requests"]); + assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]); Ok(()) } diff --git a/backend/sqlx-data.json b/backend/sqlx-data.json index 42ee3afd93..2c5a056ed3 100644 --- a/backend/sqlx-data.json +++ b/backend/sqlx-data.json @@ -1808,6 +1808,19 @@ }, "query": "\n UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n WHERE id = $2\n " }, + "4fb3881cdbb4b9e93e28f460a9b3715bdc6a52b76c89f3a3913023b13c4e085c": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + } + }, + "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2" + }, "5061c0d054bf4f028e7fe51a8f9389024c6ae4492755cadac0f7167e5300bda0": { "describe": { "columns": [], @@ -3874,19 +3887,6 @@ }, "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]" }, - "9baf1f88ab6c1df642cf8f0f49805ccac3e48cce1f0e184fb9db78536a8ca7b0": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - } - }, - "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() - ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2" - }, "9db64c9ff790d8c833c1e831a87803c32103ce9de68cc9c08d6f56cc988d7e37": { "describe": { "columns": [ diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2ef5571384..29ea44e350 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -14,6 +14,7 @@ use once_cell::sync::OnceCell; use regex::Regex; use sqlx::{Pool, Postgres, Transaction}; use windmill_api_client::Client; +use windmill_parser_go::parse_go_imports; use std::{ borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic, process::Stdio, time::Duration, sync::atomic::Ordering, @@ -340,7 +341,7 @@ const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.co const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); -const GO_REQ_SPLITTER: &str = "//go.sum"; +const GO_REQ_SPLITTER: &str = "//go.sum\n"; #[derive(Clone)] pub struct Metrics { @@ -1319,6 +1320,20 @@ mount {{ result } +async fn gen_go_mod(inner_content: &str, job_dir: &str, requirements: &str) -> error::Result<(bool, bool)> { + gen_go_mymod(inner_content, job_dir).await?; + + let md = requirements + .split_once(GO_REQ_SPLITTER); + if let Some((req, sum)) = md { + write_file(job_dir, "go.mod", &req).await?; + write_file(job_dir, "go.sum", &sum).await?; + Ok((true, true)) + } else { + write_file(job_dir, "go.mod", &requirements).await?; + Ok((true, false)) + } +} #[tracing::instrument(level = "trace", skip_all)] async fn handle_go_job( logs: &mut String, @@ -1335,19 +1350,10 @@ async fn handle_go_job( ) -> Result { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); - let skip_go_mod = if let Some(requirements) = requirements_o { - gen_go_mymod(inner_content, job_dir).await?; - - // TODO: remove after some time in favor of just requirements - // this is just migration code from a time we also stored go.sum - let md = requirements - .split_once(GO_REQ_SPLITTER) - .map(|x| x.0) - .unwrap_or(&requirements); - write_file(job_dir, "go.mod", &md).await?; - true + let (skip_go_mod, skip_tidy) = if let Some(requirements) = requirements_o { + gen_go_mod(inner_content, job_dir, &requirements).await? } else { - false + (false, false) }; logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n"); set_logs(logs, &job.id, db).await; @@ -1360,6 +1366,7 @@ async fn handle_go_job( db, true, skip_go_mod, + skip_tidy, worker_name, &job.workspace_id, ) @@ -2256,6 +2263,7 @@ async fn capture_dependency_job( db, false, false, + false, worker_name, w_id ) @@ -2331,7 +2339,7 @@ async fn pip_compile( .collect::>() .join("\n"); sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() - ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, lockfile ).fetch_optional(db).await?; @@ -2344,8 +2352,9 @@ async fn install_go_dependencies( logs: &mut String, job_dir: &str, db: &sqlx::Pool, - preview: bool, + non_dep_job: bool, skip_go_mod: bool, + has_sum: bool, worker_name: &str, w_id: &str ) -> error::Result { @@ -2360,10 +2369,39 @@ async fn install_go_dependencies( handle_child(job_id, db, logs, child, false, worker_name, w_id).await?; } + + let mut new_lockfile = false; + + let hash = if !has_sum { + calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str()) + } else { + "".to_string() + }; + + let mut skip_tidy = has_sum; + + if !has_sum { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + hash + ).fetch_optional(db).await? { + logs.push_str(&format!("\nfound cached resolution")); + gen_go_mod(code, job_dir, &cached).await?; + skip_tidy = true; + new_lockfile = false; + } else { + new_lockfile = true; + } + } + let mod_command = if skip_tidy { + "download" + } else { + "tidy" + }; let child = Command::new(GO_PATH.as_str()) .current_dir(job_dir) .env("GOPATH", GO_CACHE_DIR) - .args(vec!["mod", "tidy"]) + .args(vec!["mod", mod_command]) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; @@ -2371,14 +2409,28 @@ async fn install_go_dependencies( .await .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; - if preview { - Ok(String::new()) + if (!new_lockfile || has_sum) && non_dep_job{ + return Ok("".to_string()); + } + + + let mut req_content = "".to_string(); + + let mut file = File::open(format!("{job_dir}/go.mod")).await?; + file.read_to_string(&mut req_content).await?; + req_content.push_str(GO_REQ_SPLITTER); + let mut file = File::open(format!("{job_dir}/go.sum")).await?; + file.read_to_string(&mut req_content).await?; + + if non_dep_job { + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + hash, + req_content + ).fetch_optional(db).await?; + + return Ok(String::new()) } else { - let mut req_content = "".to_string(); - - let mut file = File::open(format!("{job_dir}/go.mod")).await?; - file.read_to_string(&mut req_content).await?; - Ok(req_content) } }