starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-19 00:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add no_uv_usage stats (#4883)

Browse Source 
* fix no_uv not affecting deploy

Before this fix no_uv, no_uv_compile and no_uv_install were not affecting Dependency jobs

These jobs are only affected if used USE_PIP_COMPILE or USE_PIP_INSTALL env variables

To make it more consistant, no_uv should also affect dep jobs.

Also make ansible use uv by default

* Make it build

* Add no_uv_usage stats

* Provide build-env for Samael through shell.nix

* Run update_sqlx.sh

* Update ee-repo-ref

* Update sqlx cache

* Update ee-repo-ref

* Update ee repo ref

* Update sqlx cache

* Update ee-repo-ref.txt
This commit is contained in:
pyranota
2024-12-12 00:32:03 +01:00
committed by GitHub
parent 4ce4fba18f
commit 3618e26326
8 changed files with 165 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'python' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_py'\n\n UNION ALL\n \n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'ansible' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_ansible'\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_deploy",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "deploys_count",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "language",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null
]
},
"hash": "240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0"
}
backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f"
}
backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_ansible', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052"
}
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
a07bc62582c809457f1c945d6cda145770b94d04
89a221f8b6e0dc431c668ed804066c97b428f6ef
backend/windmill-worker/src/ansible_executor.rs
+2 -2
View File
@@ -116,8 +116,8 @@ async fn handle_ansible_python_deps(
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
true,
true,
false,
false,
)
.await?;
additional_python_paths.append(&mut venv_path);
backend/windmill-worker/src/python_executor.rs
+2 -2
View File
@@ -51,11 +51,11 @@ lazy_static::lazy_static! {
static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
static ref PIP_INDEX_CERT: Option<String> = std::env::var("PIP_INDEX_CERT").ok();
static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
.ok().map(|flag| flag == "true").unwrap_or(false);
/// Use pip install
static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
pub static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
.ok().map(|flag| flag == "true").unwrap_or(false);
backend/windmill-worker/src/worker_lockfiles.rs
+66 -25
View File
@@ -13,7 +13,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId};
use windmill_common::get_latest_deployed_hash_for_path;
use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file};
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations};
use windmill_common::{
apps::AppScriptId,
error::{self, to_anyhow},
@@ -28,7 +28,9 @@ use windmill_parser_ts::parse_expr_for_imports;
use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel};
use crate::common::OccupancyMetrics;
use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile};
use crate::python_executor::{
create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL,
};
use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile};
use crate::{
bun_executor::gen_bun_lockfile,
@@ -1574,6 +1576,8 @@ async fn python_dep(
w_id: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
no_uv_compile: bool,
no_uv_install: bool,
) -> std::result::Result<String, Error> {
create_dependencies_dir(job_dir).await;
let req: std::result::Result<String, Error> = uv_pip_compile(
@@ -1586,7 +1590,7 @@ async fn python_dep(
worker_name,
w_id,
occupancy_metrics,
false,
no_uv_compile,
false,
)
.await;
@@ -1603,7 +1607,7 @@ async fn python_dep(
job_dir,
worker_dir,
occupancy_metrics,
false,
no_uv_install,
false,
)
.await;
@@ -1654,28 +1658,23 @@ async fn capture_dependency_job(
.join("\n")
};
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
)
.await
}
ScriptLang::Ansible => {
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for ansible".to_string(),
));
let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } =
PythonAnnotations::parse(job_raw_code);
if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_py', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e);
}
}
let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?;
let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default();
python_dep(
reqs,
@@ -1688,6 +1687,48 @@ async fn capture_dependency_job(
w_id,
worker_dir,
&mut Some(occupancy_metrics),
no_uv_compile | no_uv,
no_uv_install | no_uv,
)
.await
}
ScriptLang::Ansible => {
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for ansible".to_string(),
));
}
let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?;
let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default();
if *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_ansible', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e);
};
}
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
false,
false,
)
.await
}
shell.nix
+34
View File
@@ -7,6 +7,8 @@ let
rust_overlay = import (builtins.fetchTarball
"https://github.com/oxalica/rust-overlay/archive/master.tar.gz");
pkgs = import <nixpkgs> { overlays = [ rust_overlay ]; };
lib = pkgs.lib;
stdenv = pkgs.stdenv;
# TODO: Pin version?
# rustVersion = "latest";
rustVersion = "2024-09-30";
@@ -58,4 +60,36 @@ in pkgs.mkShell {
# Useful for development
RUST_LOG = "debug";
# ---- Samael ----
# https://github.com/njaremko/samael/blob/master/flake.nix#L104-L119
# Otherwise samael crate will fail to build
# Need to tell bindgen where to find libclang
LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
# Set C flags for Rust's bindgen program. Unlike ordinary C
# compilation, bindgen does not invoke $CC directly. Instead it
# uses LLVM's libclang. To make sure all necessary flags are
# included we need to look in a few places.
# See https://web.archive.org/web/20220523141208/https://hoverbear.org/blog/rust-bindgen-in-nix/
BINDGEN_EXTRA_CLANG_ARGS =
"${builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"} ${
builtins.readFile "${stdenv.cc}/nix-support/libc-cflags"
} ${builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"} ${
builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags"
} -idirafter ${pkgs.libiconv}/include ${
lib.optionalString stdenv.cc.isClang
"-idirafter ${stdenv.cc.cc}/lib/clang/${
lib.getVersion stdenv.cc.cc
}/include"
} ${
lib.optionalString stdenv.cc.isGNU
"-isystem ${stdenv.cc.cc}/include/c++/${
lib.getVersion stdenv.cc.cc
} -isystem ${stdenv.cc.cc}/include/c++/${
lib.getVersion stdenv.cc.cc
}/${stdenv.hostPlatform.config} -idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${
lib.getVersion stdenv.cc.cc
}/include"
}";
# ---- Samael ----
}