skip s3 python cache on windows and pro

This commit is contained in:
Ruben Fiszel
2024-11-29 08:35:08 +01:00
parent a1dd641466
commit 8338efa685
4 changed files with 27 additions and 18 deletions
+4
View File
@@ -256,6 +256,10 @@ async fn windmill_main() -> anyhow::Result<()> {
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
tracing::info!("Binary is in 'agent' mode");
+8 -8
View File
@@ -1,22 +1,22 @@
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use crate::PIP_CACHE_DIR;
// #[cfg(feature = "enterprise")]
// use rand::Rng;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use tokio::time::Instant;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use object_store::ObjectStore;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use windmill_common::error;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use std::sync::Arc;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
pub async fn build_tar_and_push(
s3_client: Arc<dyn ObjectStore>,
folder: String,
@@ -81,7 +81,7 @@ pub async fn build_tar_and_push(
Ok(())
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
pub async fn pull_from_tar(
client: Arc<dyn ObjectStore>,
folder: String,
@@ -115,7 +115,7 @@ pub async fn pull_from_tar(
Ok(())
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> {
use bytes::Buf;
use tokio::fs::{self};
+12 -9
View File
@@ -19,7 +19,7 @@ use tokio::{
task,
};
use uuid::Uuid;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use windmill_common::ee::{get_license_plan, LicensePlan};
use windmill_common::{
error::{self, Error},
@@ -71,10 +71,10 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str =
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use crate::global_cache::{build_tar_and_push, pull_from_tar};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use crate::{
@@ -1335,12 +1335,12 @@ pub async fn handle_python_reqs(
instant: std::time::Instant,
db: Pool<Postgres>,
) {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
#[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))]
{
(s3_pull, s3_push) = (false, false);
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if OBJECT_STORE_CACHE_SETTINGS.read().await.is_none() {
(s3_pull, s3_push) = (false, false);
}
@@ -1557,6 +1557,9 @@ pub async fn handle_python_reqs(
let semaphore = Arc::new(Semaphore::new(parallel_limit));
let mut handles = Vec::with_capacity(total_to_install);
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
let is_not_pro = matches!(get_license_plan().await, LicensePlan::Pro);
for ((req, venv_p), mut kill_rx) in req_with_penv.iter().zip(kill_rxs.into_iter()) {
let permit = semaphore.clone().acquire_owned().await; // Acquire a permit
@@ -1598,8 +1601,8 @@ pub async fn handle_python_reqs(
);
let start = std::time::Instant::now();
#[cfg(all(feature = "enterprise", feature = "parquet"))]
{
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if is_not_pro {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
tokio::select! {
// Cancel was called on the job
@@ -1717,9 +1720,9 @@ pub async fn handle_python_reqs(
)
.await;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if matches!(get_license_plan().await, LicensePlan::Pro) {
if !is_not_pro {
tracing::warn!("S3 cache not available in the pro plan");
} else {
tokio::spawn(build_tar_and_push(os, venv_p.clone(), no_uv_install));
+3 -1
View File
@@ -8,7 +8,6 @@
use windmill_common::{
auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET},
ee::LICENSE_KEY_VALID,
scripts::PREVIEW_IS_TAR_CODEBASE_HASH,
worker::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, write_file,
@@ -16,6 +15,9 @@ use windmill_common::{
},
};
#[cfg(feature = "enterprise")]
use windmill_common::ee::LICENSE_KEY_VALID;
use anyhow::{Context, Result};
use const_format::concatcp;
#[cfg(feature = "prometheus")]