starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-09 16:05:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support multiple object storage + parquet_csv + polars -> datafusion" (#3853)

Browse Source 
* multiple storage

* all

* rm symlinks

* all

* all
This commit is contained in:
Ruben Fiszel
2024-05-31 11:59:51 +02:00
committed by Ruben Fiszel
parent 78d704b2d6
commit 0e7a25c0bd
21 changed files with 539 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
5da40a29e6e4dfd4376fd917807e6936f244f6de
d8aaa3f3a3fc16e41cb4a530611eba78e569923c
backend/substitute_ee_code.sh
+1 -1
View File
@@ -71,7 +71,7 @@ if [ "$REVERT" == "YES" ]; then
ce_file="${ee_file/${EE_CODE_DIR}/.}"
ce_file="${root_dirpath}/backend/${ce_file}"
if [ "$REVERT_PREVIOUS" == "YES" ]; then
git checkout HEAD@{5} ${ce_file} || true
git checkout HEAD@{3} ${ce_file} || true
else
git restore --staged ${ce_file} || true
git restore ${ce_file} || true
backend/windmill-api/openapi.yaml
+133 -3
View File
@@ -7717,12 +7717,16 @@ paths:
/w/{workspace}/job_helpers/test_connection:
get:
summary: Test connection to the workspace datasets storage
summary: Test connection to the workspace object storage
operationId: datasetStorageTestConnection
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: storage
in: query
schema:
type: string
responses:
"200":
description: Connection settings
@@ -7732,7 +7736,7 @@ paths:
/w/{workspace}/job_helpers/list_stored_files:
get:
summary: List the file keys available in the workspace files storage (S3)
summary: List the file keys available in a workspace object storage
operationId: listStoredFiles
tags:
- helpers
@@ -7751,6 +7755,10 @@ paths:
in: query
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: List of file keys
@@ -7783,6 +7791,10 @@ paths:
required: true
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: FileMetadata
@@ -7828,6 +7840,10 @@ paths:
in: query
schema:
type: integer
- name: storage
in: query
schema:
type: string
responses:
"200":
description: FilePreview
@@ -7869,7 +7885,10 @@ paths:
in: query
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: Parquet Preview
@@ -7877,6 +7896,50 @@ paths:
application/json:
schema: {}
/w/{workspace}/job_helpers/load_csv_preview/{path}:
get:
summary: Load a preview of a csv file
operationId: loadCsvPreview
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
- name: offset
in: query
schema:
type: number
- name: limit
in: query
schema:
type: number
- name: sort_col
in: query
schema:
type: string
- name: sort_desc
in: query
schema:
type: boolean
- name: search_col
in: query
schema:
type: string
- name: search_term
in: query
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: Csv Preview
content:
application/json:
schema: {}
/w/{workspace}/job_helpers/delete_s3_file:
delete:
summary: Permanently delete file from S3
@@ -7890,6 +7953,10 @@ paths:
required: true
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: Confirmation
@@ -7915,6 +7982,10 @@ paths:
required: true
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: Confirmation
@@ -7950,6 +8021,10 @@ paths:
required: false
schema:
type: string
- name: storage
in: query
schema:
type: string
requestBody:
description: File content
required: true
@@ -7994,6 +8069,10 @@ paths:
required: false
schema:
type: string
- name: storage
in: query
schema:
type: string
responses:
"200":
description: Chunk of the downloaded file
@@ -8003,6 +8082,37 @@ paths:
type: string
format: binary
/w/{workspace}/job_helpers/download_s3_parquet_file_as_csv:
get:
summary: Download file to S3 bucket
operationId: fileDownloadParquetAsCsv
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: file_key
in: query
required: true
schema:
type: string
- name: s3_resource_path
in: query
required: false
schema:
type: string
- name: resource_type
in: query
required: false
schema:
type: string
responses:
"200":
description: The downloaded file
content:
text/csv:
schema:
type: string
/w/{workspace}/job_metrics/get/{id}:
post:
summary: get job metrics
@@ -10373,6 +10483,26 @@ components:
type: string
public_resource:
type: boolean
secondary_storage:
type: object
additionalProperties:
type: object
properties:
type:
type: string
enum:
[
"S3Storage",
"AzureBlobStorage",
"AzureWorkloadIdentity",
"S3AwsOidc",
]
s3_resource_path:
type: string
azure_blob_resource_path:
type: string
public_resource:
type: boolean
WindmillLargeFile:
type: object
backend/windmill-api/src/workspaces.rs
+13 -4
View File
@@ -230,9 +230,17 @@ struct EditCopilotConfig {
code_completion_enabled: bool,
}
#[derive(Deserialize)]
#[derive(Deserialize, Serialize, Debug)]
struct LargeFileStorageWithSecondary {
#[serde(flatten)]
large_file_storage: LargeFileStorage,
secondary_storage: HashMap<String, LargeFileStorage>,
}
#[derive(Deserialize, Debug)]
struct EditLargeFileStorageConfig {
large_file_storage: Option<LargeFileStorage>,
large_file_storage: Option<LargeFileStorageWithSecondary>,
}
#[derive(Deserialize)]
@@ -952,8 +960,9 @@ async fn edit_large_file_storage_config(
.await?;
if let Some(lfs_config) = new_config.large_file_storage {
let serialized_lfs_config = serde_json::to_value::<LargeFileStorage>(lfs_config)
.map_err(|err| Error::InternalErr(err.to_string()))?;
let serialized_lfs_config =
serde_json::to_value::<LargeFileStorageWithSecondary>(lfs_config)
.map_err(|err| Error::InternalErr(err.to_string()))?;
sqlx::query!(
"UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
backend/windmill-common/src/s3_helpers.rs
+1
View File
@@ -113,6 +113,7 @@ pub struct S3AwsOidcResource {
#[derive(Deserialize, Clone)]
pub struct S3Object {
pub s3: String,
pub storage: Option<String>,
}
#[cfg(feature = "parquet")]
backend/windmill-worker/src/common.rs
+53 -52
View File
@@ -1357,7 +1357,7 @@ pub async fn hash_args(
k.hash(&mut dh);
let arg_value = hm.get(k).unwrap();
#[cfg(feature = "parquet")]
let arg_additions =
let (_, arg_additions) =
arg_value_hash_additions(_db, _client, _workspace_id, hm.get(k).unwrap()).await;
arg_value.get().hash(&mut dh);
#[cfg(feature = "parquet")]
@@ -1376,21 +1376,33 @@ async fn get_workspace_s3_resource_path(
db: &DB,
client: &AuthedClient,
workspace_id: &str,
storage: Option<String>,
) -> windmill_common::error::Result<Option<ObjectStoreResource>> {
use windmill_common::{
job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType,
};
let raw_lfs_opt = sqlx::query_scalar!(
"SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1",
workspace_id
)
.fetch_one(db)
.await
.ok()
.flatten()
.map(|val| serde_json::from_value::<LargeFileStorage>(val).ok())
.flatten();
let raw_lfs_opt = if let Some(storage) = storage {
sqlx::query_scalar!(
"SELECT large_file_storage->'secondary_storage'->$2 FROM workspace_settings WHERE workspace_id = $1",
workspace_id,
storage
)
.fetch_optional(db)
.await?
.flatten()
} else {
sqlx::query_scalar!(
"SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1",
workspace_id
)
.fetch_optional(db)
.await?
.flatten()
};
let raw_lfs_opt = raw_lfs_opt
.map(|val| serde_json::from_value::<LargeFileStorage>(val).ok())
.flatten();
let (rt, path) = match raw_lfs_opt {
Some(LargeFileStorage::S3Storage(s3_storage)) => {
@@ -1440,47 +1452,26 @@ async fn arg_value_hash_additions(
client: &AuthedClient,
workspace_id: &str,
raw_value: &Box<RawValue>,
) -> HashMap<String, String> {
) -> (Option<String>, HashMap<String, String>) {
let mut result: HashMap<String, String> = HashMap::new();
let parsed_s3_values: &mut Vec<S3Object> = &mut Vec::new();
extract_all_s3_object_from_raw_value(raw_value, parsed_s3_values);
if parsed_s3_values.is_empty() {
// no s3 object, nothing to return
return result;
}
let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id).await;
if let Some(s3_resource) = s3_resource_opt.ok().flatten() {
for s3_object in parsed_s3_values {
let parsed_value = serde_json::from_str::<S3Object>(raw_value.get());
let mut storage = None;
if let Ok(s3_object) = parsed_value {
let s3_resource_opt =
get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.clone())
.await;
storage = s3_object.storage.clone();
if let Some(s3_resource) = s3_resource_opt.ok().flatten() {
let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await;
tracing::warn!("Enriching s3 arg value with etag: {:?}", etag);
result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache?
}
}
return result;
}
#[cfg(feature = "parquet")]
fn extract_all_s3_object_from_raw_value(raw_value: &Box<RawValue>, result: &mut Vec<S3Object>) {
let parsed_value = serde_json::from_str::<S3Object>(raw_value.get());
if let Ok(parsed_value) = parsed_value {
result.push(parsed_value);
} else {
let parsed_value = serde_json::from_str::<HashMap<String, Box<RawValue>>>(raw_value.get());
if let Ok(parsed_value) = parsed_value {
for (_, v) in parsed_value {
extract_all_s3_object_from_raw_value(&v, result);
}
} else {
let parsed_value = serde_json::from_str::<Vec<Box<RawValue>>>(raw_value.get());
if let Ok(parsed_value) = parsed_value {
for v in parsed_value {
extract_all_s3_object_from_raw_value(&v, result);
}
}
}
}
return (storage, result);
}
#[derive(Deserialize, Serialize)]
@@ -1489,6 +1480,7 @@ struct CachedResource {
#[serde(skip_serializing_if = "Option::is_none")]
s3_etags: Option<HashMap<String, String>>,
value: Box<RawValue>,
storage: Option<String>,
}
pub async fn get_cached_resource_value_if_valid(
@@ -1513,10 +1505,15 @@ pub async fn get_cached_resource_value_if_valid(
let object_store_resource_opt: Option<ObjectStoreResource> = if s3_etags.is_empty() {
None
} else {
get_workspace_s3_resource_path(_db, &client, _workspace_id)
.await
.ok()
.flatten()
get_workspace_s3_resource_path(
_db,
&client,
_workspace_id,
cached_resource.storage.clone(),
)
.await
.ok()
.flatten()
};
if !s3_etags.is_empty() && object_store_resource_opt.is_none() {
@@ -1527,7 +1524,10 @@ pub async fn get_cached_resource_value_if_valid(
if let Some(object_store_resource) = object_store_resource_opt.clone() {
let etag = get_etag_or_empty(
&object_store_resource,
S3Object { s3: s3_file_key.clone() },
S3Object {
s3: s3_file_key.clone(),
storage: cached_resource.storage.clone(),
},
)
.await;
if etag.is_none() || etag.clone().unwrap() != s3_file_etag {
@@ -1552,7 +1552,8 @@ pub async fn save_in_cache(
let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64;
#[cfg(feature = "parquet")]
let s3_etags = arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await;
let (storage, s3_etags) =
arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await;
#[cfg(feature = "parquet")]
let s3_etags = if s3_etags.is_empty() {
@@ -1562,9 +1563,9 @@ pub async fn save_in_cache(
};
#[cfg(not(feature = "parquet"))]
let s3_etags = None;
let (storage, s3_etags) = (None, None);
let store_cache_resource = CachedResource { expire, s3_etags, value: r.clone() };
let store_cache_resource = CachedResource { expire, s3_etags, value: r.clone(), storage };
let raw_json = sqlx::types::Json(store_cache_resource);
if let Err(e) = sqlx::query!(
frontend/src/lib/components/AppConnectDrawer.svelte
+1 -1
View File
@@ -11,7 +11,7 @@
let step = 1
let disabled = false
let isGoogleSignin = false
let manual = false
let manual = true
let appConnectInner: AppConnectInner | undefined = undefined
frontend/src/lib/components/AppConnectInner.svelte
+9 -14
View File
@@ -29,7 +29,7 @@
export let resourceType = ''
export let isGoogleSignin = false
export let disabled = false
export let manual = false
export let manual = true
let isValid = true
@@ -75,24 +75,18 @@
let pathError = ''
export async function open(rt?: string) {
if (!rt) {
loadResourceTypes()
}
step = 1
value = ''
description = ''
resourceType = rt ?? ''
valueToken = undefined
await loadConnects()
const isConnect = connects?.includes(resourceType)
if (isConnect) {
manual = false
manual = !connects?.includes(resourceType)
if (rt) {
next()
} else {
manual = true
if (rt) {
next()
} else {
loadResourceTypes()
}
}
}
@@ -190,10 +184,11 @@
export async function next() {
if (step == 1) {
if (manual) {
await getResourceTypeInfo()
getResourceTypeInfo()
args = {}
} else {
await Promise.all([getScopesAndParams(), getResourceTypeInfo()])
getResourceTypeInfo()
getScopesAndParams()
}
step += 1
} else if (step == 2 && !manual) {
frontend/src/lib/components/DisplayResult.svelte
+47 -12
View File
@@ -11,7 +11,8 @@
Table2,
Braces,
Highlighter,
InfoIcon
InfoIcon,
ArrowDownFromLine
} from 'lucide-svelte'
import Portal from 'svelte-portal'
import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
@@ -22,13 +23,14 @@
import Toggle from './Toggle.svelte'
import FileDownload from './common/fileDownload/FileDownload.svelte'
import ParqetTableRenderer from './ParqetTableRenderer.svelte'
import ParqetTableRenderer from './ParqetCsvTableRenderer.svelte'
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import MapResult from './MapResult.svelte'
import Popover from './Popover.svelte'
import DownloadCsv from './table/DownloadCsv.svelte'
import { convertJsonToCsv } from './table/tableUtils'
import Tooltip from './Tooltip.svelte'
export let result: any
export let requireHtmlApproval = false
@@ -108,7 +110,7 @@
let largeObject: boolean | undefined = undefined
function checkIfS3(result: any, keys: string[]) {
return keys.length === 1 && keys.includes('s3') && typeof result.s3 === 'string'
return keys.includes('s3') && typeof result.s3 === 'string'
}
let is_render_all = false
@@ -335,6 +337,8 @@
}
let globalForceJson: boolean = false
let seeS3PreviewFileFromList = ''
</script>
{#if is_render_all}
@@ -554,8 +558,9 @@
class="flex"
bind:checked={s3FileDisplayRawMode}
size="xs"
options={{ right: 'Raw S3 object input' }}
options={{ right: 'Raw S3 object' }}
/>
{#if s3FileDisplayRawMode}
<Highlight class="" language={json} code={toJsonStr(result).replace(/\\n/g, '\n')} />
<button
@@ -564,10 +569,13 @@
s3FileViewer?.open?.(result)
}}
><span class="flex items-center gap-1"
><PanelRightOpen size={12} />open preview</span
><PanelRightOpen size={12} />object store explorer<Tooltip
>Require admin privilege or "S3 resource details and content can be accessed by
all users of this workspace" of S3 Storage to be set in the workspace settings</Tooltip
></span
>
</button>
{:else}
{:else if !result?.disable_download}
<FileDownload s3object={result} />
<button
class="text-secondary underline text-2xs whitespace-nowrap"
@@ -575,23 +583,31 @@
s3FileViewer?.open?.(result)
}}
><span class="flex items-center gap-1"
><PanelRightOpen size={12} />open preview</span
><PanelRightOpen size={12} />object store explorer<Tooltip
>Require admin privilege or "S3 resource details and content can be accessed by
all users of this workspace" of S3 Storage to be set in the workspace settings</Tooltip
></span
>
</button>
{/if}
</div>
{#if typeof result?.s3 == 'string' && result?.s3?.endsWith('.parquet')}
<ParqetTableRenderer s3resource={result?.s3} />
{#if typeof result?.s3 == 'string' && (result?.s3?.endsWith('.parquet') || result?.s3?.endsWith('.csv'))}
<ParqetTableRenderer
disable_download={result?.disable_download}
{workspaceId}
s3resource={result?.s3}
storage={result?.storage}
/>
{/if}
</div>
{:else if !forceJson && resultKind == 's3object-list'}
<div class="h-full w-full">
<div class="flex flex-col gap-2">
<Toggle
class="flex"
class="flex mt-1"
bind:checked={s3FileDisplayRawMode}
size="xs"
options={{ right: 'Raw S3 object input' }}
options={{ right: 'Raw S3 object' }}
/>
{#each result as s3object}
{#if s3FileDisplayRawMode}
@@ -609,8 +625,27 @@
><PanelRightOpen size={12} />open preview</span
>
</button>
{:else}
{:else if !s3object?.disable_download}
<FileDownload {s3object} />
{:else}
<div class="flex text-secondary pt-2">{s3object?.s3} (download disabled)</div>
{/if}
{#if s3object?.s3?.endsWith('.parquet') || s3object?.s3?.endsWith('.csv')}
{#if seeS3PreviewFileFromList == s3object?.s3}
<ParqetTableRenderer
disable_download={s3object?.disable_download}
{workspaceId}
s3resource={s3object?.s3}
storage={s3object?.storage}
/>{:else}
<button
class="text-secondary whitespace-nowrap flex gap-2 items-center"
on:click={() => {
seeS3PreviewFileFromList = s3object?.s3
}}
>open table preview <ArrowDownFromLine />
</button>
{/if}
{/if}
{/each}
</div>
frontend/src/lib/components/ParqetTableRenderer.svelte → frontend/src/lib/components/ParqetCsvTableRenderer.svelte
+34 -7
View File
@@ -7,27 +7,38 @@
import DarkModeObserver from './DarkModeObserver.svelte'
import { HelpersService } from '$lib/gen'
import { enterpriseLicense, workspaceStore } from '$lib/stores'
import { Download } from 'lucide-svelte'
// import 'ag-grid-community/dist/styles/ag-theme-alpine-dark.css'
let selectedRowIndex = -1
export let s3resource: string
export let storage: string | undefined
export let workspaceId: string | undefined
export let disable_download: boolean = false
let datasource: IDatasource = {
rowCount: 0,
getRows: async function (params) {
try {
const searchCol = params.filterModel ? Object.keys(params.filterModel)?.[0] : undefined
const res = (await HelpersService.loadParquetPreview({
workspace: $workspaceStore!,
const requestBody = {
workspace: workspaceId ?? $workspaceStore!,
path: s3resource,
offset: params.startRow,
limit: params.endRow - params.startRow,
sortCol: params.sortModel?.[0]?.colId,
sortDesc: params.sortModel?.[0]?.sort == 'desc',
searchCol: searchCol,
searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined
})) as any
searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined,
storage: storage
}
const csv = s3resource.endsWith('.csv')
const res = (
csv
? await HelpersService.loadCsvPreview(requestBody)
: await HelpersService.loadParquetPreview(requestBody)
) as any
for (let i = 0; i < res.rows.length; i++) {
res.rows[i]['__index'] = i + params.startRow
if (!$enterpriseLicense) {
@@ -71,7 +82,8 @@
const res = await HelpersService.loadParquetPreview({
workspace: $workspaceStore!,
path: s3resource,
limit: 0
limit: 0,
storage: storage
})
createGrid(
@@ -124,8 +136,23 @@
<DarkModeObserver bind:darkMode />
<div class={twMerge('mt-4 border shadow-sm divide-y flex flex-col h-full min-h-[600px]')}>
<div class="ag-theme-alpine h-full" class:ag-theme-alpine-dark={darkMode} style="height: 600px;">
<div class={twMerge('mt-2 flex flex-col h-full min-h-[600px]')}>
{#if !disable_download && !s3resource.endsWith('.csv')}
<a
target="_blank"
href="/api/w/{workspaceId}/job_helpers/download_s3_parquet_file_as_csv?file_key={s3resource}{storage
? `&storage=${storage}`
: ''}"
class="text-secondary w-full text-right underline text-2xs whitespace-nowrap"
><div class="flex flex-row-reverse gap-2 items-center"><Download size={12} /> CSV</div></a
>
{/if}
<div
class="ag-theme-alpine shadow-sm h-full"
class:ag-theme-alpine-dark={darkMode}
style="height: 600px;"
>
<div bind:this={eGui} style="height:100%; " />
</div>
</div>
frontend/src/lib/components/S3FilePicker.svelte
+20 -8
View File
@@ -116,7 +116,8 @@
workspace: $workspaceStore!,
maxKeys: maxKeys, // fixed pages of 1000 files for now
marker: page == 0 ? undefined : listMarkers[page - 1],
prefix: filter.trim() != '' ? filter : undefined
prefix: filter.trim() != '' ? filter : undefined,
storage: storage
})
if (
availableFiles.restricted_access === null ||
@@ -195,7 +196,8 @@
fileInfoLoading = true
let fileMetadataRaw = await HelpersService.loadFileMetadata({
workspace: $workspaceStore!,
fileKey: fileKey
fileKey: fileKey,
storage: storage
})
if (fileMetadataRaw !== undefined) {
@@ -221,7 +223,8 @@
csvSeparator: csvSeparatorChar,
csvHasHeader: csvHasHeader,
readBytesFrom: 0,
readBytesLength: 128 * 1024 // For now static limit of 128Kb per file
readBytesLength: 128 * 1024, // For now static limit of 128Kb per file,
storage: storage
})
let filePreviewContent = filePreviewRaw.content
@@ -254,7 +257,8 @@
try {
await HelpersService.deleteS3File({
workspace: $workspaceStore!,
fileKey: fileKey
fileKey: fileKey,
storage: storage
})
} finally {
fileDeletionInProgress = false
@@ -279,7 +283,8 @@
await HelpersService.moveS3File({
workspace: $workspaceStore!,
srcFileKey: srcFileKey,
destFileKey: destFileKey!
destFileKey: destFileKey!,
storage: storage
})
} finally {
fileMoveInProgress = false
@@ -291,7 +296,11 @@
await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
}
export async function open(preSelectedFileKey: { s3: string } | undefined = undefined) {
let storage: string | undefined = undefined
export async function open(
preSelectedFileKey: { s3: string; storage: string | undefined } | undefined = undefined
) {
storage = preSelectedFileKey?.storage
if (preSelectedFileKey !== undefined) {
initialFileKey = { ...preSelectedFileKey }
selectedFileKey = { ...preSelectedFileKey }
@@ -313,10 +322,13 @@
initialFileKeyInternalCopy = { ...initialFileKey }
}
try {
await HelpersService.datasetStorageTestConnection({ workspace: $workspaceStore! })
await HelpersService.datasetStorageTestConnection({
workspace: $workspaceStore!,
storage: storage
})
workspaceSettingsInitialized = true
} catch (e) {
console.error('Workspace not connected to S3 bucket: ', e)
console.error('Workspace not connected to object storage: ', e)
workspaceSettingsInitialized = false
return
}
frontend/src/lib/components/common/fileDownload/FileDownload.svelte
+4 -2
View File
@@ -10,9 +10,11 @@
border border-dashed border-gray-400 hover:border-blue-500
focus-within:border-blue-500 hover:bg-blue-50 dark:hover:bg-frost-900 focus-within:bg-blue-50
duration-200 rounded-lg p-1 gap-2"
href={`/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${s3object?.s3}`}
href={`/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${s3object?.s3}${
s3object?.storage ? `&storage=${s3object.storage}` : ''
}`}
download={s3object?.s3.split('/').pop() ?? 'unnamed_download.file'}
>
<Download />
<span>s3://{s3object.s3}</span>
<span>s3://{s3object.s3} {s3object.storage ? ` (${s3object.storage})` : ''}</span>
</a>
frontend/src/lib/components/propertyPicker/ObjectViewer.svelte
+6 -2
View File
@@ -109,7 +109,9 @@
<button
class="val text-left {pureViewer
? 'cursor-auto'
: ''} rounded px-1 hover:bg-blue-100 dark:hover:bg-blue-100/10 {getTypeAsString(json[key])}"
: ''} rounded px-1 hover:bg-blue-100 dark:hover:bg-blue-100/10 {getTypeAsString(
json[key]
)}"
on:click={() => selectProp(key, json[key])}
>
{#if json[key] === NEVER_TESTED_THIS_FAR}
@@ -142,7 +144,9 @@
{#if getTypeAsString(json) === 's3object'}
<a
class="text-secondary underline font-semibold text-2xs whitespace-nowrap ml-1 w-fit"
href={`/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${json?.s3}`}
href={`/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${json?.s3}${
json?.storage ? `&storage=${json.storage}` : ''
}`}
download={json?.s3.split('/').pop() ?? 'unnamed_download.file'}
>
<span class="flex items-center gap-1"><Download size={12} />download</span>
frontend/src/lib/workspace_settings.ts
+101
View File
@@ -0,0 +1,101 @@
import type { GetSettingsResponse, LargeFileStorage } from './gen'
import { emptyString } from './utils'
type s3type = 's3' | 'azure_blob' | 's3_aws_oidc' | 'azure_workload_identity'
type s3ResourceSettingsItem = {
resourceType: s3type
resourcePath: string | undefined
publicResource: boolean | undefined
}
export type S3ResourceSettings = s3ResourceSettingsItem & {
secondaryStorage: [string, s3ResourceSettingsItem][] | undefined
}
export function convertBackendSettingsToFrontendSettings(
large_file_storage: GetSettingsResponse['large_file_storage']
): S3ResourceSettings {
let settings: Partial<S3ResourceSettings> =
convertBackendSettingsToFrontendSettingsItem(large_file_storage)
settings.secondaryStorage = Object.entries(large_file_storage?.secondary_storage ?? {}).map(
([key, value]) => [key, convertBackendSettingsToFrontendSettingsItem(value)]
)
return settings as S3ResourceSettings
}
export function convertBackendSettingsToFrontendSettingsItem(
large_file_storage: GetSettingsResponse['large_file_storage']
): s3ResourceSettingsItem {
if (large_file_storage?.type === 'S3Storage') {
return {
resourceType: 's3',
resourcePath: large_file_storage?.s3_resource_path?.replace('$res:', ''),
publicResource: large_file_storage?.public_resource
}
} else if (large_file_storage?.type === 'AzureBlobStorage') {
return {
resourceType: 'azure_blob',
resourcePath: large_file_storage?.azure_blob_resource_path?.replace('$res:', ''),
publicResource: large_file_storage?.public_resource
}
} else if (large_file_storage?.type === 'AzureWorkloadIdentity') {
return {
resourceType: 'azure_workload_identity',
resourcePath: large_file_storage?.azure_blob_resource_path?.replace('$res:', ''),
publicResource: large_file_storage?.public_resource
}
} else if (large_file_storage?.type === 'S3AwsOidc') {
return {
resourceType: 's3_aws_oidc',
resourcePath: large_file_storage?.s3_resource_path?.replace('$res:', ''),
publicResource: large_file_storage?.public_resource
}
} else {
return {
resourceType: 's3',
resourcePath: undefined,
publicResource: undefined
}
}
}
export function convertFrontendToBackendSetting(
s3ResourceSettings: S3ResourceSettings
): LargeFileStorage | undefined {
let settings = convertFrontendToBackendettingsItem(s3ResourceSettings)
if (settings) {
settings.secondary_storage = Object.fromEntries(
(s3ResourceSettings.secondaryStorage ?? [])
.map(([key, value]) => [key, convertFrontendToBackendettingsItem(value)])
.filter(([, value]) => value !== undefined)
)
}
return settings
}
export function convertFrontendToBackendettingsItem(
s3ResourceSettings: s3ResourceSettingsItem
): LargeFileStorage | undefined {
if (!emptyString(s3ResourceSettings.resourcePath)) {
let resourcePathWithPrefix = `$res:${s3ResourceSettings.resourcePath}`
let params = {
public_resource: s3ResourceSettings.publicResource
}
if (s3ResourceSettings.resourceType === 'azure_blob') {
let typ: LargeFileStorage['type'] = 'AzureBlobStorage'
params['type'] = typ
params['azure_blob_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 'azure_workload_identity') {
let typ: LargeFileStorage['type'] = 'AzureWorkloadIdentity'
params['type'] = typ
params['azure_blob_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 's3_aws_oidc') {
let typ: LargeFileStorage['type'] = 'S3AwsOidc'
params['type'] = typ
params['s3_resource_path'] = resourcePathWithPrefix
} else {
let typ: LargeFileStorage['type'] = 'S3Storage'
params['type'] = typ
params['s3_resource_path'] = resourcePathWithPrefix
}
return params
}
}
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+92 -80
View File
@@ -15,13 +15,7 @@
import Tooltip from '$lib/components/Tooltip.svelte'
import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte'
import { WORKSPACE_SHOW_SLACK_CMD, WORKSPACE_SHOW_WEBHOOK_CLI_SYNC } from '$lib/consts'
import {
type LargeFileStorage,
OauthService,
WorkspaceService,
JobService,
ResourceService
} from '$lib/gen'
import { OauthService, WorkspaceService, JobService, ResourceService } from '$lib/gen'
import {
enterpriseLicense,
copilotInfo,
@@ -53,6 +47,11 @@
import { fade } from 'svelte/transition'
import ChangeWorkspaceName from '$lib/components/settings/ChangeWorkspaceName.svelte'
import ChangeWorkspaceId from '$lib/components/settings/ChangeWorkspaceId.svelte'
import {
convertBackendSettingsToFrontendSettings,
convertFrontendToBackendSetting,
type S3ResourceSettings
} from '$lib/workspace_settings'
type GitSyncTypeMap = {
scripts: boolean
@@ -97,10 +96,12 @@
let errorHandlerExtraArgs: Record<string, any> = {}
let errorHandlerMutedOnCancel: boolean | undefined = undefined
let openaiResourceInitialPath: string | undefined = undefined
let s3ResourceSettings: {
resourceType: 's3' | 'azure_blob' | 's3_aws_oidc' | 'azure_workload_identity'
resourcePath: string | undefined
publicResource: boolean | undefined
let s3ResourceSettings: S3ResourceSettings = {
resourceType: 's3',
resourcePath: undefined,
publicResource: undefined,
secondaryStorage: undefined
}
let gitSyncSettings: {
include_path: string[]
@@ -226,44 +227,15 @@
}
async function editWindmillLFSSettings(): Promise<void> {
if (!emptyString(s3ResourceSettings.resourcePath)) {
let resourcePathWithPrefix = `$res:${s3ResourceSettings.resourcePath}`
let params = {
public_resource: s3ResourceSettings.publicResource
const large_file_storage = convertFrontendToBackendSetting(s3ResourceSettings)
await WorkspaceService.editLargeFileStorageConfig({
workspace: $workspaceStore!,
requestBody: {
large_file_storage: large_file_storage
}
if (s3ResourceSettings.resourceType === 'azure_blob') {
let typ: LargeFileStorage['type'] = 'AzureBlobStorage'
params['type'] = typ
params['azure_blob_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 'azure_workload_identity') {
let typ: LargeFileStorage['type'] = 'AzureWorkloadIdentity'
params['type'] = typ
params['azure_blob_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 's3_aws_oidc') {
let typ: LargeFileStorage['type'] = 'S3AwsOidc'
params['type'] = typ
params['s3_resource_path'] = resourcePathWithPrefix
} else {
let typ: LargeFileStorage['type'] = 'S3Storage'
params['type'] = typ
params['s3_resource_path'] = resourcePathWithPrefix
}
await WorkspaceService.editLargeFileStorageConfig({
workspace: $workspaceStore!,
requestBody: {
large_file_storage: params
}
})
sendUserToast(`Large file storage settings updated`)
} else {
await WorkspaceService.editLargeFileStorageConfig({
workspace: $workspaceStore!,
requestBody: {
large_file_storage: undefined
}
})
sendUserToast(`Large file storage settings reset`)
}
})
console.log('Large file storage settings changed', large_file_storage)
sendUserToast(`Large file storage settings changed`)
}
async function editWindmillGitSyncSettings(): Promise<void> {
@@ -440,37 +412,8 @@
codeCompletionEnabled = settings.code_completion_enabled
workspaceDefaultAppPath = settings.default_app
if (settings.large_file_storage?.type === 'S3Storage') {
s3ResourceSettings = {
resourceType: 's3',
resourcePath: settings.large_file_storage?.s3_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else if (settings.large_file_storage?.type === 'AzureBlobStorage') {
s3ResourceSettings = {
resourceType: 'azure_blob',
resourcePath: settings.large_file_storage?.azure_blob_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else if (settings.large_file_storage?.type === 'AzureWorkloadIdentity') {
s3ResourceSettings = {
resourceType: 'azure_workload_identity',
resourcePath: settings.large_file_storage?.azure_blob_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else if (settings.large_file_storage?.type === 'S3AwsOidc') {
s3ResourceSettings = {
resourceType: 's3_aws_oidc',
resourcePath: settings.large_file_storage?.s3_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else {
s3ResourceSettings = {
resourceType: 's3',
resourcePath: undefined,
publicResource: undefined
}
}
s3ResourceSettings = convertBackendSettingsToFrontendSettings(settings.large_file_storage)
if (settings.git_sync !== undefined && settings.git_sync !== null) {
gitSyncTestJobs = []
gitSyncSettings = {
@@ -1114,7 +1057,7 @@
right:
'S3 resource details and content can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the S3 bucket, as well as the resource details. this effectively by-pass the permissions set on the resource and makes it public to everyone.'
'If set, all users of this workspace will have access the to entire content of the S3 bucket, as well as the resource details and the "open preview" button. This effectively by-pass the permissions set on the resource and makes it public to everyone.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
@@ -1150,6 +1093,75 @@
{/if}
</div>
{/if}
<div class="mt-6">
<div class="flex mt-2 flex-col gap-y-4 max-w-3xl">
{#each s3ResourceSettings.secondaryStorage ?? [] as secondaryStorage, idx}
<div class="flex gap-1 items-center">
<input
class="max-w-[200px]"
type="text"
bind:value={secondaryStorage[0]}
placeholder="Storage name"
/>
<select class="max-w-[125px]" bind:value={secondaryStorage[1].resourceType}>
<option value="s3">S3</option>
<option value="azure_blob">Azure Blob</option>
<option value="s3_aws_oidc">AWS OIDC</option>
<option value="azure_workload_identity">Azure Workload Identity</option>
</select>
<ResourcePicker
resourceType={secondaryStorage[1].resourceType}
bind:value={secondaryStorage[1].resourcePath}
/>
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(secondaryStorage[1].resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.({ s3: '', storage: secondaryStorage[0] })
}
}}>Browse content (save first)</Button
>
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover ml-2"
aria-label="Clear"
on:click={() => {
if (s3ResourceSettings.secondaryStorage) {
s3ResourceSettings.secondaryStorage.splice(idx, 1)
s3ResourceSettings.secondaryStorage = [...s3ResourceSettings.secondaryStorage]
}
}}
>
<X size={14} />
</button>
</div>
{/each}
<div class="flex gap-1">
<Button
size="xs"
variant="border"
on:click={() => {
if (s3ResourceSettings.secondaryStorage === undefined) {
s3ResourceSettings.secondaryStorage = []
}
s3ResourceSettings.secondaryStorage.push([
`storage_${s3ResourceSettings.secondaryStorage.length + 1}`,
{ resourcePath: '', resourceType: 's3', publicResource: false }
])
s3ResourceSettings.secondaryStorage = s3ResourceSettings.secondaryStorage
}}><Plus size={14} />Add secondary storage</Button
>
<Tooltip>
Secondary storage is a feature that allows you to read and write from storage that
isn't your main storage by specifying it in the s3 object as "secondary_storage"
with the name of it
</Tooltip>
</div>
</div>
</div>
<div class="flex mt-5 mb-5 gap-1">
<Button
color="blue"
frontend/src/routes/(root)/embed_connect/+page.svelte
+1 -1
View File
@@ -11,7 +11,7 @@
let step = 1
let disabled = false
let isGoogleSignin = false
let manual = false
let manual = true
let appConnect: AppConnectInner | undefined = undefined
python-client/wmill/wmill/client.py
+3 -1
View File
@@ -440,7 +440,7 @@ class Windmill:
print(file_reader.read())
'''
"""
reader = S3BufferedReader(f"{self.workspace}", self.client, s3object["s3"], s3_resource_path)
reader = S3BufferedReader(f"{self.workspace}", self.client, s3object["s3"], s3_resource_path, s3object["storage"])
return reader
def write_s3_file(
@@ -479,6 +479,8 @@ class Windmill:
query_params["file_key"] = s3object["s3"]
if s3_resource_path is not None and s3_resource_path != "":
query_params["s3_resource_path"] = s3_resource_path
if s3object is not None and s3object["storage"] is not None:
query_params["storage"] = s3object["storage"]
try:
# need a vanilla client b/c content-type is not application/json here
python-client/wmill/wmill/s3_reader.py
+3 -1
View File
@@ -5,12 +5,14 @@ import httpx
class S3BufferedReader(BufferedReader):
def __init__(self, workspace: str, windmill_client: httpx.Client, file_key: str, s3_resource_path: str | None):
def __init__(self, workspace: str, windmill_client: httpx.Client, file_key: str, s3_resource_path: str | None, storage: str | None):
params = {
"file_key": file_key,
}
if s3_resource_path is not None:
params["s3_resource_path"] = s3_resource_path
if storage is not None:
params["storage"] = storage
self._context_manager = windmill_client.stream(
"GET",
f"/w/{workspace}/job_helpers/download_s3_file",
python-client/wmill/wmill/s3_types.py
+1
View File
@@ -1,5 +1,6 @@
class S3Object(dict):
s3: str
storage: str | None
def __getattr__(self, attr):
return self[attr]
typescript-client/client.ts
+3
View File
@@ -563,6 +563,9 @@ export async function loadS3FileStream(
if (s3ResourcePath !== undefined) {
params["s3_resource_path"] = s3ResourcePath;
}
if (s3object.storage !== undefined) {
params["storage"] = s3object.storage;
}
const queryParams = new URLSearchParams(params);
// We use raw fetch here b/c OpenAPI generated client doesn't handle Blobs nicely
typescript-client/s3Types.ts
+12 -12
View File
@@ -1,14 +1,14 @@
export type S3Object = {
s3: string
}
s3: string;
storage?: string;
};
export type DenoS3LightClientSettings = {
endPoint: string,
region: string ,
bucket?: string,
useSSL?: boolean,
accessKey?: string,
secretKey?: string,
pathStyle?: boolean,
}
endPoint: string;
region: string;
bucket?: string;
useSSL?: boolean;
accessKey?: string;
secretKey?: string;
pathStyle?: boolean;
};