feat: File path is option when uploading a file to S3 (#3029)

* feat: File path is option when uploading a file to S3

* Add frontend

* Fix lock

* Drag and drop file upload UI
This commit is contained in:
Guillaume Bouvignies
2024-01-19 14:17:42 +01:00
committed by GitHub
parent 00f4a65929
commit 326eac46fc
21 changed files with 543 additions and 568 deletions
@@ -48,7 +48,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -69,7 +69,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -28,7 +28,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -62,7 +62,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "?column?",
"name": "bool",
"type_info": "Bool"
}
],
@@ -37,7 +37,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE instance_group SET name = $1 where name = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "a28a83edc40e32815cb465338b53c8e892ac4fac6d78bc825cd0b0b1099f4e07"
}
@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",
+7 -3
View File
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 1.246.4
version: 1.246.15
title: Windmill API
contact:
name: Windmill Team
@@ -10503,6 +10503,8 @@ paths:
properties:
file_key:
type: string
file_extension:
type: string
part_content:
type: array
items:
@@ -10528,8 +10530,7 @@ paths:
s3_resource_path:
type: string
required:
- file_key
- part_content_base64
- part_content
- parts
- is_final
- cancel_upload
@@ -10551,7 +10552,10 @@ paths:
required: *ref_97
is_done:
type: boolean
file_key:
type: string
required:
- file_key
- upload_id
- parts
- is_done
+6 -2
View File
@@ -7070,6 +7070,8 @@ paths:
properties:
file_key:
type: string
file_extension:
type: string
part_content:
type: array
items:
@@ -7087,8 +7089,7 @@ paths:
s3_resource_path:
type: string
required:
- file_key
- part_content_base64
- part_content
- parts
- is_final
- cancel_upload
@@ -7108,7 +7109,10 @@ paths:
$ref: "#/components/schemas/UploadFilePart"
is_done:
type: boolean
file_key:
type: string
required:
- file_key
- upload_id
- parts
- is_done
+26 -5
View File
@@ -1,3 +1,4 @@
use std::time::{SystemTime, UNIX_EPOCH};
use std::{cmp, time::Duration};
use crate::{db::DB, resources::get_resource_value_interpolated_internal, users::Tokened};
@@ -741,7 +742,8 @@ async fn move_s3_file(
#[derive(Deserialize)]
struct UploadFileQuery {
pub file_key: String,
pub file_key: Option<String>, // if none, the file will be placed in windmill_uploads/ with a random name.
pub file_extension: Option<String>, // preferred extension for the file in case a random name has to be generated
pub part_content: Vec<u8>,
pub upload_id: Option<String>, // should be None for the first call to initiate the upload
@@ -764,6 +766,7 @@ struct UploadFileResponse {
pub upload_id: String,
pub parts: Vec<UploadFilePart>, // parts already uploaded, with their part_number and the tag associated
pub is_done: bool, // whether the transfer is finished, either b/c it got cancelled or because the last chunk was uploaded
pub file_key: String,
}
async fn multipart_upload_s3_file(
@@ -779,7 +782,23 @@ async fn multipart_upload_s3_file(
query.parts.len(),
query.is_final
);
let file_key = query.file_key.clone();
let file_key = match query.file_key.clone() {
Some(fk) => fk,
None => {
// for now, we place all files into `windmill_uploads` folder with a random name
// TODO: make the folder configurable via the workspace settings
format!(
"windmill_uploads/upload_{}_{}.{}",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis(),
rand::random::<u16>(),
query.file_extension.unwrap_or("file".to_string())
)
.to_string()
}
};
let s3_resource_opt = match query.s3_resource_path.clone() {
Some(s3_resource_path) => {
@@ -816,9 +835,10 @@ async fn multipart_upload_s3_file(
error::Error::InternalErr(err.to_string())
})?;
return Ok(Json(UploadFileResponse {
upload_id: upload_id,
upload_id,
parts: vec![], // empty parts as the transfer has been cancelled
is_done: true,
file_key,
}));
}
@@ -885,7 +905,7 @@ async fn multipart_upload_s3_file(
let _complete_multipart_upload_res = s3_client
.complete_multipart_upload()
.bucket(&s3_resource.bucket)
.key(&query.file_key)
.key(&file_key)
.upload_id(&upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
@@ -901,9 +921,10 @@ async fn multipart_upload_s3_file(
}
return Ok(Json(UploadFileResponse {
upload_id: upload_id,
upload_id,
parts: new_parts,
is_done: query.is_final,
file_key,
}));
}
+2
View File
@@ -11,6 +11,8 @@ services:
- db_data:/var/lib/postgresql/data
expose:
- 5432
ports:
- 5432:5432
environment:
POSTGRES_PASSWORD: changeme
POSTGRES_DB: windmill
+44 -19
View File
@@ -27,6 +27,7 @@
import DateTimeInput from './DateTimeInput.svelte'
import S3FilePicker from './S3FilePicker.svelte'
import CurrencyInput from './apps/components/inputs/currency/CurrencyInput.svelte'
import FileUpload from './common/fileUpload/FileUpload.svelte'
export let label: string = ''
export let value: any
@@ -78,6 +79,7 @@
let error: string = ''
let s3FilePicker: S3FilePicker
let s3FileUploadRawMode: false
let el: HTMLTextAreaElement | undefined = undefined
@@ -475,26 +477,49 @@
.replace('_', '')
.toLowerCase() == 's3object'}
<div class="flex flex-col w-full gap-1">
<JsonEditor
bind:editor
on:focus={(e) => {
dispatch('focus')
}}
code={JSON.stringify({ s3: '' }, null, 2)}
bind:value
/>
<Button
variant="border"
color="light"
<Toggle
class="flex justify-end"
bind:checked={s3FileUploadRawMode}
size="xs"
btnClasses="mt-1"
on:click={() => {
s3FilePicker?.open?.(value)
}}
startIcon={{ icon: Pipette }}
>
Choose an object from the catalog
</Button>
options={{ left: 'Raw S3 object input' }}
/>
{#if s3FileUploadRawMode}
<JsonEditor
bind:editor
on:focus={(e) => {
dispatch('focus')
}}
code={JSON.stringify({ s3: '' }, null, 2)}
bind:value
/>
<Button
variant="border"
color="light"
size="xs"
btnClasses="mt-1"
on:click={() => {
s3FilePicker?.open?.(value)
}}
startIcon={{ icon: Pipette }}
>
Choose an object from the catalog
</Button>
{:else}
<FileUpload
allowMultiple={false}
randomFileKey={true}
on:addition={(evt) => {
value = {
s3: evt.detail?.path ?? ''
}
}}
on:deletion={(evt) => {
value = {
s3: ''
}
}}
/>
{/if}
</div>
{:else if inputCat == 'object' || inputCat == 'resource-object'}
{#if properties && Object.keys(properties).length > 0}
@@ -11,7 +11,7 @@
MoveRight
} from 'lucide-svelte'
import { workspaceStore } from '$lib/stores'
import { HelpersService, type UploadFilePart } from '$lib/gen'
import { HelpersService } from '$lib/gen'
import { displayDate, displaySize, emptyString, sendUserToast } from '$lib/utils'
import { Alert, Button, Drawer } from './common'
import DrawerContent from './common/drawer/DrawerContent.svelte'
@@ -30,11 +30,6 @@
let fileMoveInProgress = false
let uploadModalOpen = false
let fileToUpload: File | undefined = undefined
let fileToUploadKey: string | undefined = undefined
let fileUploadProgress: number | undefined = undefined
let fileUploadCancelled: boolean = false
let fileUploadErrorMsg: string | undefined = undefined
let workspaceSettingsInitialized = true
@@ -251,81 +246,6 @@
await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
}
async function uploadFileToS3() {
fileUploadErrorMsg = undefined
if (fileToUpload === undefined || fileToUploadKey === undefined) {
return
}
if (allFilesByKey[fileToUploadKey] !== undefined) {
fileUploadErrorMsg =
'A file with this name already exists in the S3 bucket. If you want to replace it, delete it first.'
return
}
let upload_id: string | undefined = undefined
let parts: UploadFilePart[] = []
let reader = fileToUpload?.stream().getReader()
let { value: chunk, done: readerDone } = await reader.read()
if (chunk === undefined || readerDone) {
sendUserToast('Error reading file, no data read', true)
return
}
fileUploadProgress = 0
while (true) {
let { value: chunk_2, done: readerDone } = await reader.read()
if (!readerDone && chunk_2 !== undefined && chunk.length <= 5 * 1024 * 1024) {
// AWS enforces part to be bigger than 5MB, so we accumulate bytes until we reach that limit before triggering the request to the BE
chunk = new Uint8Array([...chunk, ...chunk_2])
continue
}
fileUploadProgress += (chunk.length * 100) / fileToUpload.size
let response = await HelpersService.multipartFileUpload({
workspace: $workspaceStore!,
requestBody: {
file_key: fileToUploadKey,
part_content: Array.from(chunk),
upload_id: upload_id,
parts: parts,
is_final: readerDone,
cancel_upload: fileUploadCancelled
}
})
upload_id = response.upload_id
parts = response.parts
if (response.is_done) {
if (fileUploadCancelled) {
sendUserToast('File upload cancelled!')
} else {
sendUserToast('File upload finished!')
}
break
}
if (chunk_2 === undefined) {
sendUserToast(
'File upload is not finished, yet there is no more data to stream. This is unexpected',
true
)
return
}
chunk = chunk_2
}
uploadModalOpen = false
if (!fileUploadCancelled) {
selectedFileKey = { s3: fileToUploadKey }
await loadFiles()
await loadFileMetadataPlusPreviewAsync(selectedFileKey['s3'])
}
fileToUpload = undefined
fileToUploadKey = undefined
fileUploadProgress = undefined
fileUploadCancelled = false
fileUploadErrorMsg = undefined
}
export async function open(preSelectedFileKey: { s3: string } | undefined = undefined) {
if (preSelectedFileKey !== undefined) {
initialFileKey = { ...preSelectedFileKey }
@@ -701,24 +621,12 @@
<FileUploadModal
open={uploadModalOpen}
title="Upload file to S3 bucket"
bind:fileToUpload
bind:fileKey={fileToUploadKey}
on:canceled={() => {
if (fileUploadProgress !== undefined) {
fileUploadCancelled = true
fileUploadErrorMsg = 'Cancelling in progress, it might take a few seconds...'
} else {
fileUploadErrorMsg = undefined
uploadModalOpen = false
on:close={async (evt) => {
uploadModalOpen = false
if (evt.detail !== undefined && evt.detail !== null) {
selectedFileKey = { s3: evt.detail }
loadFiles()
loadFileMetadataPlusPreviewAsync(evt.detail)
}
}}
on:confirmed={() => {
uploadFileToS3()
}}
on:close={() => {
fileUploadCancelled = true
uploadModalOpen = false
}}
bind:progressPct={fileUploadProgress}
bind:errorMsg={fileUploadErrorMsg}
/>
@@ -1,8 +1,5 @@
<script lang="ts">
import { getContext } from 'svelte'
import { twMerge } from 'tailwind-merge'
import { FileInput } from '../../../common'
import FileProgressBar from '../../../common/FileProgressBar.svelte'
import { initConfig, initOutput } from '../../editor/appUtils'
import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
@@ -10,12 +7,8 @@
import ResolveStyle from '../helpers/ResolveStyle.svelte'
import ResolveConfig from '../helpers/ResolveConfig.svelte'
import { components } from '../../editor/component'
import Button from '$lib/components/common/button/Button.svelte'
import { sendUserToast } from '$lib/toast'
import { workspaceStore } from '$lib/stores'
import { HelpersService, type UploadFilePart } from '$lib/gen'
import { writable, type Writable } from 'svelte/store'
import { Ban, CheckCheck, FileWarning, Files, RefreshCcw, Trash } from 'lucide-svelte'
import FileUpload from '$lib/components/common/fileUpload/FileUpload.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -54,166 +47,9 @@
jobId: undefined
})
async function handleChange(files: File[] | undefined) {
for (const file of files ?? []) {
uploadFileToS3(file, file.name)
}
}
$: resolvedConfigS3 = resolvedConfig.type.configuration.s3
let css = initCss($app.css?.fileinputcomponent, customCss)
let allFilesByKey: Record<
string,
{
type: 'folder' | 'leaf'
full_key: string
display_name: string
collapsed: boolean
parentPath: string | undefined
nestingLevel: number
}
> = {}
async function uploadFileToS3(fileToUpload: File, fileToUploadKey: string) {
if (fileToUpload === undefined || fileToUploadKey === undefined) {
return
}
let pathTemplate = resolvedConfig?.type?.configuration?.s3?.pathTemplate as any
const path =
typeof pathTemplate == 'function'
? (await pathTemplate?.({
file: fileToUpload
})) ?? fileToUploadKey
: fileToUploadKey
$fileUploads = $fileUploads.filter((fileUpload) => fileUpload.name !== fileToUpload.name)
const uploadData: FileUploadData = {
name: fileToUpload.name,
size: fileToUpload.size,
progress: 1, // We set it to 1 so that the progress bar is visible
cancelled: false,
path: path,
file: fileToUpload
}
if (allFilesByKey[fileToUploadKey] !== undefined) {
uploadData.errorMessage =
'A file with this name already exists in the S3 bucket. If you want to replace it, delete it first.'
$fileUploads = [...$fileUploads, uploadData]
return
}
$fileUploads = [...$fileUploads, uploadData]
let upload_id: string | undefined = undefined
let parts: UploadFilePart[] = []
let reader = fileToUpload?.stream().getReader()
let { value: chunk, done: readerDone } = await reader.read()
if (chunk === undefined || readerDone) {
sendUserToast('Error reading file, no data read', true)
return
}
let fileUploadProgress = 0
while (true) {
const currentFileUpload = $fileUploads.find(
(fileUpload) => fileUpload.name === uploadData.name
)!
if (currentFileUpload.cancelled) {
return
}
let { value: chunk_2, done: readerDone } = await reader.read()
if (!readerDone && chunk_2 !== undefined && chunk.length <= 5 * 1024 * 1024) {
// AWS enforces part to be bigger than 5MB, so we accumulate bytes until we reach that limit before triggering the request to the BE
chunk = new Uint8Array([...chunk, ...chunk_2])
continue
}
fileUploadProgress += (chunk.length * 100) / fileToUpload.size
uploadData.progress = fileUploadProgress
$fileUploads = $fileUploads.map((fileUpload) => {
if (fileUpload.name === uploadData.name) {
return uploadData
}
return fileUpload
})
try {
let response = await HelpersService.multipartFileUpload({
workspace: $workspaceStore!,
requestBody: {
file_key: path ?? fileToUploadKey,
part_content: Array.from(chunk),
upload_id: upload_id,
parts: parts,
is_final: readerDone,
cancel_upload: currentFileUpload.cancelled ?? false,
s3_resource_path: resolvedConfigS3 ? resolvedConfigS3.resource.split(':')[1] : undefined
}
})
upload_id = response.upload_id
parts = response.parts
if (response.is_done) {
if (currentFileUpload.cancelled) {
sendUserToast('File upload cancelled!')
} else {
const curr = outputs.result.peak()
outputs.result.set(
curr.concat({
path: path ?? fileToUploadKey
})
)
sendUserToast('File upload finished!')
}
break
}
if (chunk_2 === undefined) {
sendUserToast(
'File upload is not finished, yet there is no more data to stream. This is unexpected',
true
)
return
}
chunk = chunk_2
} catch (e) {
sendUserToast(e, true)
$fileUploads = $fileUploads.map((fileUpload) => {
if (fileUpload.name === uploadData.name) {
fileUpload.errorMessage = e
return fileUpload
}
return fileUpload
})
return
}
}
}
async function deleteFile(fileKey: string) {
await HelpersService.deleteS3File({
workspace: $workspaceStore!,
fileKey: fileKey
})
const curr = outputs.result.peak()
outputs.result.set(curr.filter((file) => file.path !== fileKey))
sendUserToast('File deleted!')
}
/*
{#if resolvedConfig.displayDirectLink && fileUpload.progress === 100}
@@ -272,196 +108,24 @@
{/if} -->
{#if render}
<div class="w-full h-full p-2 flex">
{#if $fileUploads.length > 0 && !forceDisplayUploads}
<div class="border rounded-md flex flex-col gap-1 divide-y h-full w-full p-1">
<div class="flex h-full overflow-y-auto flex-col">
{#each $fileUploads as fileUpload}
<div class="w-full flex flex-col gap-1 p-2">
<div class="flex flex-row items-center justify-between">
<div class="flex flex-col gap-1">
<span class="text-xs font-bold">{fileUpload.name}</span>
<span class="text-xs"
>{`${Math.round((fileUpload.size / 1024 / 1024) * 100) / 100} MB`}</span
>
</div>
<div class="flex flex-row gap-1 items-center">
{#if fileUpload.errorMessage}
<FileWarning class="w-4 h-4 text-red-500" />
{:else if fileUpload.cancelled}
<FileWarning class="w-4 h-4 text-yellow-500" />
{:else if fileUpload.progress === 100}
<CheckCheck class="w-4 h-4 text-green-500" />
{/if}
{#if fileUpload.cancelled || fileUpload.errorMessage !== undefined}
<Button
size="xs2"
color="light"
variant="border"
on:click={() => {
const file = fileUpload.file
if (!file) {
return
}
$fileUploads = $fileUploads.filter(
(_fileUpload) => _fileUpload.name !== fileUpload.name
)
uploadFileToS3(file, file.name)
}}
startIcon={{
icon: RefreshCcw
}}
>
Retry Upload
</Button>
<Button
size="xs2"
color="light"
variant="border"
on:click={() => {
const file = fileUpload.file
if (!file) {
return
}
$fileUploads = $fileUploads.filter(
(_fileUpload) => _fileUpload.name !== fileUpload.name
)
}}
startIcon={{
icon: RefreshCcw
}}
>
Remove from list
</Button>
{/if}
{#if fileUpload.progress < 100 && !fileUpload.cancelled && !fileUpload.errorMessage}
<Button
size="xs2"
color="light"
variant="border"
on:click={() => {
fileUpload.cancelled = true
fileUpload.progress = 0
}}
startIcon={{
icon: Ban
}}
>
Cancel Upload
</Button>
{/if}
{#if fileUpload.progress === 100 && !fileUpload.cancelled}
<Button
size="xs2"
color="red"
variant="border"
on:click={() => {
$fileUploads = $fileUploads.filter(
(_fileUpload) => _fileUpload.name !== fileUpload.name
)
if (fileUpload.path) {
deleteFile(fileUpload.path)
}
}}
startIcon={{
icon: Trash
}}
>
Delete
</Button>
{/if}
</div>
</div>
<FileProgressBar
progress={fileUpload.progress}
color={fileUpload.errorMessage
? '#ef4444'
: fileUpload.cancelled
? '#eab308'
: fileUpload.progress === 100
? '#22c55e'
: '#3b82f6'}
ended={fileUpload.cancelled || fileUpload.errorMessage !== undefined}
>
{#if fileUpload.errorMessage}
<span class="text-xs text-red-600">{fileUpload.errorMessage}</span>
{:else if fileUpload.cancelled}
<span class="text-xs text-yellow-600">Upload cancelled</span>
{/if}
</FileProgressBar>
{#if !(fileUpload.cancelled || fileUpload.errorMessage !== undefined)}
<span class="text-xs text-gray-500 dark:text-gray-200">
{fileUpload.progress === 100 ? 'Upload finished' : `Uploading`} to path: {fileUpload.path}
</span>
{/if}
</div>
{/each}
</div>
<div class="flex flex-row gap-1 items-center justify-end p-1">
{#if !$fileUploads.every((fileUpload) => fileUpload.progress === 100 || fileUpload.cancelled)}
<Button
size="xs2"
color="light"
on:click={() => {
$fileUploads = $fileUploads.map((fileUpload) => {
if (fileUpload.progress === 100 || fileUpload.cancelled) {
return fileUpload
}
fileUpload.cancelled = true
fileUpload.progress = 0
return fileUpload
})
}}
startIcon={{
icon: Ban
}}
>
Cancel All Uploads
</Button>
{/if}
<Button
size="xs2"
color="light"
on:click={() => {
forceDisplayUploads = true
}}
startIcon={{
icon: Files
}}
disabled={$fileUploads.some(
(fileUpload) => fileUpload.progress !== 100 && !fileUpload.cancelled
)}
>
Upload more files
</Button>
</div>
</div>
{:else}
<FileInput
accept={resolvedConfigS3.acceptedFileTypes?.length
? resolvedConfigS3.acceptedFileTypes?.join(', ')
: undefined}
multiple={resolvedConfigS3.allowMultiple}
returnFileNames
includeMimeType
on:change={({ detail }) => {
forceDisplayUploads = false
handleChange(detail)
}}
class={twMerge('w-full h-full', css?.container?.class, 'wm-file-input')}
style={css?.container?.style}
>
{resolvedConfigS3.text}
</FileInput>
{/if}
</div>
<FileUpload
acceptedFileTypes={resolvedConfigS3.acceptedFileTypes?.length
? resolvedConfigS3.acceptedFileTypes
: undefined}
pathTransformer={resolvedConfig?.type?.configuration?.s3?.pathTemplate}
allowMultiple={resolvedConfigS3.allowMultiple}
containerText={resolvedConfigS3.text}
customS3ResourcePath={resolvedConfigS3.resource}
customClass={css?.container?.class}
customStyle={css?.container?.style}
on:addition={(evt) => {
const curr = outputs.result.peak()
outputs.result.set(curr.concat(evt.detail))
}}
on:deletion={(evt) => {
const curr = outputs.result.peak()
outputs.result.set(curr.filter((file) => file.path !== evt.detail?.path))
}}
{forceDisplayUploads}
/>
{/if}
@@ -9,7 +9,7 @@
export let size: ButtonType.Size = 'md'
export let spacingSize: ButtonType.Size = size
export let color: ButtonType.Color | string = 'blue';
export let color: ButtonType.Color | string = 'blue'
export let variant: ButtonType.Variant = 'contained'
export let btnClasses: string = ''
export let wrapperClasses: string = ''
@@ -116,25 +116,25 @@
}
function getColorClass(color, variant) {
if (color in colorVariants) {
return colorVariants[color][variant];
} else {
return color;
}
}
if (color in colorVariants) {
return colorVariants[color][variant]
} else {
return color
}
}
$: buttonClass = twMerge(
'w-full',
getColorClass(color, variant),
variant === 'border' ? 'border' : '',
ButtonType.FontSizeClasses[size],
ButtonType.SpacingClasses[spacingSize][variant],
'focus:ring-2 font-semibold',
dropdownItems ? 'rounded-l-md h-full' : 'rounded-md',
'justify-center items-center text-center whitespace-nowrap inline-flex gap-2',
btnClasses,
'transition-all'
);
$: buttonClass = twMerge(
'w-full',
getColorClass(color, variant),
variant === 'border' ? 'border' : '',
ButtonType.FontSizeClasses[size],
ButtonType.SpacingClasses[spacingSize][variant],
'focus:ring-2 font-semibold',
dropdownItems ? 'rounded-l-md h-full' : 'rounded-md',
'justify-center items-center text-center whitespace-nowrap inline-flex gap-2',
btnClasses,
'transition-all'
)
const iconMap = {
xs: 14,
@@ -0,0 +1,368 @@
<script lang="ts">
import { FileInput } from '../'
import FileProgressBar from '../FileProgressBar.svelte'
import Button from '$lib/components/common/button/Button.svelte'
import { sendUserToast } from '$lib/toast'
import { workspaceStore } from '$lib/stores'
import { HelpersService, type UploadFilePart } from '$lib/gen'
import { writable, type Writable } from 'svelte/store'
import { Ban, CheckCheck, FileWarning, Files, RefreshCcw, Trash } from 'lucide-svelte'
import { twMerge } from 'tailwind-merge'
import { createEventDispatcher } from 'svelte'
import { emptyString } from '$lib/utils'
export let acceptedFileTypes: string[] | undefined = ['*']
export let allowMultiple: boolean = true
export let containerText: string = 'Drag and drop files here or click to browse'
export let customS3ResourcePath: string | undefined = undefined
export let customClass: string = ''
export let customStyle: string = ''
export let randomFileKey: boolean = false
export let pathTransformer: any = undefined // function taking as input {file: File} and returning a string
export let forceDisplayUploads: boolean = false
const dispatch = createEventDispatcher()
type FileUploadData = {
name: string
size: number
progress: number
cancelled?: boolean
errorMessage?: string
path?: string
file?: File
}
let fileUploads: Writable<FileUploadData[]> = writable([])
async function handleChange(files: File[] | undefined) {
for (const file of files ?? []) {
uploadFileToS3(file, file.name)
}
}
async function uploadFileToS3(fileToUpload: File, fileToUploadKey: string) {
if (fileToUpload === undefined || fileToUploadKey === undefined) {
return
}
let path: string | undefined = undefined
let fileExtension: string | undefined = undefined
if (randomFileKey) {
fileExtension = fileToUpload.name.split('.').pop()
if (emptyString(fileExtension)) {
fileExtension = undefined
}
} else {
path =
typeof pathTransformer == 'function'
? (await pathTransformer?.({
file: fileToUpload
})) ?? fileToUploadKey
: fileToUploadKey
}
const uploadData: FileUploadData = {
name: fileToUpload.name,
size: fileToUpload.size,
progress: 1, // We set it to 1 so that the progress bar is visible
cancelled: false,
path: path,
file: fileToUpload
}
$fileUploads = [...$fileUploads, uploadData]
let upload_id: string | undefined = undefined
let parts: UploadFilePart[] = []
let reader = fileToUpload?.stream().getReader()
let { value: chunk, done: readerDone } = await reader.read()
if (chunk === undefined || readerDone) {
sendUserToast('Error reading file, no data read', true)
return
}
let fileUploadProgress = 0
while (true) {
const currentFileUpload = $fileUploads.find(
(fileUpload) => fileUpload.name === uploadData.name
)!
if (currentFileUpload.cancelled) {
return
}
let { value: chunk_2, done: readerDone } = await reader.read()
if (!readerDone && chunk_2 !== undefined && chunk.length <= 5 * 1024 * 1024) {
// AWS enforces part to be bigger than 5MB, so we accumulate bytes until we reach that limit before triggering the request to the BE
chunk = new Uint8Array([...chunk, ...chunk_2])
continue
}
try {
let response = await HelpersService.multipartFileUpload({
workspace: $workspaceStore!,
requestBody: {
file_key: path,
file_extension: fileExtension,
part_content: Array.from(chunk),
upload_id: upload_id,
parts: parts,
is_final: readerDone,
cancel_upload: currentFileUpload.cancelled ?? false,
s3_resource_path:
customS3ResourcePath !== undefined ? customS3ResourcePath.split(':')[1] : undefined
}
})
uploadData.path = response.file_key
path = response.file_key
upload_id = response.upload_id
parts = response.parts
// update upload progress
fileUploadProgress += (chunk.length * 100) / fileToUpload.size
uploadData.progress = fileUploadProgress
$fileUploads = $fileUploads.map((fileUpload) => {
if (fileUpload.name === uploadData.name) {
return uploadData
}
return fileUpload
})
if (response.is_done) {
if (currentFileUpload.cancelled) {
sendUserToast('File upload cancelled!')
} else {
dispatch('addition', { path: path })
sendUserToast('File upload finished!')
}
break
}
if (chunk_2 === undefined) {
sendUserToast(
'File upload is not finished, yet there is no more data to stream. This is unexpected',
true
)
return
}
chunk = chunk_2
} catch (e) {
sendUserToast(e, true)
$fileUploads = $fileUploads.map((fileUpload) => {
if (fileUpload.name === uploadData.name) {
fileUpload.errorMessage = e
return fileUpload
}
return fileUpload
})
return
}
}
}
async function deleteFile(fileKey: string) {
await HelpersService.deleteS3File({
workspace: $workspaceStore!,
fileKey: fileKey
})
dispatch('deletion', { path: fileKey })
sendUserToast('File deleted!')
}
</script>
<div class="w-full h-full p-2 flex">
{#if $fileUploads.length > 0 && !forceDisplayUploads}
<div class="border rounded-md flex flex-col gap-1 divide-y h-full w-full p-1">
<div class="flex h-full overflow-y-auto flex-col">
{#each $fileUploads as fileUpload}
<div class="w-full flex flex-col gap-1 p-2">
<div class="flex flex-row items-center justify-between">
<div class="flex flex-col gap-1">
<span class="text-xs font-bold">{fileUpload.name}</span>
<span class="text-xs"
>{`${Math.round((fileUpload.size / 1024 / 1024) * 100) / 100} MB`}</span
>
</div>
<div class="flex flex-row gap-1 items-center">
{#if fileUpload.errorMessage}
<FileWarning class="w-4 h-4 text-red-500" />
{:else if fileUpload.cancelled}
<FileWarning class="w-4 h-4 text-yellow-500" />
{:else if fileUpload.progress === 100}
<CheckCheck class="w-4 h-4 text-green-500" />
{/if}
{#if fileUpload.cancelled || fileUpload.errorMessage !== undefined}
<Button
size="xs2"
color="light"
variant="border"
on:click={() => {
const file = fileUpload.file
if (!file) {
return
}
$fileUploads = $fileUploads.filter(
(_fileUpload) => _fileUpload.name !== fileUpload.name
)
uploadFileToS3(file, file.name)
}}
startIcon={{
icon: RefreshCcw
}}
>
Retry Upload
</Button>
<Button
size="xs2"
color="light"
variant="border"
on:click={() => {
const file = fileUpload.file
if (!file) {
return
}
$fileUploads = $fileUploads.filter(
(_fileUpload) => _fileUpload.name !== fileUpload.name
)
}}
startIcon={{
icon: RefreshCcw
}}
>
Remove from list
</Button>
{/if}
{#if fileUpload.progress < 100 && !fileUpload.cancelled && !fileUpload.errorMessage}
<Button
size="xs2"
color="light"
variant="border"
on:click={() => {
fileUpload.cancelled = true
fileUpload.progress = 0
}}
startIcon={{
icon: Ban
}}
>
Cancel Upload
</Button>
{/if}
{#if fileUpload.progress === 100 && !fileUpload.cancelled}
<Button
size="xs2"
color="red"
variant="border"
on:click={() => {
$fileUploads = $fileUploads.filter(
(_fileUpload) => _fileUpload.name !== fileUpload.name
)
if (fileUpload.path) {
deleteFile(fileUpload.path)
}
}}
startIcon={{
icon: Trash
}}
>
Delete
</Button>
{/if}
</div>
</div>
<FileProgressBar
progress={fileUpload.progress}
color={fileUpload.errorMessage
? '#ef4444'
: fileUpload.cancelled
? '#eab308'
: fileUpload.progress === 100
? '#22c55e'
: '#3b82f6'}
ended={fileUpload.cancelled || fileUpload.errorMessage !== undefined}
>
{#if fileUpload.errorMessage}
<span class="text-xs text-red-600">{fileUpload.errorMessage}</span>
{:else if fileUpload.cancelled}
<span class="text-xs text-yellow-600">Upload cancelled</span>
{/if}
</FileProgressBar>
{#if !(fileUpload.cancelled || fileUpload.errorMessage !== undefined)}
<span class="text-xs text-gray-500 dark:text-gray-200">
{fileUpload.progress === 100 ? 'Upload finished' : `Uploading`} to path: {fileUpload.path ??
'N/A'}
</span>
{/if}
</div>
{/each}
</div>
{#if allowMultiple}
<div class="flex flex-row gap-1 items-center justify-end p-1">
{#if !$fileUploads.every((fileUpload) => fileUpload.progress === 100 || fileUpload.cancelled)}
<Button
size="xs2"
color="light"
on:click={() => {
$fileUploads = $fileUploads.map((fileUpload) => {
if (fileUpload.progress === 100 || fileUpload.cancelled) {
return fileUpload
}
fileUpload.cancelled = true
fileUpload.progress = 0
return fileUpload
})
}}
startIcon={{
icon: Ban
}}
>
Cancel All Uploads
</Button>
{/if}
<Button
size="xs2"
color="light"
on:click={() => {
forceDisplayUploads = true
}}
startIcon={{
icon: Files
}}
disabled={$fileUploads.some(
(fileUpload) => fileUpload.progress !== 100 && !fileUpload.cancelled
)}
>
Upload more files
</Button>
</div>
{/if}
</div>
{:else}
<FileInput
accept={acceptedFileTypes?.join(',')}
multiple={allowMultiple}
returnFileNames
includeMimeType
on:change={({ detail }) => {
forceDisplayUploads = false
handleChange(detail)
}}
class={twMerge('w-full h-full', customClass, 'wm-file-input')}
style={customStyle}
>
{containerText}
</FileInput>
{/if}
</div>
@@ -1,23 +1,32 @@
<script lang="ts">
import { classNames } from '$lib/utils'
import { classNames, emptyString } from '$lib/utils'
import { createEventDispatcher } from 'svelte'
import { fade } from 'svelte/transition'
import Button from '../button/Button.svelte'
import { FileUp, Loader2, X } from 'lucide-svelte'
import { X } from 'lucide-svelte'
import FileUpload from './FileUpload.svelte'
export let title: string
export let open: boolean = false
export let progressPct: number | undefined = undefined
export let errorMsg: string | undefined = undefined
export let fileKey: string | undefined = undefined
export let fileToUpload: File | undefined = undefined
let s3Folder: string = ''
const dispatch = createEventDispatcher()
function fadeFast(node: HTMLElement) {
return fade(node, { duration: 100 })
}
function cleanFilePath(rawFolder: string, fileName: string) {
if (emptyString(rawFolder)) {
return fileName
}
if (!rawFolder.endsWith('/')) {
rawFolder = `${rawFolder}/`
}
return `${rawFolder}${fileName}`
}
</script>
{#if open}
@@ -49,8 +58,8 @@
{title}
</h3>
<Button
on:click={() => dispatch('close')}
title="Close - This will cancel any upload in progress"
on:click={() => dispatch('close', fileKey)}
title="Close"
color="light"
size="sm"
iconOnly={true}
@@ -58,63 +67,27 @@
/>
</div>
<div class="flex items-center gap-2">
<span>Key: </span>
<span>Folder: </span>
<input
type="text"
placeholder="folder/nested/file.txt"
bind:value={fileKey}
placeholder="folder/nested/"
bind:value={s3Folder}
class="text-2xl grow"
/>
</div>
<div class="w-full h-full">
<input
type="file"
title={fileToUpload ? `${fileToUpload.name}` : 'No file chosen'}
on:change={({ currentTarget }) => {
if (
currentTarget.files === undefined ||
currentTarget.files === null ||
currentTarget.files.length === 0
) {
fileToUpload = undefined
} else {
fileToUpload = currentTarget.files[0]
if (fileKey === undefined || fileKey === '') {
fileKey = fileToUpload.name
}
}
<FileUpload
allowMultiple={true}
pathTransformer={(file) => cleanFilePath(s3Folder, file.file.name)}
on:addition={(evt) => {
fileKey = evt.detail?.path
}}
on:deletion={(evt) => {
fileKey = undefined
}}
accept="*"
multiple={false}
/>
</div>
<div class="flex w-full bg-gray-200 rounded-full h-4 overflow-hidden">
<div class="h-full bg-blue-400" style="width: {progressPct ?? 0}%" />
</div>
{#if errorMsg !== undefined}
<div class="text-red-500 dark:text-red-400 text-sm">
{errorMsg}
</div>
{/if}
</div>
<div class="flex items-center space-x-2 flex-row-reverse space-x-reverse mt-4">
<Button
disabled={progressPct !== undefined}
on:click={() => dispatch('confirmed')}
color="blue"
size="sm"
startIcon={progressPct !== undefined
? { icon: Loader2, classes: 'animate-spin' }
: { icon: FileUp }}
>
<span>Upload</span>
</Button>
<Button on:click={() => dispatch('canceled')} color="light" size="sm">
<span>Cancel</span>
</Button>
</div>
</div>
</div>