feat: code content search (#2367)

* code search

* foo

* foo

* all

* all
This commit is contained in:
Ruben Fiszel
2023-10-01 21:03:08 +02:00
committed by GitHub
parent 2be257e594
commit 76d9dcbe4a
11 changed files with 708 additions and 3 deletions
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)] WHERE workspace_id = $1 LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
true
]
},
"hash": "fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5"
}
+102
View File
@@ -2135,6 +2135,31 @@ paths:
items:
$ref: "#/components/schemas/ListableResource"
/w/{workspace}/resources/list_search:
get:
summary: list resources for search
operationId: listSearchResource
tags:
- resource
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: resource list
content:
application/json:
schema:
type: array
items:
type: object
properties:
path:
type: string
value: {}
required:
- path
- value
/w/{workspace}/resources/list_names/{name}:
get:
summary: list resource names
@@ -2557,6 +2582,33 @@ paths:
required:
- id
/w/{workspace}/scripts/list_search:
get:
summary: list scripts for search
operationId: listSearchScript
tags:
- script
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: script list
content:
application/json:
schema:
type: array
items:
type: object
properties:
path:
type: string
content:
type: string
required:
- path
- content
/w/{workspace}/scripts/list:
get:
summary: list all available scripts
@@ -3202,6 +3254,31 @@ paths:
items:
type: string
/w/{workspace}/flows/list_search:
get:
summary: list flows for search
operationId: listSearchFlow
tags:
- flow
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: flow list
content:
application/json:
schema:
type: array
items:
type: object
properties:
path:
type: string
value: {}
required:
- path
- value
/w/{workspace}/flows/list:
get:
summary: list all available flows
@@ -3505,6 +3582,31 @@ paths:
schema:
type: string
/w/{workspace}/apps/list_search:
get:
summary: list apps for search
operationId: listSearchApp
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: app list
content:
application/json:
schema:
type: array
items:
type: object
properties:
path:
type: string
value: {}
required:
- path
- value
/w/{workspace}/apps/list:
get:
summary: list all available apps
+33
View File
@@ -43,6 +43,7 @@ use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_apps))
.route("/list_search", get(list_search_apps))
.route("/get/p/*path", get(get_app))
.route("/get/draft/*path", get(get_app_w_draft))
.route("/secret_of/*path", get(get_secret_id))
@@ -160,6 +161,38 @@ pub struct EditApp {
pub policy: Option<Policy>,
}
#[derive(Serialize, FromRow)]
pub struct SearchApp {
path: String,
value: serde_json::Value,
}
async fn list_search_apps(
authed: ApiAuthed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchApp>> {
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 3;
let rows = sqlx::query_as!(
SearchApp,
"SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)] WHERE workspace_id = $1 LIMIT $2",
&w_id,
n
)
.fetch_all(&mut *tx)
.await?
.into_iter()
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(rows))
}
async fn list_apps(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
+34 -1
View File
@@ -24,7 +24,7 @@ use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use sql_builder::prelude::*;
use sql_builder::SqlBuilder;
use sqlx::{Postgres, Transaction};
use sqlx::{FromRow, Postgres, Transaction};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
db::UserDB,
@@ -42,6 +42,7 @@ use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, Que
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_flows))
.route("/list_search", get(list_search_flows))
.route("/create", post(create_flow))
.route("/update/*path", post(update_flow))
.route("/archive/*path", post(archive_flow_by_path))
@@ -58,6 +59,38 @@ pub fn global_service() -> Router {
.route("/hub/get/:id", get(get_hub_flow_by_id))
}
#[derive(Serialize, FromRow)]
pub struct SearchFlow {
path: String,
value: serde_json::Value,
}
async fn list_search_flows(
authed: ApiAuthed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchFlow>> {
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 3;
let rows = sqlx::query_as!(
SearchFlow,
"SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
&w_id,
n
)
.fetch_all(&mut *tx)
.await?
.into_iter()
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(rows))
}
async fn list_flows(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
+32
View File
@@ -31,6 +31,7 @@ use windmill_common::{
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_resources))
.route("/list_search", get(list_search_resources))
.route("/list_names/:type", get(list_names))
.route("/get/*path", get(get_resource))
.route("/exists/*path", get(exists_resource))
@@ -144,6 +145,37 @@ async fn list_names(
Ok(Json(rows))
}
#[derive(Serialize, FromRow)]
pub struct SearchResource {
path: String,
value: serde_json::Value,
}
async fn list_search_resources(
authed: ApiAuthed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchResource>> {
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 3;
let rows = sqlx::query_as!(
SearchResource,
"SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
&w_id,
n
)
.fetch_all(&mut *tx)
.await?
.into_iter()
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(rows))
}
async fn list_resources(
authed: ApiAuthed,
Query(lq): Query<ListResourceQuery>,
+32
View File
@@ -96,6 +96,7 @@ pub fn global_unauthed_service() -> Router {
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_scripts))
.route("/list_search", get(list_search_scripts))
.route("/create", post(create_script))
.route("/archive/p/*path", post(archive_script_by_path))
.route("/get/draft/*path", get(get_script_by_path_w_draft))
@@ -111,6 +112,37 @@ pub fn workspaced_service() -> Router {
.route("/list_paths", get(list_paths))
}
#[derive(Serialize, FromRow)]
pub struct SearchScript {
path: String,
content: String,
}
async fn list_search_scripts(
authed: ApiAuthed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchScript>> {
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 10;
let rows = sqlx::query_as!(
SearchScript,
"SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2",
&w_id,
n
)
.fetch_all(&mut *tx)
.await?
.into_iter()
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(rows))
}
async fn list_scripts(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -0,0 +1,346 @@
<script lang="ts">
import { AppService, FlowService, ResourceService, ScriptService } from '$lib/gen'
import { enterpriseLicense, workspaceStore } from '$lib/stores'
import { clickOutside } from '$lib/utils'
import { Boxes, Code2, LayoutDashboard, Loader2, X } from 'lucide-svelte'
import Portal from 'svelte-portal'
import { twMerge } from 'tailwind-merge'
import SearchItems from './SearchItems.svelte'
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import FlowIcon from './home/FlowIcon.svelte'
import { Alert, Button } from './common'
import { faEdit } from '@fortawesome/free-solid-svg-icons'
import { goto } from '$app/navigation'
let search: string = ''
export function open() {
isOpen = true
loadScripts()
loadResources()
loadApps()
loadFlows()
}
export async function loadScripts() {
scripts = await ScriptService.listSearchScript({ workspace: $workspaceStore ?? '' })
}
export async function loadResources() {
resources = await ResourceService.listSearchResource({ workspace: $workspaceStore ?? '' })
}
export async function loadApps() {
apps = await AppService.listSearchApp({ workspace: $workspaceStore ?? '' })
}
export async function loadFlows() {
flows = await FlowService.listSearchFlow({ workspace: $workspaceStore ?? '' })
}
let isOpen = false
let scripts: undefined | { path: string; content: string }[] = undefined
let filteredScriptItems: { path: string; content: string; marked: any }[] = []
let resources: undefined | { path: string; value: any }[] = undefined
let filteredResourceItems: { path: string; value: any; marked: any }[] = []
let flows: undefined | { path: string; value: any }[] = undefined
let filteredFlowItems: { path: string; value: any; marked: any }[] = []
let apps: undefined | { path: string; value: any }[] = undefined
let filteredAppItems: { path: string; value: any; marked: any }[] = []
let searchKind: 'all' | 'scripts' | 'flows' | 'apps' | 'resources' = 'all'
function getCounts(n: number) {
return ` (${n})`
}
$: counts =
search == '' || !scripts || !resources || !flows || !apps
? {
all: '',
apps: '',
flows: '',
resources: '',
scripts: ''
}
: {
all: getCounts(
filteredAppItems.length +
filteredFlowItems.length +
filteredResourceItems.length +
filteredScriptItems.length
),
apps: getCounts(filteredAppItems.length),
resources: getCounts(filteredResourceItems.length),
flows: getCounts(filteredFlowItems.length),
scripts: getCounts(filteredScriptItems.length)
}
</script>
<SearchItems
filter={search}
items={scripts}
f={(s) => {
return s.content
}}
bind:filteredItems={filteredScriptItems}
/>
<SearchItems
filter={search}
items={resources}
f={(s) => {
return JSON.stringify(s.value, null, 4)
}}
bind:filteredItems={filteredResourceItems}
/>
<SearchItems
filter={search}
items={flows}
f={(s) => {
return JSON.stringify(s.value, null, 4)
}}
bind:filteredItems={filteredFlowItems}
/>
<SearchItems
filter={search}
items={apps}
f={(s) => {
return JSON.stringify(s.value, null, 4)
}}
bind:filteredItems={filteredAppItems}
/>
{#if isOpen}
<Portal>
<div
class={twMerge(
`fixed top-0 bottom-0 left-0 right-0 transition-all duration-50`,
' bg-black bg-opacity-60',
'z-[1100]'
)}
>
<div
class={'max-w-4xl lg:mx-auto mx-10 mt-8 bg-surface rounded-lg relative'}
use:clickOutside={false}
on:click_outside={() => {
isOpen = false
}}
>
<div class="px-4 py-2 border-b flex justify-between items-center">
<div>Search by content</div>
<div class="w-8">
<button
on:click|stopPropagation={() => {
isOpen = false
}}
class="hover:bg-surface-hover bg-surface-secondary rounded-full w-8 h-8 flex items-center justify-center transition-all"
>
<X class="text-tertiary" />
</button>
</div>
</div>
<div class="px-2 py-2 overflow-auto">
<div class="flex gap-2 flex-wrap">
<div class="flex justify-start">
<ToggleButtonGroup bind:selected={searchKind} class="h-10">
<ToggleButton small value="all" label={'All' + counts.all} />
<ToggleButton
small
value="scripts"
icon={Code2}
label={'Scripts' + counts.scripts}
/>
<ToggleButton
small
value="resources"
icon={Boxes}
label={'Resources' + counts.resources}
/>
<ToggleButton
small
value="flows"
label={'Flows' + counts.flows}
icon={FlowIcon}
selectedColor="#14b8a6"
/>
<ToggleButton
small
value="apps"
label={'Apps' + counts.apps}
icon={LayoutDashboard}
selectedColor="#fb923c"
/>
</ToggleButtonGroup>
</div>
<div class="relative text-tertiary grow min-w-[100px]">
<!-- svelte-ignore a11y-autofocus -->
<input
autofocus
placeholder={'Search in the content of resources, scripts, flows and apps'}
bind:value={search}
class="bg-surface !h-10 !px-4 !pr-10 !rounded-lg text-sm focus:outline-none"
/>
<button type="submit" class="absolute right-0 top-0 mt-3 mr-4">
<svg
class="h-4 w-4 fill-current"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
version="1.1"
id="Capa_1"
x="0px"
y="0px"
viewBox="0 0 56.966 56.966"
style="enable-background:new 0 0 56.966 56.966;"
xml:space="preserve"
width="512px"
height="512px"
>
<path
d="M55.146,51.887L41.588,37.786c3.486-4.144,5.396-9.358,5.396-14.786c0-12.682-10.318-23-23-23s-23,10.318-23,23 s10.318,23,23,23c4.761,0,9.298-1.436,13.177-4.162l13.661,14.208c0.571,0.593,1.339,0.92,2.162,0.92 c0.779,0,1.518-0.297,2.079-0.837C56.255,54.982,56.293,53.08,55.146,51.887z M23.984,6c9.374,0,17,7.626,17,17s-7.626,17-17,17 s-17-7.626-17-17S14.61,6,23.984,6z"
/>
</svg>
</button>
</div>
</div>
<div class="mt-1">
<div class="text-xs text-secondary"
>Searching among <div class="inline-flex"
>{#if scripts}{scripts?.length}{:else}
<Loader2 size={10} class="animate-spin " />
{/if}</div
>
scripts,
<div class="inline-flex"
>{#if resources}{resources?.length}{:else}
<Loader2 size={10} class="animate-spin " />
{/if}</div
>
resources,
<div class="inline-flex"
>{#if flows}{flows?.length}{:else}
<Loader2 size={10} class="animate-spin " />
{/if}</div
>
flows,
<div class="inline-flex"
>{#if apps}{apps?.length}{:else}
<Loader2 size={10} class="animate-spin " />
{/if}</div
>
apps</div
>
</div>
<div class="mt-1 overflow-auto max-h-[80vh]">
{#if !enterpriseLicense}
<Alert title="Content Search is an EE feature" type="warning">
Without EE, content search will only search among 10 scripts, 3 flows, 3 apps and 3
resources.
</Alert>
<div class="py-1" />
{/if}
{#if search.length > 0}
<div class="flex flex-col gap-4">
{#if (searchKind == 'all' || searchKind == 'scripts') && filteredScriptItems.length > 0}
{#each filteredScriptItems as item}
<div>
<div class="text-sm font-semibold"
><a href="/scripts/get/{item.path}">Script: {item.path}</a></div
>
<div class="flex gap-2 justify-between">
<pre class="text-xs border p-2 overflow-auto max-h-40 w-full max-w-2xl"
><code>{@html item.marked}</code></pre
>
<div>
<div class="flex gap-2">
<Button
on:click|once={() => {
goto(`/scripts/edit/${item.path}?no_draft=true`)
}}
color="light"
size="sm"
startIcon={{ icon: faEdit }}>Edit</Button
>
</div>
</div>
</div>
</div>
{/each}
{/if}
{#if (searchKind == 'all' || searchKind == 'resources') && filteredResourceItems.length > 0}
{#each filteredResourceItems as item}
<div>
<div class="text-sm font-semibold">Resource: {item.path}</div>
<div class="flex gap-2 justify-between">
<pre class="text-xs border p-2 overflow-auto max-h-40 w-full max-w-2xl"
><code>{@html item.marked}</code></pre
>
</div>
</div>
{/each}
{/if}
{#if (searchKind == 'all' || searchKind == 'flows') && filteredFlowItems.length > 0}
{#each filteredFlowItems as item}
<div>
<div class="text-sm font-semibold"
><a href="/flows/get/{item.path}">Flow: {item.path}</a></div
>
<div class="flex gap-2 justify-between">
<pre class="text-xs border p-2 overflow-auto max-h-40 w-full max-w-2xl"
><code>{@html item.marked}</code></pre
>
<div>
<div class="flex gap-2">
<Button
on:click|once={() => {
goto(`/flows/edit/${item.path}?no_draft=true`)
}}
color="light"
size="sm"
startIcon={{ icon: faEdit }}>Edit</Button
>
</div>
</div>
</div>
</div>
{/each}
{/if}
{#if (searchKind == 'all' || searchKind == 'apps') && filteredAppItems.length > 0}
{#each filteredAppItems as item}
<div>
<div class="text-sm font-semibold"
><a href="/apps/get/{item.path}">App: {item.path}</a></div
>
<div class="flex gap-2 justify-between">
<pre class="text-xs border p-2 overflow-auto max-h-40 w-full max-w-2xl"
><code>{@html item.marked}</code></pre
>
</div>
</div>
{/each}
{/if}
</div>
{:else}
<div class="flex justify-center items-center h-48">
<div class="text-tertiary text-center">
<div class="text-2xl font-bold">Empty Search Filter</div>
<div class="text-sm"
>Start writing, search everywhere a path is referenced for instance</div
>
</div>
</div>
{/if}
</div>
</div></div
></div
></Portal
>
{/if}
@@ -1,6 +1,6 @@
<script lang="ts">
import CenteredPage from '$lib/components/CenteredPage.svelte'
import { Alert, Badge, Skeleton } from '$lib/components/common'
import { Alert, Badge, Button, Skeleton } from '$lib/components/common'
import ShareModal from '$lib/components/ShareModal.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import {
@@ -15,7 +15,7 @@
} from '$lib/gen'
import { userStore, workspaceStore } from '$lib/stores'
import type uFuzzy from '@leeoniya/ufuzzy'
import { Code2, LayoutDashboard } from 'lucide-svelte'
import { Code2, LayoutDashboard, SearchCode } from 'lucide-svelte'
export let filter = ''
export let subtab: 'flow' | 'script' | 'app' = 'script'
@@ -39,6 +39,7 @@
import { page } from '$app/stores'
import { setQuery } from '$lib/navigation'
import DeployWorkspaceDrawer from '../DeployWorkspaceDrawer.svelte'
import ContentSearch from '../ContentSearch.svelte'
type TableItem<T, U extends 'script' | 'flow' | 'app' | 'raw_app'> = T & {
canWrite: boolean
@@ -254,6 +255,8 @@
$: items && resetScroll()
let archived = false
let contentSearch: ContentSearch
</script>
<SearchItems
@@ -285,6 +288,7 @@
}}
/>
<ContentSearch bind:this={contentSearch} />
<CenteredPage>
<div class="flex flex-wrap gap-2 items-center justify-between w-full mt-2">
<div class="flex justify-start">
@@ -348,6 +352,13 @@
</svg>
</button>
</div>
<Button
on:click={contentSearch?.open}
variant="border"
btnClasses="py-2.5"
size="xs"
color="light">Content&nbsp;<SearchCode size={16} /></Button
>
</div>
<div class="relative">
<ListFilters