mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 08:20:12 +00:00
back off multiplayer reconnects and re-sign the token per attempt (#12166)
* fix: back off multiplayer reconnects and re-sign the token per attempt Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: drop a collaboration connection that opens after it was turned off Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: declare the collaboration generation before the init-time join Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dee18d9ee0
commit
e4d74a3c97
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
import { buildWsUrl } from '$lib/wsUrl'
|
||||
import { MultiplayerConnection } from '$lib/multiplayerConnection'
|
||||
import { paneMinPercent } from '$lib/utils/splitpaneSizing'
|
||||
import { processSecretArgs } from './secretArgUtils'
|
||||
import type { Schema, SupportedLanguage } from '$lib/common'
|
||||
@@ -36,7 +36,6 @@
|
||||
import * as Y from 'yjs'
|
||||
import { scriptLangToEditorLang } from '$lib/scripts'
|
||||
import { langToExt } from '$lib/editorLangUtils'
|
||||
import { WebsocketProvider } from 'y-websocket'
|
||||
import Modal from './common/modal/Modal.svelte'
|
||||
import Popover from './meltComponents/Popover.svelte'
|
||||
import DiffEditor from './DiffEditor.svelte'
|
||||
@@ -71,7 +70,6 @@
|
||||
getDebugFileExtension,
|
||||
fetchContextualVariables,
|
||||
signDebugRequest,
|
||||
signMultiplayerRequest,
|
||||
getDebugErrorMessage
|
||||
} from '$lib/components/debug'
|
||||
import { SvelteSet } from 'svelte/reactivity'
|
||||
@@ -819,7 +817,11 @@
|
||||
| undefined = undefined
|
||||
let downloadingRecording = $state(false)
|
||||
|
||||
let wsProvider: WebsocketProvider | undefined = $state(undefined)
|
||||
let collab: MultiplayerConnection | undefined = $state(undefined)
|
||||
// Bumped on every disable, so a connection that finishes opening after collaboration
|
||||
// was turned off or the editor unmounted is dropped instead of reconnecting forever.
|
||||
// Declared here because `?collab=1` calls setCollaborationMode during init, below.
|
||||
let collabGeneration = 0
|
||||
let yContent: Y.Text | undefined = $state(undefined)
|
||||
let peers: { name: string }[] = $state([])
|
||||
let showCollabPopup = $state(false)
|
||||
@@ -1543,6 +1545,7 @@
|
||||
|
||||
setLicense()
|
||||
export async function setCollaborationMode() {
|
||||
const generation = ++collabGeneration
|
||||
await setLicense()
|
||||
if (!$enterpriseLicense) {
|
||||
sendUserToast(`Multiplayer is an enterprise feature`, true, [
|
||||
@@ -1556,29 +1559,32 @@
|
||||
return
|
||||
}
|
||||
|
||||
let token: string | undefined
|
||||
const ydoc = new Y.Doc()
|
||||
let yContentInit = ydoc.getText('content')
|
||||
|
||||
let opened: MultiplayerConnection
|
||||
try {
|
||||
token = await signMultiplayerRequest(opWs ?? '')
|
||||
opened = await MultiplayerConnection.open(
|
||||
opWs ?? '',
|
||||
opWs + '/' + (path ?? 'no-room-name'),
|
||||
ydoc
|
||||
)
|
||||
} catch (e) {
|
||||
console.error('Failed to sign multiplayer request:', e)
|
||||
sendUserToast('Failed to authorize multiplayer session', true)
|
||||
return
|
||||
}
|
||||
|
||||
const ydoc = new Y.Doc()
|
||||
if (wsProvider) {
|
||||
wsProvider.destroy()
|
||||
if (generation !== collabGeneration) {
|
||||
opened.destroy()
|
||||
return
|
||||
}
|
||||
let yContentInit = ydoc.getText('content')
|
||||
collab?.destroy()
|
||||
collab = opened
|
||||
|
||||
wsProvider = new WebsocketProvider(
|
||||
buildWsUrl('/ws_mp/'),
|
||||
opWs + '/' + (path ?? 'no-room-name'),
|
||||
ydoc,
|
||||
{ connect: false, params: { token } }
|
||||
)
|
||||
|
||||
wsProvider.on('sync', (isSynced: boolean) => {
|
||||
// One toast per outage: the connection keeps retrying on its own.
|
||||
let errorToastShown = false
|
||||
opened.provider.on('sync', (isSynced: boolean) => {
|
||||
if (isSynced) errorToastShown = false
|
||||
if (isSynced && yContentInit?.toJSON() == '') {
|
||||
showCollabPopup = true
|
||||
yContentInit?.insert(0, code)
|
||||
@@ -1586,12 +1592,13 @@
|
||||
yContent = yContentInit
|
||||
})
|
||||
|
||||
wsProvider.on('connection-error', (WSErrorEvent) => {
|
||||
opened.provider.on('connection-error', (WSErrorEvent) => {
|
||||
console.error(WSErrorEvent)
|
||||
if (errorToastShown) return
|
||||
errorToastShown = true
|
||||
sendUserToast('Multiplayer server connection had an error', true)
|
||||
})
|
||||
wsProvider.connect()
|
||||
const awareness = wsProvider.awareness
|
||||
const awareness = opened.provider.awareness
|
||||
|
||||
awareness.setLocalStateField('user', {
|
||||
name: $userStore?.username
|
||||
@@ -1609,11 +1616,11 @@
|
||||
}
|
||||
|
||||
export function disableCollaboration() {
|
||||
if (!wsProvider?.shouldConnect) return
|
||||
collabGeneration++
|
||||
if (!collab) return
|
||||
peers = []
|
||||
wsProvider?.disconnect()
|
||||
wsProvider.destroy()
|
||||
wsProvider = undefined
|
||||
collab.destroy()
|
||||
collab = undefined
|
||||
}
|
||||
|
||||
onDestroy(() => {
|
||||
@@ -1895,7 +1902,7 @@
|
||||
workspace={opWs}
|
||||
scriptPath={edit ? path : undefined}
|
||||
on:toggleCollabMode={() => {
|
||||
if (wsProvider?.shouldConnect) {
|
||||
if (collab) {
|
||||
disableCollaboration()
|
||||
} else {
|
||||
setCollaborationMode()
|
||||
@@ -1904,7 +1911,7 @@
|
||||
on:showDiffMode={showDiffMode}
|
||||
on:hideDiffMode={hideDiffMode}
|
||||
customUi={customUi?.editorBar}
|
||||
collabLive={wsProvider?.shouldConnect}
|
||||
collabLive={collab !== undefined}
|
||||
{collabMode}
|
||||
{validCode}
|
||||
{validAssets}
|
||||
@@ -2760,7 +2767,7 @@
|
||||
bind:this={editor}
|
||||
schemaContractMarkers={contractMarkers}
|
||||
{yContent}
|
||||
awareness={wsProvider?.awareness}
|
||||
awareness={collab?.provider.awareness}
|
||||
on:change={(e) => {
|
||||
if (activeModuleTab === null) {
|
||||
// `editorCode`, not the payload: `setCode` dispatches the string it was
|
||||
|
||||
@@ -4,16 +4,14 @@
|
||||
|
||||
import { enterpriseLicense, userStore, workspaceStore, awarenessStore } from '$lib/stores'
|
||||
|
||||
import { WebsocketProvider } from 'y-websocket'
|
||||
import * as Y from 'yjs'
|
||||
import type { Awareness } from 'y-protocols/awareness'
|
||||
import { page } from '$app/stores'
|
||||
import { slide } from 'svelte/transition'
|
||||
import { buildWsUrl } from '$lib/wsUrl'
|
||||
import { signMultiplayerRequest } from '$lib/components/debug/debugUtils'
|
||||
import { MultiplayerConnection } from '$lib/multiplayerConnection'
|
||||
|
||||
let awareness: Awareness | undefined = $state(undefined)
|
||||
let wsProvider: WebsocketProvider | undefined = undefined
|
||||
let connection: MultiplayerConnection | undefined = undefined
|
||||
|
||||
let connected = $state(false)
|
||||
|
||||
@@ -24,34 +22,39 @@
|
||||
url: $page.url.pathname
|
||||
})
|
||||
}
|
||||
// Bumped on every disconnect, so a connection that finishes opening after the
|
||||
// workspace changed or the menu unmounted is dropped instead of reconnecting forever.
|
||||
let connectGeneration = 0
|
||||
function disconnectWorkspace() {
|
||||
if (wsProvider) {
|
||||
wsProvider.destroy()
|
||||
wsProvider = undefined
|
||||
connectGeneration++
|
||||
if (connection) {
|
||||
connection.destroy()
|
||||
connection = undefined
|
||||
}
|
||||
connected = false
|
||||
awareness = undefined
|
||||
}
|
||||
async function connectWorkspace(workspace: string) {
|
||||
disconnectWorkspace()
|
||||
const generation = connectGeneration
|
||||
|
||||
let token: string | undefined
|
||||
let opened: MultiplayerConnection
|
||||
try {
|
||||
token = await signMultiplayerRequest(workspace)
|
||||
opened = await MultiplayerConnection.open(workspace, workspace, new Y.Doc())
|
||||
} catch (e) {
|
||||
console.error('Failed to sign multiplayer request:', e)
|
||||
return
|
||||
}
|
||||
|
||||
const ydoc = new Y.Doc()
|
||||
wsProvider = new WebsocketProvider(buildWsUrl('/ws_mp/'), workspace, ydoc, {
|
||||
params: { token }
|
||||
})
|
||||
wsProvider.on('sync', (isSynced: boolean) => {
|
||||
if (generation !== connectGeneration) {
|
||||
opened.destroy()
|
||||
return
|
||||
}
|
||||
connection = opened
|
||||
opened.provider.on('sync', (isSynced: boolean) => {
|
||||
connected = true
|
||||
})
|
||||
|
||||
awareness = wsProvider.awareness
|
||||
awareness = opened.provider.awareness
|
||||
|
||||
updateLocalState()
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import * as Y from 'yjs'
|
||||
|
||||
let signed = 0
|
||||
vi.mock('$lib/components/debug/debugUtils', () => ({
|
||||
signMultiplayerRequest: vi.fn(async () => `token-${++signed}`)
|
||||
}))
|
||||
vi.mock('$lib/wsUrl', () => ({ buildWsUrl: (path: string) => `ws://mp.test${path}` }))
|
||||
|
||||
import { MultiplayerConnection } from './multiplayerConnection'
|
||||
|
||||
const attempts: string[] = []
|
||||
|
||||
// What the multiplayer server does with a token it rejects: accept the upgrade,
|
||||
// then close with 4403.
|
||||
class RejectingWebSocket {
|
||||
binaryType = ''
|
||||
onopen: (() => void) | null = null
|
||||
onclose: ((e: { code: number }) => void) | null = null
|
||||
onerror: (() => void) | null = null
|
||||
onmessage: (() => void) | null = null
|
||||
constructor(url: string) {
|
||||
attempts.push(url)
|
||||
setTimeout(() => {
|
||||
this.onopen?.()
|
||||
this.onclose?.({ code: 4403 })
|
||||
}, 0)
|
||||
}
|
||||
send() {}
|
||||
close() {}
|
||||
}
|
||||
|
||||
describe('MultiplayerConnection', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.stubGlobal('WebSocket', RejectingWebSocket)
|
||||
attempts.length = 0
|
||||
signed = 0
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('backs off on rejected connections and signs a fresh token for each attempt', async () => {
|
||||
const connection = await MultiplayerConnection.open('ws', 'ws/room', new Y.Doc())
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
|
||||
// 1s doubling to a 60s cap, halved by jitter at worst: at most 15 in 5 minutes,
|
||||
// where y-websocket's own reconnect makes thousands.
|
||||
expect(attempts.length).toBeGreaterThan(3)
|
||||
expect(attempts.length).toBeLessThanOrEqual(15)
|
||||
const tokens = attempts.map((u) => new URL(u).searchParams.get('token'))
|
||||
expect(new Set(tokens).size).toBe(attempts.length)
|
||||
|
||||
connection.destroy()
|
||||
const after = attempts.length
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
expect(attempts.length).toBe(after)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,82 @@
|
||||
import { WebsocketProvider } from 'y-websocket'
|
||||
import type * as Y from 'yjs'
|
||||
import { buildWsUrl } from '$lib/wsUrl'
|
||||
import { signMultiplayerRequest } from '$lib/components/debug/debugUtils'
|
||||
|
||||
const BASE_DELAY_MS = 1000
|
||||
const MAX_DELAY_MS = 60_000
|
||||
|
||||
/**
|
||||
* A y-websocket provider for the multiplayer server that reconnects with its own
|
||||
* exponential backoff and a freshly signed token on every attempt.
|
||||
*
|
||||
* y-websocket's built-in reconnect is not usable here: it reuses the token it was
|
||||
* built with, which lives 60s, and the server rejects a bad token only after the
|
||||
* socket opened, which resets y-websocket's backoff to 100ms. Any drop past the
|
||||
* first minute would otherwise retry ten times a second, forever. An attempt only
|
||||
* counts as successful once the document synced.
|
||||
*/
|
||||
export class MultiplayerConnection {
|
||||
readonly provider: WebsocketProvider
|
||||
#workspace: string
|
||||
#failures = 0
|
||||
#timer: ReturnType<typeof setTimeout> | undefined
|
||||
#destroyed = false
|
||||
|
||||
private constructor(workspace: string, provider: WebsocketProvider) {
|
||||
this.#workspace = workspace
|
||||
this.provider = provider
|
||||
provider.on('sync', (synced: boolean) => {
|
||||
if (synced) this.#failures = 0
|
||||
})
|
||||
provider.on('connection-close', () => {
|
||||
if (this.#destroyed) return
|
||||
// Stops y-websocket's own reconnect timer from firing setupWS.
|
||||
provider.shouldConnect = false
|
||||
this.#scheduleReconnect()
|
||||
})
|
||||
}
|
||||
|
||||
/** Throws when the first token cannot be signed. */
|
||||
static async open(workspace: string, room: string, doc: Y.Doc): Promise<MultiplayerConnection> {
|
||||
const token = await signMultiplayerRequest(workspace)
|
||||
const provider = new WebsocketProvider(buildWsUrl('/ws_mp/'), room, doc, {
|
||||
connect: false,
|
||||
params: { token }
|
||||
})
|
||||
const connection = new MultiplayerConnection(workspace, provider)
|
||||
provider.connect()
|
||||
return connection
|
||||
}
|
||||
|
||||
#scheduleReconnect() {
|
||||
clearTimeout(this.#timer)
|
||||
const ceiling = Math.min(BASE_DELAY_MS * 2 ** this.#failures, MAX_DELAY_MS)
|
||||
this.#failures++
|
||||
const delay = ceiling / 2 + Math.random() * (ceiling / 2)
|
||||
this.#timer = setTimeout(() => this.#reconnect(), delay)
|
||||
}
|
||||
|
||||
async #reconnect() {
|
||||
let token: string
|
||||
try {
|
||||
token = await signMultiplayerRequest(this.#workspace)
|
||||
} catch (e) {
|
||||
if (this.#destroyed) return
|
||||
console.error('Failed to sign multiplayer request:', e)
|
||||
this.#scheduleReconnect()
|
||||
return
|
||||
}
|
||||
if (this.#destroyed) return
|
||||
const url = new URL(this.provider.url)
|
||||
url.searchParams.set('token', token)
|
||||
this.provider.url = url.toString()
|
||||
this.provider.connect()
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.#destroyed = true
|
||||
clearTimeout(this.#timer)
|
||||
this.provider.destroy()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user