back off multiplayer reconnects and re-sign the token per attempt (#12166)

* fix: back off multiplayer reconnects and re-sign the token per attempt

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: drop a collaboration connection that opens after it was turned off

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: declare the collaboration generation before the init-time join

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-11-03 06:00:04 +01:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dee18d9ee0
commit e4d74a3c97
4 changed files with 198 additions and 45 deletions
+36 -29
View File
@@ -1,5 +1,5 @@
<script lang="ts">
import { buildWsUrl } from '$lib/wsUrl'
import { MultiplayerConnection } from '$lib/multiplayerConnection'
import { paneMinPercent } from '$lib/utils/splitpaneSizing'
import { processSecretArgs } from './secretArgUtils'
import type { Schema, SupportedLanguage } from '$lib/common'
@@ -36,7 +36,6 @@
import * as Y from 'yjs'
import { scriptLangToEditorLang } from '$lib/scripts'
import { langToExt } from '$lib/editorLangUtils'
import { WebsocketProvider } from 'y-websocket'
import Modal from './common/modal/Modal.svelte'
import Popover from './meltComponents/Popover.svelte'
import DiffEditor from './DiffEditor.svelte'
@@ -71,7 +70,6 @@
getDebugFileExtension,
fetchContextualVariables,
signDebugRequest,
signMultiplayerRequest,
getDebugErrorMessage
} from '$lib/components/debug'
import { SvelteSet } from 'svelte/reactivity'
@@ -819,7 +817,11 @@
| undefined = undefined
let downloadingRecording = $state(false)
let wsProvider: WebsocketProvider | undefined = $state(undefined)
let collab: MultiplayerConnection | undefined = $state(undefined)
// Bumped on every disable, so a connection that finishes opening after collaboration
// was turned off or the editor unmounted is dropped instead of reconnecting forever.
// Declared here because `?collab=1` calls setCollaborationMode during init, below.
let collabGeneration = 0
let yContent: Y.Text | undefined = $state(undefined)
let peers: { name: string }[] = $state([])
let showCollabPopup = $state(false)
@@ -1543,6 +1545,7 @@
setLicense()
export async function setCollaborationMode() {
const generation = ++collabGeneration
await setLicense()
if (!$enterpriseLicense) {
sendUserToast(`Multiplayer is an enterprise feature`, true, [
@@ -1556,29 +1559,32 @@
return
}
let token: string | undefined
const ydoc = new Y.Doc()
let yContentInit = ydoc.getText('content')
let opened: MultiplayerConnection
try {
token = await signMultiplayerRequest(opWs ?? '')
opened = await MultiplayerConnection.open(
opWs ?? '',
opWs + '/' + (path ?? 'no-room-name'),
ydoc
)
} catch (e) {
console.error('Failed to sign multiplayer request:', e)
sendUserToast('Failed to authorize multiplayer session', true)
return
}
const ydoc = new Y.Doc()
if (wsProvider) {
wsProvider.destroy()
if (generation !== collabGeneration) {
opened.destroy()
return
}
let yContentInit = ydoc.getText('content')
collab?.destroy()
collab = opened
wsProvider = new WebsocketProvider(
buildWsUrl('/ws_mp/'),
opWs + '/' + (path ?? 'no-room-name'),
ydoc,
{ connect: false, params: { token } }
)
wsProvider.on('sync', (isSynced: boolean) => {
// One toast per outage: the connection keeps retrying on its own.
let errorToastShown = false
opened.provider.on('sync', (isSynced: boolean) => {
if (isSynced) errorToastShown = false
if (isSynced && yContentInit?.toJSON() == '') {
showCollabPopup = true
yContentInit?.insert(0, code)
@@ -1586,12 +1592,13 @@
yContent = yContentInit
})
wsProvider.on('connection-error', (WSErrorEvent) => {
opened.provider.on('connection-error', (WSErrorEvent) => {
console.error(WSErrorEvent)
if (errorToastShown) return
errorToastShown = true
sendUserToast('Multiplayer server connection had an error', true)
})
wsProvider.connect()
const awareness = wsProvider.awareness
const awareness = opened.provider.awareness
awareness.setLocalStateField('user', {
name: $userStore?.username
@@ -1609,11 +1616,11 @@
}
export function disableCollaboration() {
if (!wsProvider?.shouldConnect) return
collabGeneration++
if (!collab) return
peers = []
wsProvider?.disconnect()
wsProvider.destroy()
wsProvider = undefined
collab.destroy()
collab = undefined
}
onDestroy(() => {
@@ -1895,7 +1902,7 @@
workspace={opWs}
scriptPath={edit ? path : undefined}
on:toggleCollabMode={() => {
if (wsProvider?.shouldConnect) {
if (collab) {
disableCollaboration()
} else {
setCollaborationMode()
@@ -1904,7 +1911,7 @@
on:showDiffMode={showDiffMode}
on:hideDiffMode={hideDiffMode}
customUi={customUi?.editorBar}
collabLive={wsProvider?.shouldConnect}
collabLive={collab !== undefined}
{collabMode}
{validCode}
{validAssets}
@@ -2760,7 +2767,7 @@
bind:this={editor}
schemaContractMarkers={contractMarkers}
{yContent}
awareness={wsProvider?.awareness}
awareness={collab?.provider.awareness}
on:change={(e) => {
if (activeModuleTab === null) {
// `editorCode`, not the payload: `setCode` dispatches the string it was
@@ -4,16 +4,14 @@
import { enterpriseLicense, userStore, workspaceStore, awarenessStore } from '$lib/stores'
import { WebsocketProvider } from 'y-websocket'
import * as Y from 'yjs'
import type { Awareness } from 'y-protocols/awareness'
import { page } from '$app/stores'
import { slide } from 'svelte/transition'
import { buildWsUrl } from '$lib/wsUrl'
import { signMultiplayerRequest } from '$lib/components/debug/debugUtils'
import { MultiplayerConnection } from '$lib/multiplayerConnection'
let awareness: Awareness | undefined = $state(undefined)
let wsProvider: WebsocketProvider | undefined = undefined
let connection: MultiplayerConnection | undefined = undefined
let connected = $state(false)
@@ -24,34 +22,39 @@
url: $page.url.pathname
})
}
// Bumped on every disconnect, so a connection that finishes opening after the
// workspace changed or the menu unmounted is dropped instead of reconnecting forever.
let connectGeneration = 0
function disconnectWorkspace() {
if (wsProvider) {
wsProvider.destroy()
wsProvider = undefined
connectGeneration++
if (connection) {
connection.destroy()
connection = undefined
}
connected = false
awareness = undefined
}
async function connectWorkspace(workspace: string) {
disconnectWorkspace()
const generation = connectGeneration
let token: string | undefined
let opened: MultiplayerConnection
try {
token = await signMultiplayerRequest(workspace)
opened = await MultiplayerConnection.open(workspace, workspace, new Y.Doc())
} catch (e) {
console.error('Failed to sign multiplayer request:', e)
return
}
const ydoc = new Y.Doc()
wsProvider = new WebsocketProvider(buildWsUrl('/ws_mp/'), workspace, ydoc, {
params: { token }
})
wsProvider.on('sync', (isSynced: boolean) => {
if (generation !== connectGeneration) {
opened.destroy()
return
}
connection = opened
opened.provider.on('sync', (isSynced: boolean) => {
connected = true
})
awareness = wsProvider.awareness
awareness = opened.provider.awareness
updateLocalState()
@@ -0,0 +1,61 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
import * as Y from 'yjs'
let signed = 0
vi.mock('$lib/components/debug/debugUtils', () => ({
signMultiplayerRequest: vi.fn(async () => `token-${++signed}`)
}))
vi.mock('$lib/wsUrl', () => ({ buildWsUrl: (path: string) => `ws://mp.test${path}` }))
import { MultiplayerConnection } from './multiplayerConnection'
const attempts: string[] = []
// What the multiplayer server does with a token it rejects: accept the upgrade,
// then close with 4403.
class RejectingWebSocket {
binaryType = ''
onopen: (() => void) | null = null
onclose: ((e: { code: number }) => void) | null = null
onerror: (() => void) | null = null
onmessage: (() => void) | null = null
constructor(url: string) {
attempts.push(url)
setTimeout(() => {
this.onopen?.()
this.onclose?.({ code: 4403 })
}, 0)
}
send() {}
close() {}
}
describe('MultiplayerConnection', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.stubGlobal('WebSocket', RejectingWebSocket)
attempts.length = 0
signed = 0
})
afterEach(() => {
vi.unstubAllGlobals()
vi.useRealTimers()
})
it('backs off on rejected connections and signs a fresh token for each attempt', async () => {
const connection = await MultiplayerConnection.open('ws', 'ws/room', new Y.Doc())
await vi.advanceTimersByTimeAsync(5 * 60_000)
// 1s doubling to a 60s cap, halved by jitter at worst: at most 15 in 5 minutes,
// where y-websocket's own reconnect makes thousands.
expect(attempts.length).toBeGreaterThan(3)
expect(attempts.length).toBeLessThanOrEqual(15)
const tokens = attempts.map((u) => new URL(u).searchParams.get('token'))
expect(new Set(tokens).size).toBe(attempts.length)
connection.destroy()
const after = attempts.length
await vi.advanceTimersByTimeAsync(5 * 60_000)
expect(attempts.length).toBe(after)
})
})
+82
View File
@@ -0,0 +1,82 @@
import { WebsocketProvider } from 'y-websocket'
import type * as Y from 'yjs'
import { buildWsUrl } from '$lib/wsUrl'
import { signMultiplayerRequest } from '$lib/components/debug/debugUtils'
const BASE_DELAY_MS = 1000
const MAX_DELAY_MS = 60_000
/**
* A y-websocket provider for the multiplayer server that reconnects with its own
* exponential backoff and a freshly signed token on every attempt.
*
* y-websocket's built-in reconnect is not usable here: it reuses the token it was
* built with, which lives 60s, and the server rejects a bad token only after the
* socket opened, which resets y-websocket's backoff to 100ms. Any drop past the
* first minute would otherwise retry ten times a second, forever. An attempt only
* counts as successful once the document synced.
*/
export class MultiplayerConnection {
readonly provider: WebsocketProvider
#workspace: string
#failures = 0
#timer: ReturnType<typeof setTimeout> | undefined
#destroyed = false
private constructor(workspace: string, provider: WebsocketProvider) {
this.#workspace = workspace
this.provider = provider
provider.on('sync', (synced: boolean) => {
if (synced) this.#failures = 0
})
provider.on('connection-close', () => {
if (this.#destroyed) return
// Stops y-websocket's own reconnect timer from firing setupWS.
provider.shouldConnect = false
this.#scheduleReconnect()
})
}
/** Throws when the first token cannot be signed. */
static async open(workspace: string, room: string, doc: Y.Doc): Promise<MultiplayerConnection> {
const token = await signMultiplayerRequest(workspace)
const provider = new WebsocketProvider(buildWsUrl('/ws_mp/'), room, doc, {
connect: false,
params: { token }
})
const connection = new MultiplayerConnection(workspace, provider)
provider.connect()
return connection
}
#scheduleReconnect() {
clearTimeout(this.#timer)
const ceiling = Math.min(BASE_DELAY_MS * 2 ** this.#failures, MAX_DELAY_MS)
this.#failures++
const delay = ceiling / 2 + Math.random() * (ceiling / 2)
this.#timer = setTimeout(() => this.#reconnect(), delay)
}
async #reconnect() {
let token: string
try {
token = await signMultiplayerRequest(this.#workspace)
} catch (e) {
if (this.#destroyed) return
console.error('Failed to sign multiplayer request:', e)
this.#scheduleReconnect()
return
}
if (this.#destroyed) return
const url = new URL(this.provider.url)
url.searchParams.set('token', token)
this.provider.url = url.toString()
this.provider.connect()
}
destroy() {
this.#destroyed = true
clearTimeout(this.#timer)
this.provider.destroy()
}
}