snapshot of https://github.com/docmost/docmost.git @ 6205bbeb908fe846f87dd6a2cbf562e24777db38 (2026-09-15)

This commit is contained in:
upstream-mirror
2026-09-15 15:15:34 +05:00
commit 2653349432
1615 changed files with 202632 additions and 0 deletions

View File

@@ -0,0 +1,46 @@
import type RedisClient from 'ioredis';
import type { WebSocketLike } from '@hocuspocus/server';
import type { Pack, RSAMessageClose, RSAMessageSend } from './redis-sync.types';
// Stands in for the client WebSocket on the server that owns the document.
// Outgoing traffic is relayed over redis to the origin server, which holds the real socket.
export class CollabProxySocket implements WebSocketLike {
private readonly replyTo: string;
private readonly socketId: string;
private pub: RedisClient;
private readonly pack: Pack;
readyState = 1;
onClose?: (code?: number, reason?: string) => void;
constructor(pub: RedisClient, pack: Pack, replyTo: string, socketId: string) {
this.replyTo = replyTo;
this.socketId = socketId;
this.pub = pub;
this.pack = pack;
}
private publish(msg: RSAMessageClose | RSAMessageSend) {
this.pub.publish(this.replyTo, this.pack(msg));
}
// The origin server already closed the real socket; stop relaying without echoing a close back
markClosed() {
this.readyState = 3;
}
close(code?: number, reason?: string) {
if (this.readyState !== 1) return;
this.readyState = 3;
this.onClose?.(code, reason);
}
send(message: Uint8Array) {
if (this.readyState !== 1) return;
const msg: RSAMessageSend = {
type: 'send',
socketId: this.socketId,
message,
};
this.publish(msg);
}
}

View File

@@ -0,0 +1,2 @@
export * from './redis-sync.extension';
export type { SerializedHTTPRequest } from './redis-sync.extension';

View File

@@ -0,0 +1,428 @@
// Source https://github.com/ueberdosis/hocuspocus/pull/1008 - MIT
import {
Extension,
Hocuspocus,
IncomingMessage,
onConfigurePayload,
onLoadDocumentPayload,
afterUnloadDocumentPayload,
WebSocketLike,
} from '@hocuspocus/server';
import { ConnectionTimeout, Unauthorized } from '@hocuspocus/common';
import RedisClient from 'ioredis';
import { CollabProxySocket } from './collab-proxy-socket';
import {
Configuration,
CustomEvents,
Pack,
RSAMessage,
RSAMessageClose,
RSAMessageCloseProxy,
RSAMessageCustomEventComplete,
RSAMessageCustomEventStart,
RSAMessageProxy,
RSAMessageUnload,
SerializedHTTPRequest,
Unpack,
OriginConnection,
ProxyConnection,
toWebRequest,
} from './redis-sync.types';
export type { Pack, SerializedHTTPRequest } from './redis-sync.types';
type ServerId = string;
type DocumentName = string;
type SocketId = string;
export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
priority = 1000;
private readonly pub: RedisClient;
private sub: RedisClient;
private readonly pack: Pack;
private readonly unpack: Unpack;
private originConnections: Record<SocketId, OriginConnection> = {};
private locks: Record<DocumentName, NodeJS.Timeout> = {};
private lockPromises: Record<DocumentName, Promise<ServerId | null>> = {};
private proxyConnections: Record<SocketId, ProxyConnection> = {};
private readonly prefix: string;
private readonly lockPrefix: string;
private readonly msgChannel: string;
private readonly serverId: ServerId;
private readonly customEventTTL: number;
private readonly lockTTL: number;
private instance!: Hocuspocus;
private readonly customEvents: TCE;
private replyIdCounter: number = 0;
// @ts-ignore
private pendingReplies: Record<number, PromiseWithResolvers<any>['resolve']> =
{};
private deriveContext: (
serializedHTTPRequest: SerializedHTTPRequest,
) => Record<string, any>;
constructor(configuration: Configuration<TCE>) {
const {
redis,
pack,
unpack,
serverId,
lockTTL,
prefix,
customEvents,
customEventTTL,
deriveContext,
} = configuration;
this.pub = redis.duplicate();
this.sub = redis.duplicate();
this.pack = pack;
this.unpack = unpack;
this.serverId = serverId;
this.lockTTL = lockTTL ?? 10_000;
this.customEventTTL = customEventTTL ?? 30_000;
this.prefix = prefix ?? 'collab';
this.lockPrefix = `${this.prefix}Lock`;
this.msgChannel = `${this.prefix}Msg`;
this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents);
this.deriveContext = deriveContext ?? (() => ({}));
this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`);
this.sub.on('messageBuffer', this.handleRedisMessage);
this.pub.on('error', () => {});
this.sub.on('error', () => {});
}
private getKey(documentName: string) {
return `${this.lockPrefix}:${documentName}`;
}
private closeProxy(socketId: string) {
const entry = this.proxyConnections[socketId];
if (entry) {
delete this.proxyConnections[socketId];
const { socket, clientConnection } = entry;
// The origin socket is already gone; don't echo a close message back
socket.markClosed();
clientConnection.handleClose({
code: 1000,
reason: 'provider_initiated',
});
}
}
private handleProxyMessage(
msg: Pick<RSAMessageProxy, 'replyTo' | 'message' | 'serializedHTTPRequest'>,
) {
const { replyTo, message, serializedHTTPRequest } = msg;
const { headers } = serializedHTTPRequest;
const socketId = headers['sec-websocket-key'];
let entry = this.proxyConnections[socketId];
if (!entry) {
const socket = new CollabProxySocket(
this.pub,
this.pack,
replyTo,
socketId,
);
// A proxy connection with no live documents (client left the page, auth
// failed, or the origin server crashed) is reaped by hocuspocus' message
// timeout. Dispose it silently in that case: relaying the timeout close
// to the origin would kill the client's real socket, which may be busy
// serving other documents. Genuine protocol closes are still relayed.
socket.onClose = (code, reason) => {
delete this.proxyConnections[socketId];
if (code !== ConnectionTimeout.code) {
const msg: RSAMessageClose = {
type: 'close',
code,
reason,
socketId,
};
this.pub.publish(replyTo, this.pack(msg));
}
};
const clientConnection = this.instance.handleConnection(
socket,
toWebRequest(serializedHTTPRequest),
this.deriveContext(serializedHTTPRequest),
);
entry = { clientConnection, socket };
this.proxyConnections[socketId] = entry;
}
entry.clientConnection.handleMessage(message);
}
private getLock(documentName: string) {
return this.pub.get(this.getKey(documentName));
}
private getOrClaimLock(documentName: string) {
const lockPromise = this.pub.set(
this.getKey(documentName),
this.serverId,
'PX',
this.lockTTL,
'NX',
'GET',
);
this.lockPromises[documentName] = lockPromise;
// Briefly cache the serverId that claimed the doc to reduce load on redis
// When the claimant unloads the doc, it will send an unload message to immediately clear this
// a lockTTL / 2 guarantees stale reads < lockTTL upon server crash
setTimeout(() => {
delete this.lockPromises[documentName];
}, this.lockTTL / 2);
return lockPromise;
}
private getOrClaimLockThrottled(documentName: string) {
const existingWorkerIdPromise = this.lockPromises[documentName];
if (existingWorkerIdPromise) return existingWorkerIdPromise;
return this.getOrClaimLock(documentName);
}
private handleRedisMessage = async (
_channel: Buffer,
packedMessage: Buffer,
) => {
const msg = this.unpack(packedMessage) as RSAMessage;
const { type } = msg;
if (type === 'proxy') {
this.handleProxyMessage(msg);
return;
}
if (type === 'closeProxy') {
this.closeProxy(msg.socketId);
return;
}
if (type === 'unload') {
delete this.lockPromises[msg.documentName];
return;
}
if (type === 'customEventStart') {
const { documentName, eventName, payload, replyTo, replyId } = msg;
const res = await this.handleEventLocally(
eventName as Extract<keyof TCE, string>,
documentName,
payload,
);
const reply: RSAMessageCustomEventComplete = {
type: 'customEventComplete',
replyId,
payload: res,
};
this.pub.publish(`${replyTo}`, this.pack(reply));
return;
}
if (type === 'customEventComplete') {
const { replyId, payload } = msg;
const resolveFn = this.pendingReplies[replyId];
if (!resolveFn) return;
delete this.pendingReplies[replyId];
resolveFn(payload);
return;
}
const { socketId } = msg;
const entry = this.originConnections[socketId];
if (!entry) {
// origin socket already cleaned up
return;
}
const { socket } = entry;
if (type === 'close') {
socket.close(msg.code, msg.reason);
} else if (type === 'send') {
socket.send(msg.message);
}
};
async maintainLock(documentName: string) {
this.locks[documentName] = setInterval(() => {
this.pub.set(
this.getKey(documentName),
this.serverId,
'PX',
this.lockTTL,
);
}, this.lockTTL / 2);
}
async releaseLock(documentName: string) {
clearInterval(this.locks[documentName]);
delete this.locks[documentName];
return this.pub.del(this.getKey(documentName));
}
private async handleEventLocally<TName extends Extract<keyof TCE, string>>(
eventName: TName,
documentName: string,
payload: any,
) {
const handler = this.customEvents[eventName];
if (!handler) throw new Error(`Invalid eventName: ${eventName}`);
const result = await handler(documentName, payload);
return result as Promise<ReturnType<TCE[TName]>>;
}
async handleEvent<TName extends Extract<keyof TCE, string>>(
eventName: TName,
documentName: string,
payload: any,
// if true, don't claim the lock. Useful for targeting pages that are currently open
onlyIfOpen = false,
) {
const isDocLoadedOnInstance = this.instance.documents.has(documentName);
if (isDocLoadedOnInstance) {
return this.handleEventLocally(eventName, documentName, payload);
}
const proxyTo = await (onlyIfOpen
? this.getLock(documentName)
: this.getOrClaimLockThrottled(documentName));
if (!proxyTo && onlyIfOpen) {
return;
}
if (proxyTo && proxyTo !== this.serverId) {
++this.replyIdCounter; // bug in biome thinks this.replyIdCounter is not used if written on the line below
const replyId = this.replyIdCounter;
// another server owns the doc
const proxyMessage: RSAMessageCustomEventStart = {
eventName,
documentName,
payload,
replyTo: `${this.msgChannel}:${this.serverId}`,
replyId,
type: 'customEventStart',
};
const msg = this.pack(proxyMessage);
this.pub.publish(`${this.msgChannel}:${proxyTo}`, msg);
// @ts-ignore
const { promise, resolve, reject } = Promise.withResolvers();
this.pendingReplies[replyId] = resolve;
setTimeout(() => {
delete this.pendingReplies[replyId];
reject(new Error('TIMEOUT'));
}, this.customEventTTL);
return promise as Promise<ReturnType<TCE[TName]>>;
}
// This server owns the document, but hocuspocus hasn't loaded it yet
return this.handleEventLocally(eventName, documentName, payload);
}
async lockDocument(documentName: string) {
const proxyTo = await this.getOrClaimLockThrottled(documentName);
if (proxyTo && proxyTo !== this.serverId) {
throw new Error(`Could not lock document: ${documentName}`);
}
this.maintainLock(documentName);
return () => this.releaseLock(documentName);
}
/* WebSocket Server Hooks */
onSocketOpen(
ws: WebSocketLike,
serializedHTTPRequest: SerializedHTTPRequest,
) {
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
const clientConnection = this.instance.handleConnection(
ws,
toWebRequest(serializedHTTPRequest),
this.deriveContext(serializedHTTPRequest),
);
this.originConnections[socketId] = { clientConnection, socket: ws };
}
async onSocketMessage(
serializedHTTPRequest: SerializedHTTPRequest,
detachableMsg: ArrayBuffer,
) {
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
const entry = this.originConnections[socketId];
if (!entry) return;
const { clientConnection } = entry;
let message: Uint8Array;
let documentName: string;
try {
message = new Uint8Array(detachableMsg.slice());
const tmpMsg = new IncomingMessage(detachableMsg);
const documentNameAndSessionId = tmpMsg.readVarString();
// session-aware providers suffix the documentName with \0sessionId
const sepIdx = documentNameAndSessionId.indexOf('\0');
documentName =
sepIdx === -1
? documentNameAndSessionId
: documentNameAndSessionId.slice(0, sepIdx);
} catch (error) {
entry.socket.close(Unauthorized.code, Unauthorized.reason);
return;
}
const isDocLoadedOnInstance = this.instance.documents.has(documentName);
if (isDocLoadedOnInstance) {
clientConnection.handleMessage(message);
return;
}
const proxyTo = await this.getOrClaimLockThrottled(documentName);
if (proxyTo && proxyTo !== this.serverId) {
// Proxied messages bypass handleMessage, so refresh the connection's
// liveness fields manually or hocuspocus' message timeout would reap the
// real socket every `timeout` ms. connectionEstablishedAt is the
// reference while unauthenticated (auth for remote docs is proxied too)
// and is private upstream.
clientConnection.lastMessageReceivedAt = Date.now();
(clientConnection as any).connectionEstablishedAt = Date.now();
// another server owns the doc
const proxyMessage: RSAMessageProxy = {
serializedHTTPRequest: serializedHTTPRequest,
replyTo: `${this.msgChannel}:${this.serverId}`,
message,
type: 'proxy',
};
const msg = this.pack(proxyMessage);
this.pub.publish(`${this.msgChannel}:${proxyTo}`, msg);
return;
}
// This server owns the document, but hocuspocus hasn't loaded it yet
clientConnection.handleMessage(message);
}
onSocketClose(socketId: string, code?: number, reason?: ArrayBuffer) {
const entry = this.originConnections[socketId];
if (!entry) return;
delete this.originConnections[socketId];
entry.clientConnection.handleClose({
code: code ?? 1000,
reason: reason ? Buffer.from(reason).toString() : '',
});
const msg: RSAMessageCloseProxy = { type: 'closeProxy', socketId };
this.pub.publish(this.msgChannel, this.pack(msg)).catch(() => {});
}
/* Hocuspocus hooks */
async onConfigure({ instance }: onConfigurePayload) {
this.instance = instance;
}
async onLoadDocument(data: onLoadDocumentPayload) {
const { documentName } = data;
// Refresh the lock TTL
this.maintainLock(documentName);
}
async afterUnloadDocument(data: afterUnloadDocumentPayload) {
const { documentName } = data;
this.releaseLock(documentName);
// Broadcast to cluster to immediately remove the cached redis value
const msg: RSAMessageUnload = { type: 'unload', documentName };
this.pub.publish(this.msgChannel, this.pack(msg));
}
async onDestroy() {
this.pendingReplies = {};
this.pub.disconnect(false);
this.sub.disconnect(false);
}
}

View File

@@ -0,0 +1,138 @@
import { IncomingHttpHeaders } from 'node:http2';
import RedisClient from 'ioredis';
import { CollabProxySocket } from './collab-proxy-socket';
import { type Hocuspocus, type WebSocketLike } from '@hocuspocus/server';
export type SecondParam<T> = T extends (
arg1: any,
arg2: infer A,
...args: any[]
) => any
? A
: never;
export type SerializedHTTPRequest = {
method: string;
url: string;
headers: IncomingHttpHeaders;
socket: { remoteAddress: string };
};
export type RSAMessageProxy = {
type: 'proxy';
replyTo: string;
message: Uint8Array<ArrayBufferLike>;
serializedHTTPRequest: SerializedHTTPRequest;
};
export type RSAMessageCloseProxy = {
type: 'closeProxy';
socketId: string;
};
export type RSAMessageUnload = {
type: 'unload';
documentName: string;
};
export type RSAMessageClose = {
type: 'close';
code?: number;
reason?: string;
socketId: string;
};
export type RSAMessageSend = {
type: 'send';
// @ts-ignore
message: Uint8Array<ArrayBufferLike>;
socketId: string;
};
export type RSAMessageCustomEventStart<TName = string, TPayload = any> = {
type: 'customEventStart';
documentName: string;
eventName: TName;
payload: TPayload;
replyTo: string;
replyId: number;
};
export type RSAMessageCustomEventComplete = {
type: 'customEventComplete';
replyId: number;
payload: any;
};
export type RSAMessage =
| RSAMessageProxy
| RSAMessageCloseProxy
| RSAMessageUnload
| RSAMessageClose
| RSAMessageSend
| RSAMessageCustomEventStart
| RSAMessageCustomEventComplete;
// @ts-ignore
export type Pack = (msg: RSAMessage) => string | Buffer<ArrayBufferLike>;
export type Unpack = (
// @ts-ignore
packedMessage: Uint8Array | Buffer<ArrayBufferLike>,
) => RSAMessage;
type ServerId = string;
type DocumentName = string;
type CustomEventName = string;
export type CustomEvents = Record<
CustomEventName,
(documentName: string, payload: any) => Promise<any>
>;
// Not exported by @hocuspocus/server
export type ClientConnection = ReturnType<Hocuspocus['handleConnection']>;
export type OriginConnection = {
clientConnection: ClientConnection;
socket: WebSocketLike;
};
export type ProxyConnection = {
clientConnection: ClientConnection;
socket: CollabProxySocket;
};
export interface Configuration<TCE> {
redis: RedisClient;
pack: Pack;
unpack: Unpack;
serverId: ServerId;
lockTTL?: number;
customEventTTL?: number;
prefix?: string;
customEvents?: TCE;
// Derive the hocuspocus context once per socket instead of re-deriving it in a
// per-document hook like onConnect/onAuthenticate. Runs on the origin server when
// the socket opens and on the doc owner when the first proxied message arrives.
deriveContext?: (
serializedHTTPRequest: SerializedHTTPRequest,
) => Record<string, any>;
}
// Hocuspocus expects a web-standard Request, so rehydrate one from what crossed the wire
export const toWebRequest = (serializedHTTPRequest: SerializedHTTPRequest) => {
const { method, url, headers } = serializedHTTPRequest;
const webHeaders = new Headers();
Object.entries(headers).forEach(([name, value]) => {
if (Array.isArray(value)) {
value.forEach((v) => {
webHeaders.append(name, v);
});
} else if (value !== undefined) {
webHeaders.set(name, value);
}
});
return new Request(new URL(url, 'http://localhost'), {
method,
headers: webHeaders,
});
};

View File

@@ -0,0 +1,35 @@
import type WebSocket from 'ws';
import type { WebSocketLike } from '@hocuspocus/server';
/**
* Wrapper around ws WebSocket that Hocuspocus only writes to.
* Incoming socket events are forwarded separately by the gateway,
* which prevents double-handling with RedisSyncExtension.
*/
export class WsSocketWrapper implements WebSocketLike {
private ws: WebSocket;
readyState = 1;
constructor(ws: WebSocket) {
this.ws = ws;
}
close(code?: number, reason?: string) {
if (this.readyState !== 1) return;
this.readyState = 3;
try {
this.ws.close(code, reason);
} catch (e) {
// Socket already closed
}
}
send(message: Uint8Array) {
if (this.readyState !== 1) return;
try {
this.ws.send(message);
} catch (e) {
// Socket already closed
}
}
}