snapshot of https://github.com/RocketChat/Rocket.Chat.git (develop, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Create issues from TODOs / todos (push) Failing after 3s
Close inactive issues / close-issues (push) Has been cancelled
Deploy GitHub Pages / deploy-preview (push) Has been cancelled
CI / ⚙️ Variables Setup (push) Has been cancelled
CI / ⚙️ Test Guard (push) Has been cancelled
CI / 🚀 Notify external services - draft (push) Has been cancelled
CI / 📦 Build Packages (push) Has been cancelled
CI / 📦 Meteor Build (coverage) (push) Has been cancelled
CI / 📦 Meteor Build (production) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [account-service presence-service omnichannel-transcript-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [account-service presence-service omnichannel-transcript-service], fips) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [authorization-service queue-worker-service ddp-streamer-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [authorization-service queue-worker-service ddp-streamer-service], fips) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], coverage) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], fips) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [account-service presence-service omnichannel-transcript-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [authorization-service queue-worker-service ddp-streamer-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [rocketchat], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [rocketchat], coverage) (push) Has been cancelled
CI / 🚢 Publish Docker Images (ghcr.io) (push) Has been cancelled
CI / 📦 Track Image Sizes (push) Has been cancelled
CI / 🔎 Code Check - Action Lint (push) Has been cancelled
CI / 🔎 Code Check (push) Has been cancelled
CI / 🔨 Test Storybook (push) Has been cancelled
CI / 🔨 Test Unit (push) Has been cancelled
CI / 🔨 Test API (CE) (push) Has been cancelled
CI / 🔨 Test API Livechat (CE) (push) Has been cancelled
CI / 🔨 Test UI (CE) (push) Has been cancelled
CI / 🔨 Test API (EE) (push) Has been cancelled
CI / 🔨 Test API Livechat (EE) (push) Has been cancelled
CI / 🔨 Test API Apps (deno-runtime - EE) (push) Has been cancelled
CI / 🔨 Test UI (EE) (push) Has been cancelled
CI / 🔨 Test API (FIPS) (push) Has been cancelled
CI / 🔨 Test API Livechat (FIPS) (push) Has been cancelled
CI / 🔨 Test UI (FIPS) (push) Has been cancelled
CI / 🔨 Test Federation Matrix (push) Has been cancelled
CI / 📊 Report Coverage (push) Has been cancelled
CI / ✅ Tests Done (push) Has been cancelled
CI / 🚀 Publish build assets (push) Has been cancelled
CI / 🚀 Publish Docker Images (DockerHub) (push) Has been cancelled
CI / 🚀 Notify external services (push) Has been cancelled
CI / Update Version Durability (push) Has been cancelled
Release / Release (push) Has been cancelled
Auto-close duplicate issues / auto-close-duplicates (push) Failing after 1s
Code scanning - action / CodeQL-Build (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-16 01:21:02 +05:00
commit 30810eecd7
10161 changed files with 987110 additions and 0 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,16 @@
import 'ws';
declare module 'ws' {
namespace Sender {
function frame(
data: Buffer,
options: {
opcode: number; // The opcode
readOnly: boolean; // Specifies whether `data` can be modified
fin: boolean; // Specifies whether or not to set the FIN bit
mask: boolean; // Specifies whether or not to mask `data`
rsv1: boolean; // Specifies whether or not to set the RSV1 bit
},
): Buffer[];
}
}

View File

@@ -0,0 +1,7 @@
import server from '@rocket.chat/jest-presets/server';
import type { Config } from 'jest';
export default {
preset: server.preset,
testMatch: ['<rootDir>/src/**/*.spec.(ts|js|mjs)'],
} satisfies Config;

View File

@@ -0,0 +1,71 @@
{
"name": "@rocket.chat/ddp-streamer",
"version": "0.4.1",
"private": true,
"description": "Rocket.Chat DDP-Streamer service",
"keywords": [
"rocketchat"
],
"author": "Rocket.Chat",
"main": "./dist/service.js",
"files": [
"/dist"
],
"scripts": {
"build": "tsc -p tsconfig.json",
"lint": "eslint .",
"ms": "TRANSPORTER=${TRANSPORTER:-TCP} MONGO_URL=${MONGO_URL:-mongodb://localhost:3001/meteor} ts-node --files src/service.ts",
"test": "jest",
"testunit": "jest",
"typecheck": "tsc --noEmit --skipLibCheck -p tsconfig.json"
},
"dependencies": {
"@rocket.chat/core-services": "workspace:^",
"@rocket.chat/core-typings": "workspace:^",
"@rocket.chat/emitter": "workspace:~",
"@rocket.chat/instance-status": "workspace:^",
"@rocket.chat/logger": "workspace:^",
"@rocket.chat/model-typings": "workspace:^",
"@rocket.chat/models": "workspace:^",
"@rocket.chat/network-broker": "workspace:^",
"@rocket.chat/rest-typings": "workspace:^",
"@rocket.chat/tracing": "workspace:^",
"colorette": "^1.4.0",
"ejson": "^2.2.3",
"eventemitter3": "^5.0.4",
"jaeger-client": "^3.19.0",
"mem": "^8.1.1",
"moleculer": "^0.14.35",
"mongodb": "6.16.0",
"nats": "^2.28.2",
"pino": "10.3.1",
"polka": "^0.5.2",
"prom-client": "^14.2.0",
"prometheus-gc-stats": "^1.1.0",
"sharp": "^0.35.4",
"underscore": "^1.13.8",
"uuid": "^11.1.1",
"ws": "~8.21.3"
},
"devDependencies": {
"@rocket.chat/apps-engine": "workspace:^",
"@rocket.chat/ddp-client": "workspace:~",
"@rocket.chat/jest-presets": "workspace:*",
"@types/ejson": "^2.2.2",
"@types/jest": "~30.0.0",
"@types/node": "~24.13.2",
"@types/polka": "^0.5.8",
"@types/prometheus-gc-stats": "^0.6.4",
"@types/underscore": "^1.13.0",
"@types/uuid": "^10.0.0",
"@types/ws": "^8.18.1",
"eslint": "~9.39.5",
"jest": "~30.2.0",
"pino-pretty": "13.1.3",
"ts-node": "^10.9.2",
"typescript": "~5.9.3"
},
"volta": {
"extends": "../../../package.json"
}
}

View File

@@ -0,0 +1,239 @@
import { EventEmitter } from 'events';
import type { IncomingMessage } from 'http';
import { Presence } from '@rocket.chat/core-services';
import type { ISocketConnection } from '@rocket.chat/core-typings';
import { throttle } from 'underscore';
import { v1 as uuidv1 } from 'uuid';
import type WebSocket from 'ws';
import { SERVER_ID } from './Server';
import { server } from './configureServer';
import { DDP_EVENTS, WS_ERRORS, WS_ERRORS_MESSAGES, TIMEOUT } from './constants';
import type { IPacket } from './types/IPacket';
// TODO why localhost not as 127.0.0.1?
// based on Meteor's implementation (link)
const getClientAddress = (req: IncomingMessage): string | undefined => {
// For the reported client address for a connection to be correct,
// the developer must set the HTTP_FORWARDED_COUNT environment
// variable to an integer representing the number of hops they
// expect in the `x-forwarded-for` header. E.g., set to "1" if the
// server is behind one proxy.
//
// This could be computed once at startup instead of every time.
const httpForwardedCount = parseInt(process.env.HTTP_FORWARDED_COUNT || '') || 0;
if (httpForwardedCount === 0) {
return req.socket.remoteAddress;
}
const forwardedFor =
(req.headers['x-forwarded-for'] && Array.isArray(req.headers['x-forwarded-for'])
? req.headers['x-forwarded-for'][0]
: req.headers['x-forwarded-for']) || '';
if (!forwardedFor) {
return;
}
const forwardedForClean = forwardedFor
.trim()
.split(',')
.map((ip) => ip.trim());
// Typically the first value in the `x-forwarded-for` header is
// the original IP address of the client connecting to the first
// proxy. However, the end user can easily spoof the header, in
// which case the first value(s) will be the fake IP address from
// the user pretending to be a proxy reporting the original IP
// address value. By counting HTTP_FORWARDED_COUNT back from the
// end of the list, we ensure that we get the IP address being
// reported by *our* first proxy.
if (httpForwardedCount < 0 || httpForwardedCount > forwardedForClean.length) {
return;
}
return forwardedForClean[forwardedForClean.length - httpForwardedCount];
};
export const clientMap = new WeakMap<WebSocket, Client>();
export class Client extends EventEmitter {
private chain = Promise.resolve();
protected timeout: NodeJS.Timeout;
public readonly session = uuidv1();
public subscriptions = new Map();
public connection: ISocketConnection;
public wait = false;
public userId?: string;
public userToken?: string;
private updatePresence = throttle(
() => {
if (this.userId) {
void Presence.updateConnection(this.userId, this.connection.id).catch((err) => {
console.error('Error updating connection presence:', err);
});
}
},
TIMEOUT,
{ leading: true, trailing: false },
);
constructor(
public ws: WebSocket,
public meteorClient = false,
req: IncomingMessage,
) {
super();
this.connection = {
id: this.session,
instanceId: server.id,
onClose: (fn): void => {
this.on('close', fn);
},
clientAddress: getClientAddress(req),
httpHeaders: req.headers,
};
this.renewTimeout(TIMEOUT / 1000);
this.ws.on('message', this.handler);
this.ws.on('close', (...args) => {
server.emit(DDP_EVENTS.DISCONNECTED, this);
this.emit('close', ...args);
this.subscriptions.clear();
clearTimeout(this.timeout);
});
this.ws.on('error', (err) => {
console.error('Unexpected error:', err);
this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR, WS_ERRORS_MESSAGES.CLOSE_PROTOCOL_ERROR);
});
this.setMaxListeners(50);
this.greeting();
server.emit(DDP_EVENTS.CONNECTED, this);
this.ws.on('message', () => this.renewTimeout(TIMEOUT));
this.once('message', ({ msg }) => {
if (msg !== DDP_EVENTS.CONNECT) {
return this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR, WS_ERRORS_MESSAGES.CLOSE_PROTOCOL_ERROR);
}
return this.send(server.serialize({ [DDP_EVENTS.MSG]: DDP_EVENTS.CONNECTED, session: this.session }));
});
this.send(SERVER_ID);
clientMap.set(ws, this);
}
greeting(): void {
// no greeting by default
if (this.meteorClient) {
return this.ws.send('o');
}
}
async callMethod(packet: IPacket): Promise<void> {
this.chain = this.chain.then(() => server.call(this, packet)).catch();
}
async callSubscribe(packet: IPacket): Promise<void> {
this.chain = this.chain.then(() => server.subscribe(this, packet)).catch();
}
process(action: string, packet: IPacket): void {
switch (action) {
case DDP_EVENTS.PING:
this.pong(packet.id);
break;
case DDP_EVENTS.METHOD:
if (!packet.method) {
return this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR);
}
if (!packet.id) {
return this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR);
}
void this.callMethod(packet);
break;
case DDP_EVENTS.SUBSCRIBE:
if (!packet.name) {
return this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR);
}
if (!packet.id) {
return this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR);
}
void this.callSubscribe(packet);
break;
case DDP_EVENTS.UNSUBSCRIBE:
if (!packet.id) {
return this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR);
}
const subscription = this.subscriptions.get(packet.id);
if (!subscription) {
return;
}
subscription.stop();
break;
}
}
closeTimeout = (): void => {
this.ws.close(WS_ERRORS.TIMEOUT, WS_ERRORS_MESSAGES.TIMEOUT);
};
ping(id?: string): void {
this.send(server.serialize({ [DDP_EVENTS.MSG]: DDP_EVENTS.PING, ...(id && { [DDP_EVENTS.ID]: id }) }));
}
pong(id?: string): void {
this.send(server.serialize({ [DDP_EVENTS.MSG]: DDP_EVENTS.PONG, ...(id && { [DDP_EVENTS.ID]: id }) }));
}
handleIdle = (): void => {
this.ping();
this.timeout = setTimeout(this.closeTimeout, TIMEOUT);
};
renewTimeout(timeout = TIMEOUT): void {
clearTimeout(this.timeout);
this.timeout = setTimeout(this.handleIdle, timeout);
}
handler = async (payload: WebSocket.Data, isBinary: boolean): Promise<void> => {
try {
const packet = server.parse(payload, isBinary);
this.updatePresence();
this.emit('message', packet);
if (this.wait) {
return new Promise((resolve) => this.once(DDP_EVENTS.LOGGED, () => resolve(this.process(packet.msg, packet))));
}
this.process(packet.msg, packet);
} catch (err) {
console.error(err);
return this.ws.close(WS_ERRORS.UNSUPPORTED_DATA, WS_ERRORS_MESSAGES.UNSUPPORTED_DATA);
}
};
encodePayload(payload: string): string {
if (this.meteorClient) {
return `a${JSON.stringify([payload])}`;
}
return payload;
}
send(payload: string): void {
return this.ws.send(this.encodePayload(payload));
}
}

View File

@@ -0,0 +1,308 @@
import crypto from 'crypto';
import { MeteorService, Presence, ServiceClass } from '@rocket.chat/core-services';
import { InstanceStatus } from '@rocket.chat/instance-status';
import { Users } from '@rocket.chat/models';
import polka from 'polka';
import { throttle } from 'underscore';
import WebSocket from 'ws';
import { Client, clientMap } from './Client';
import { events, server } from './configureServer';
import { DDP_EVENTS } from './constants';
import { Autoupdate } from './lib/Autoupdate';
import { proxy } from './proxy';
import { ListenersModule } from '../../../../apps/meteor/server/modules/listeners/listeners.module';
import type { NotificationsModule } from '../../../../apps/meteor/server/modules/notifications/notifications.module';
import { invalidate as invalidatePublicationUserCache } from '../../../../apps/meteor/server/modules/streamer/publication-user-cache';
import { StreamerCentral } from '../../../../apps/meteor/server/modules/streamer/streamer.module';
const { PORT = 4000 } = process.env;
export class DDPStreamer extends ServiceClass {
protected name = 'streamer';
private app?: polka.Polka;
private wss?: WebSocket.Server;
constructor(notifications: NotificationsModule) {
super();
new ListenersModule(this, notifications);
// TODO this is triggered by local events too, need to find a way to ignore if it's local
this.onEvent('stream', ([streamer, eventName, args]): void => {
// TODO rename StreamerCentral to StreamerStore or something to use it only as a store
const stream = StreamerCentral.instances[streamer];
return stream?.emitWithoutBroadcast(eventName, ...args);
});
this.onEvent('watch.loginServiceConfiguration', ({ clientAction, id, data }) => {
if (clientAction === 'removed') {
events.emit('meteor.loginServiceConfiguration', 'removed', {
_id: id,
});
return;
}
if (data) {
events.emit('meteor.loginServiceConfiguration', clientAction === 'inserted' ? 'added' : 'changed', data);
}
});
this.onEvent('user.forceLogout', (uid: string, sessionId?: string) => {
this.wss?.clients.forEach((ws) => {
const client = clientMap.get(ws);
if (sessionId) {
if (client?.connection.id === sessionId) {
ws.close();
}
return;
}
if (client?.userId === uid) {
// Graceful close: lets the WS lib flush queued frames (including
// the `notify-user/<uid>/force_logout` stream message that the
// monolith listener at apps/meteor/server/modules/listeners/listeners.module.ts:49
// just enqueued) before the socket goes down. Previously this was
// `ws.terminate()`, which sends a TCP RST immediately and drops
// the queued frames — clients depending on the stream message
// (useForceLogout hook → Accounts._unstoreLoginToken + setUserId(null))
// then never see the cleanup, leaving stale credentials in
// localStorage. Falls back to terminate() after a short grace
// period for unresponsive sockets.
ws.close();
const guard = setTimeout(() => {
if (ws.readyState !== ws.CLOSED) {
ws.terminate();
}
}, 5000);
ws.once('close', () => clearTimeout(guard));
}
});
});
this.onEvent('meteor.clientVersionUpdated', (versions): void => {
Autoupdate.updateVersion(versions);
});
// The publication user cache lives inside the NotificationsModule process. In a
// microservices deployment, ddp-streamer hosts that module but does not host
// MeteorService, so the watch.users invalidation registered in
// apps/meteor/server/services/meteor/service.ts never reaches this process.
// Without this listener, role/ban/user changes would only propagate after the
// 60s TTL expires.
this.onEvent('watch.users', (data): void => {
invalidatePublicationUserCache(data.id);
});
}
// update connections count every 30 seconds
updateConnections = throttle(() => {
void InstanceStatus.updateConnections(this.wss?.clients.size ?? 0);
}, 30000);
override async created(): Promise<void> {
if (!this.context) {
return;
}
const { broker, nodeID } = this.context;
if (!broker || !nodeID) {
return;
}
const { metrics } = broker;
if (!metrics) {
return;
}
metrics.register({
name: 'rocketchat_subscription',
type: 'histogram',
labelNames: ['subscription'],
description: 'Client subscriptions to Rocket.Chat',
unit: 'millisecond',
quantiles: true,
});
metrics.register({
name: 'users_connected',
type: 'gauge',
labelNames: ['nodeID'],
description: 'Users connected by streamer',
});
metrics.register({
name: 'users_logged',
type: 'gauge',
labelNames: ['nodeID'],
description: 'Users logged by streamer',
});
server.setMetrics(metrics);
server.on(DDP_EVENTS.CONNECTED, () => {
metrics.increment('users_connected', { nodeID }, 1);
});
server.on(DDP_EVENTS.LOGGED, () => {
metrics.increment('users_logged', { nodeID }, 1);
});
server.on(DDP_EVENTS.DISCONNECTED, ({ userId }) => {
metrics.decrement('users_connected', { nodeID }, 1);
if (userId) {
metrics.decrement('users_logged', { nodeID }, 1);
}
});
async function sendUserData(client: Client, userId: string) {
// TODO figure out what fields to send. maybe to to export function getBaseUserFields to a package
const loggedUser = await Users.findOneById(userId, {
projection: {
'name': 1,
'username': 1,
'nickname': 1,
'emails': 1,
'status': 1,
'statusDefault': 1,
'statusText': 1,
'statusConnection': 1,
'bio': 1,
'avatarOrigin': 1,
'utcOffset': 1,
'language': 1,
'settings': 1,
'enableAutoAway': 1,
'idleTimeLimit': 1,
'roles': 1,
'active': 1,
'defaultRoom': 1,
'customFields': 1,
'requirePasswordChange': 1,
'requirePasswordChangeReason': 1,
'statusLivechat': 1,
'banners': 1,
'oauth.authorizedClients': 1,
'_updatedAt': 1,
'avatarETag': 1,
'openBusinessHours': 1,
'services.totp.enabled': 1,
'services.email2fa.enabled': 1,
},
});
if (!loggedUser) {
return;
}
// using setImmediate here so login's method result is sent before we send the user data
setImmediate(async () => server.added(client, 'users', userId, loggedUser));
}
server.on(DDP_EVENTS.LOGGED, async (info: Client) => {
const { userId, connection } = info;
if (!userId) {
throw new Error('User not logged in');
}
await Presence.newConnection(userId, connection.id, nodeID);
this.updateConnections();
// mimic Meteor's default publication that sends user data after login
await sendUserData(info, userId);
server.emit('presence', { userId, connection });
void this.api?.broadcast('accounts.login', { userId, connection });
});
server.on(DDP_EVENTS.LOGGEDOUT, (info) => {
const { userId, connection } = info;
void this.api?.broadcast('accounts.logout', { userId, connection });
void this.updateConnections();
if (!userId) {
return;
}
void Presence.removeConnection(userId, connection.id, nodeID);
});
server.on(DDP_EVENTS.DISCONNECTED, (info) => {
const { userId, connection } = info;
void this.api?.broadcast('socket.disconnected', connection);
this.updateConnections();
if (!userId) {
return;
}
void Presence.removeConnection(userId, connection.id, nodeID);
});
server.on(DDP_EVENTS.CONNECTED, ({ connection }) => {
void this.api?.broadcast('socket.connected', connection);
});
}
override async started(): Promise<void> {
// TODO this call creates a dependency to MeteorService, should it be a hard dependency? or can this call fail and be ignored?
try {
const versions = await MeteorService.getAutoUpdateClientVersions();
Object.keys(versions || {}).forEach((key) => {
Autoupdate.updateVersion(versions[key]);
});
this.app = polka()
.use(proxy())
.get('/health', async (_req, res) => {
try {
if (!this.api) {
throw new Error('API not available');
}
await this.api.nodeList();
res.end('ok');
} catch (err) {
console.error('Service not healthy', err);
res.writeHead(500);
res.end('not healthy');
}
})
.get('*', function (_req, res) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json');
res.writeHead(200);
res.end(
`{"websocket":true,"origins":["*:*"],"cookie_needed":false,"entropy":${crypto.randomBytes(4).readUInt32LE(0)},"ms":true}`,
);
})
.listen(PORT);
this.wss = new WebSocket.Server({ server: this.app.server });
this.wss.on('connection', (ws, req) => new Client(ws, req.url !== '/websocket', req));
void InstanceStatus.registerInstance('ddp-streamer', {});
} catch (err) {
console.error('DDPStreamer did not start correctly', err);
}
}
override async stopped(): Promise<void> {
this.wss?.clients.forEach(function (client) {
client.terminate();
});
this.app?.server?.close();
this.wss?.close();
}
}

View File

@@ -0,0 +1,73 @@
import { EventEmitter } from 'events';
import type { Client } from './Client';
import type { Server } from './Server';
import type { IPacket } from './types/IPacket';
import type { IPublication } from '../../../../apps/meteor/server/modules/streamer/types';
export class Publication extends EventEmitter implements IPublication {
_session: IPublication['_session'];
connection: IPublication['connection'];
constructor(
public client: Client,
private packet: IPacket,
private server: Server,
) {
super();
this.packet = packet;
client.subscriptions.set(packet.id, this);
client.once('close', () => this.emit('stop', this.client, this.packet));
this.once('stop', () => client.subscriptions.delete(packet.id));
this._session = {
sendAdded: this.added.bind(this),
socket: client,
userId: client.userId,
};
this.connection = client.connection;
}
_isDeactivated(): this is IPublication & { _session: null } {
return this._session === null;
}
error(_error: Error): void {
throw new Error('Method not implemented.');
}
unblock(): void {
throw new Error('Method not implemented.');
}
ready(): void {
return this.server.ready(this.client, this.packet);
}
stop(): void {
this.server.nosub(this.client, this.packet);
this.emit('stop', this.client, this.packet);
}
onStop(fn: (...args: any[]) => void): void {
this.once('stop', fn);
}
added(collection: string, id: string, fields: any): void {
this.server.added(this.client, collection, id, fields);
}
changed(collection: string, id: string, fields: any): void {
this.server.changed(this.client, collection, id, fields);
}
removed(collection: string, id: string): void {
this.server.removed(this.client, collection, id);
}
get userId() {
return this.client.userId ?? null;
}
}

View File

@@ -0,0 +1,108 @@
import { MeteorService } from '@rocket.chat/core-services';
import WebSocket from 'ws';
import { Server } from './Server';
import type { IPacket } from './types/IPacket';
jest.mock('@rocket.chat/core-services', () => ({
...jest.requireActual('@rocket.chat/core-services'),
MeteorService: {
callMethodWithToken: jest.fn(),
},
}));
jest.mock('@rocket.chat/logger', () => ({
Logger: jest.fn().mockReturnValue({
error: jest.fn(),
}),
}));
const mockCallMethodWithToken = jest.mocked(MeteorService.callMethodWithToken);
function makeClient(readyState: number = WebSocket.OPEN) {
return {
ws: { readyState },
userId: 'user1',
userToken: 'token1',
send: jest.fn(),
} as unknown as Parameters<Server['call']>[0];
}
function makePacket(method: string, id = 'test-id'): IPacket {
return { msg: 'method', method, id, params: [] } as unknown as IPacket;
}
describe('Server.call', () => {
let server: Server;
beforeEach(() => {
server = new Server();
jest.clearAllMocks();
});
describe('when the method is delegated to MeteorService', () => {
it('returns the result value from MeteorService', async () => {
mockCallMethodWithToken.mockResolvedValue({ result: 'some-value' } as any);
const client = makeClient();
const resultSpy = jest.spyOn(server, 'result');
await server.call(client, makePacket('someMethod'));
expect(resultSpy).toHaveBeenCalledWith(client, expect.objectContaining({ id: 'test-id' }), 'some-value');
});
it('does not return an error when the method returns void', async () => {
mockCallMethodWithToken.mockResolvedValue({ result: undefined } as any);
const client = makeClient();
const resultSpy = jest.spyOn(server, 'result');
await server.call(client, makePacket('setAvatarFromService'));
expect(resultSpy).toHaveBeenCalledWith(client, expect.objectContaining({ id: 'test-id' }), undefined);
});
it('calls result with an error when MeteorService throws', async () => {
mockCallMethodWithToken.mockRejectedValue(new Error('boom'));
const client = makeClient();
const resultSpy = jest.spyOn(server, 'result');
await server.call(client, makePacket('someMethod'));
expect(resultSpy).toHaveBeenCalledWith(client, expect.objectContaining({ id: 'test-id' }), null, expect.any(Error));
});
});
describe('when the method is registered locally', () => {
it('returns the result value from the local method', async () => {
server.methods({ localMethod: async () => 'local-result' });
const client = makeClient();
const resultSpy = jest.spyOn(server, 'result');
await server.call(client, makePacket('localMethod'));
expect(resultSpy).toHaveBeenCalledWith(client, expect.objectContaining({ id: 'test-id' }), 'local-result');
});
it('does not return an error when the local method returns void', async () => {
server.methods({ voidMethod: async () => undefined });
const client = makeClient();
const resultSpy = jest.spyOn(server, 'result');
await server.call(client, makePacket('voidMethod'));
expect(resultSpy).toHaveBeenCalledWith(client, expect.objectContaining({ id: 'test-id' }), undefined);
});
});
describe('when the client WebSocket is not open', () => {
it('does nothing', async () => {
const client = makeClient(WebSocket.CLOSED);
const resultSpy = jest.spyOn(server, 'result');
await server.call(client, makePacket('anyMethod'));
expect(resultSpy).not.toHaveBeenCalled();
expect(mockCallMethodWithToken).not.toHaveBeenCalled();
});
});
});

View File

@@ -0,0 +1,199 @@
import { EventEmitter } from 'events';
import type { IServiceMetrics } from '@rocket.chat/core-services';
import { MeteorService, isMeteorError, MeteorError } from '@rocket.chat/core-services';
import { Logger } from '@rocket.chat/logger';
import ejson from 'ejson';
import { v1 as uuidv1 } from 'uuid';
import WebSocket from 'ws';
import type { Client } from './Client';
import { Publication } from './Publication';
import { DDP_EVENTS } from './constants';
import type { IPacket } from './types/IPacket';
const logger = new Logger('DDP-Streamer');
type SubscriptionFn = (this: Publication, eventName: string, options: object) => void;
type MethodFn = (this: Client, ...args: any[]) => any;
type Methods = {
[k: string]: MethodFn;
};
const handleInternalException = (err: unknown, msg: string): MeteorError => {
if (err instanceof MeteorError) {
return err;
}
// default errors are logged to the console and redacted from the client
// TODO switch to using the logger (ideally broker.logger)
logger.error({ msg, err });
return new MeteorError(500, 'Internal server error');
};
export const SERVER_ID = ejson.stringify({ msg: 'server_id', server_id: '0' });
export class Server extends EventEmitter {
private _subscriptions = new Map<string, SubscriptionFn>();
private _methods = new Map<string, MethodFn>();
private metrics?: IServiceMetrics;
public readonly id = uuidv1();
serialize = ejson.stringify;
parse = (data: WebSocket.Data, isBinary: boolean): IPacket => {
if (isBinary) {
throw new MeteorError(500, 'Binary data not supported');
}
const packet = data.toString();
const payload = packet.startsWith('[') ? JSON.parse(packet)[0] : packet;
return ejson.parse(payload);
};
setMetrics(metrics: IServiceMetrics): void {
this.metrics = metrics;
}
async call(client: Client, packet: IPacket): Promise<void> {
// if client is not connected we don't need to do anything
if (client.ws.readyState !== WebSocket.OPEN) {
return;
}
try {
// if method was not defined on DDP Streamer we fall back to Meteor
if (!this._methods.has(packet.method)) {
const result = await MeteorService.callMethodWithToken(client.userId, client.userToken, packet.method, packet.params);
return this.result(client, packet, result.result);
}
const fn = this._methods.get(packet.method);
if (!fn) {
throw new MeteorError(404, `Method '${packet.method}' not found`);
}
const result = await fn.apply(client, packet.params);
return this.result(client, packet, result);
} catch (err: unknown) {
return this.result(client, packet, null, handleInternalException(err, 'Method call error'));
}
}
methods(obj: Methods): void {
Object.entries(obj).forEach(([name, fn]) => {
if (this._methods.has(name)) {
return;
}
this._methods.set(name, fn);
});
}
async subscribe(client: Client, packet: IPacket): Promise<void> {
// if client is not connected we don't need to do anything
if (client.ws.readyState !== WebSocket.OPEN) {
return;
}
try {
if (!this._subscriptions.has(packet.name)) {
throw new MeteorError(404, `Subscription '${packet.name}' not found`);
}
const fn = this._subscriptions.get(packet.name);
if (!fn) {
throw new MeteorError(404, `Subscription '${packet.name}' not found`);
}
const end = this.metrics?.timer('rocketchat_subscription', { subscription: packet.name });
const publication = new Publication(client, packet, this);
const [eventName, options] = packet.params;
await fn.call(publication, eventName, options);
end?.();
} catch (err: unknown) {
return this.nosub(client, packet, handleInternalException(err, 'Subscription error'));
}
}
publish(name: string, fn: SubscriptionFn): void {
if (this._subscriptions.has(name)) {
return;
}
this._subscriptions.set(name, fn);
}
stream(stream: string, fn: SubscriptionFn): void {
return this.publish(`stream-${stream}`, fn);
}
result(client: Client, { id }: IPacket, result?: any, error?: Error | MeteorError): void {
client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.RESULT,
id,
...(result && { result }),
...(error && { error: isMeteorError(error) ? error.toJSON() : error }),
}),
);
return client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.UPDATED,
[DDP_EVENTS.METHODS]: [id],
}),
);
}
nosub(client: Client, { id }: IPacket, error?: Error | MeteorError): void {
return client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.NO_SUBSCRIBE,
id,
...(error && { error: isMeteorError(error) ? error.toJSON() : error }),
}),
);
}
ready(client: Client, packet: IPacket): void {
return client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.READY,
[DDP_EVENTS.SUBSCRIPTIONS]: [packet.id],
}),
);
}
added(client: Client, collection: string, id: string, fields: any): void {
return client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.ADDED,
[DDP_EVENTS.COLLECTION]: collection,
[DDP_EVENTS.ID]: id,
[DDP_EVENTS.FIELDS]: fields,
}),
);
}
changed(client: Client, collection: string, id: string, fields: any): void {
return client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.CHANGED,
[DDP_EVENTS.COLLECTION]: collection,
[DDP_EVENTS.ID]: id,
[DDP_EVENTS.FIELDS]: fields,
}),
);
}
removed(client: Client, collection: string, id: string): void {
return client.send(
this.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.REMOVED,
[DDP_EVENTS.COLLECTION]: collection,
[DDP_EVENTS.ID]: id,
}),
);
}
}

View File

@@ -0,0 +1,102 @@
import { api } from '@rocket.chat/core-services';
import type { StreamNames } from '@rocket.chat/ddp-client';
import WebSocket from 'ws';
import { server } from './configureServer';
import { DDP_EVENTS } from './constants';
import { isEmpty } from './lib/utils';
import { Streamer, StreamerCentral } from '../../../../apps/meteor/server/modules/streamer/streamer.module';
import type { DDPSubscription, Connection, TransformMessage } from '../../../../apps/meteor/server/modules/streamer/types';
StreamerCentral.on('broadcast', (name, eventName, args) => {
void api.broadcast('stream', [name, eventName, args]);
});
export class Stream<N extends StreamNames> extends Streamer<N> {
registerPublication(name: string, fn: (eventName: string, options: boolean | { useCollection?: boolean; args?: any }) => void): void {
server.publish(name, fn);
}
registerMethod(methods: Record<string, (eventName: string, ...args: any[]) => any>): void {
server.methods(methods);
}
changedPayload(collection: string, id: string, fields: Record<string, any>): string | false {
return (
!isEmpty(fields) &&
server.serialize({
[DDP_EVENTS.MSG]: DDP_EVENTS.CHANGED,
[DDP_EVENTS.COLLECTION]: collection,
[DDP_EVENTS.ID]: id,
[DDP_EVENTS.FIELDS]: fields,
})
);
}
override async sendToManySubscriptions(
subscriptions: Set<DDPSubscription>,
origin: Connection | undefined,
eventName: string,
args: any[],
getMsg: string | TransformMessage,
): Promise<void> {
if (typeof getMsg === 'function') {
return super.sendToManySubscriptions(subscriptions, origin, eventName, args, getMsg);
}
const options = {
fin: true, // sending a single fragment message
rsv1: false, // don"t set rsv1 bit (no compression)
opcode: 1, // opcode for a text frame
mask: false, // set false for client-side
readOnly: false, // the data can be modified as needed
};
const data = {
meteor: [Buffer.concat(WebSocket.Sender.frame(Buffer.from(`a${JSON.stringify([getMsg])}`), options))],
normal: [Buffer.concat(WebSocket.Sender.frame(Buffer.from(getMsg), options))],
};
for (const { subscription } of subscriptions) {
// if the connection state is not open anymore, it somehow got to a weird state,
// we'll emit close so it can clean up the weird state, and so we stop emitting to it
if (subscription.client.ws.readyState !== WebSocket.OPEN) {
subscription.stop();
subscription.client.ws.close();
continue;
}
if (this.retransmitToSelf === false && origin && origin === subscription.connection) {
continue;
}
if (!(await this.isEmitAllowed(subscription, eventName, ...args))) {
continue;
}
try {
await new Promise<void>((resolve, reject) => {
const frame = data[subscription.client.meteorClient ? 'meteor' : 'normal'];
subscription.client.ws._sender.sendFrame(frame, (err: unknown) => {
if (err) {
return reject(err);
}
resolve();
});
});
} catch (error: any) {
if (error.code === 'ERR_STREAM_DESTROYED') {
console.warn('Trying to send data to destroyed stream, closing connection.');
// if we still tried to send data to a destroyed stream, we'll try again to close the connection
if (subscription.client.ws.readyState !== WebSocket.OPEN) {
subscription.stop();
subscription.client.ws.close();
}
}
console.error('Error trying to send data to stream.', error);
}
}
}
}

View File

@@ -0,0 +1,140 @@
import { EventEmitter } from 'events';
import { Account, Presence, MeteorService, MeteorError } from '@rocket.chat/core-services';
import { UserStatus } from '@rocket.chat/core-typings';
import { Server } from './Server';
import { DDP_EVENTS, WS_ERRORS } from './constants';
import { Autoupdate } from './lib/Autoupdate';
export const server = new Server();
export const events = new EventEmitter();
const loginServiceConfigurationCollection = 'meteor_accounts_loginServiceConfiguration';
const loginServiceConfigurationPublication = 'meteor.loginServiceConfiguration';
const loginServices = new Map<string, any>();
MeteorService.getLoginServiceConfiguration()
.then((records = []) => records.forEach((record) => loginServices.set(record._id, record)))
.catch((err) => console.error('DDPStreamer not able to retrieve login services configuration', err));
server.publish(loginServiceConfigurationPublication, async function () {
loginServices.forEach((record) => this.added(loginServiceConfigurationCollection, record._id, record));
const fn = (action: string, record: any): void => {
switch (action) {
case 'added':
case 'changed':
loginServices.set(record._id, record);
this[action](loginServiceConfigurationCollection, record._id, record);
break;
case 'removed':
loginServices.delete(record._id);
this[action](loginServiceConfigurationCollection, record._id);
}
};
events.on(loginServiceConfigurationPublication, fn);
this.onStop(() => {
events.removeListener(loginServiceConfigurationPublication, fn);
});
this.ready();
});
const autoUpdateCollection = 'meteor_autoupdate_clientVersions';
server.publish(autoUpdateCollection, function () {
Autoupdate.getVersions().forEach((version, arch) => {
this.added(autoUpdateCollection, arch, version);
});
const fn = (record: any): void => {
const { _id, ...version } = record;
this.changed(autoUpdateCollection, _id, version);
};
Autoupdate.on('update', fn);
this.onStop(() => {
Autoupdate.removeListener('update', fn);
});
this.ready();
});
server.methods({
async 'login'({ resume }: { resume: string }) {
try {
const result = await Account.login({ resume });
if (!result) {
throw new MeteorError(403, "You've been logged out by the server. Please log in again");
}
this.userId = result.uid;
this.userToken = result.hashedToken;
this.connection.loginToken = result.hashedToken;
this.emit(DDP_EVENTS.LOGGED);
server.emit(DDP_EVENTS.LOGGED, this);
return {
id: result.uid,
token: result.token,
tokenExpires: result.tokenExpires,
type: result.type,
};
} catch (error) {
throw error;
}
},
async 'logout'() {
if (this.userToken && this.userId) {
await Account.logout({ userId: this.userId, token: this.userToken });
}
this.emit(DDP_EVENTS.LOGGEDOUT);
server.emit(DDP_EVENTS.LOGGEDOUT, this);
this.userToken = undefined;
this.userId = undefined;
// Close connection after return success to the method call.
// This ensures all the subscriptions will be closed, meteor makes it manually
// here https://github.com/meteor/meteor/blob/2377ebe879d9b965d699f599392d4e8047eb7d78/packages/ddp-server/livedata_server.js#L781
// re doing the default subscriptions.
setTimeout(() => {
this.ws.close(WS_ERRORS.CLOSE_PROTOCOL_ERROR);
}, 1);
},
'UserPresence:setDefaultStatus'(status) {
const { userId } = this;
if (!userId) {
return;
}
return Presence.setStatus(userId, status);
},
'UserPresence:online'() {
const { userId, session } = this;
if (!userId) {
return;
}
return Presence.setConnectionStatus(userId, UserStatus.ONLINE, session);
},
'UserPresence:away'() {
const { userId, session } = this;
if (!userId) {
return;
}
return Presence.setConnectionStatus(userId, UserStatus.AWAY, session);
},
'setUserStatus'(status, statusText) {
const { userId } = this;
if (!userId) {
return;
}
return Presence.setStatus(userId, status, statusText);
},
});

View File

@@ -0,0 +1,48 @@
export const STREAMER_EVENTS = {
STREAM: 'stream',
USER_CHANGED: 'user-changed',
};
export const DDP_EVENTS = {
ID: 'id',
FIELDS: 'fields',
COLLECTION: 'collection',
CLEARED: 'cleared',
METHODS: 'methods',
MSG: 'msg',
READY: 'ready',
ADDED: 'added',
CHANGED: 'changed',
REMOVED: 'removed',
RESUME: 'resume',
RESULT: 'result',
METHOD: 'method',
UPDATED: 'updated',
PING: 'ping',
PONG: 'pong',
SUBSCRIBE: 'sub',
CONNECT: 'connect',
CONNECTED: 'connected',
SUBSCRIPTIONS: 'subs',
NO_SUBSCRIBE: 'nosub',
UNSUBSCRIBE: 'unsub',
DISCONNECTED: 'disconnected',
LOGGED: 'logged',
LOGGEDOUT: 'loggedout',
};
export const WS_ERRORS = {
CLOSE_PROTOCOL_ERROR: 1002,
UNSUPPORTED_DATA: 1007,
TIMEOUT: 4000,
};
export const WS_ERRORS_MESSAGES = {
CLOSE_PROTOCOL_ERROR: 'CLOSE_PROTOCOL_ERROR',
UNSUPPORTED_DATA: 'UNSUPPORTED_DATA',
TIMEOUT: 'TIMEOUT',
};
export const TIMEOUT = 1000 * 30; // 30 seconds

View File

@@ -0,0 +1,9 @@
import crypto from 'crypto';
crypto.setFips(true);
if (!crypto.getFips()) {
throw new Error('FIPS mode was not enabled after crypto.setFips(true)');
}
console.log('FIPS COMPLIANCE CHECK: YES');

View File

@@ -0,0 +1,20 @@
import { EventEmitter } from 'events';
import type { AutoUpdateRecord } from '@rocket.chat/core-services';
class AutoupdateSingleton extends EventEmitter {
private versions = new Map<string, Omit<AutoUpdateRecord, '_id'>>();
public updateVersion(record: AutoUpdateRecord): void {
const { _id, ...version } = record;
this.versions.set(_id, version);
this.emit('update', record);
}
public getVersions(): Map<string, Omit<AutoUpdateRecord, '_id'>> {
return this.versions;
}
}
export const Autoupdate = new AutoupdateSingleton();

View File

@@ -0,0 +1,9 @@
export const isEmpty = function (obj: object | any[] | string): boolean {
if (obj == null) {
return true;
}
if (Array.isArray(obj) || typeof obj === 'string') {
return obj.length === 0;
}
return !Object.values(obj).some((value) => value !== null);
};

View File

@@ -0,0 +1,41 @@
import type { IncomingMessage, RequestOptions, ServerResponse } from 'http';
import http from 'http';
import url from 'url';
import type polka from 'polka';
const isProdEnv = process.env.NODE_ENV === 'production';
const skipProxyPaths = [/^\/sockjs\/info\?cb=/, /^\/health/];
export function proxy(): (req: IncomingMessage, res: ServerResponse, next: polka.Next) => void {
if (isProdEnv) {
return (_req, _res, next) => next();
}
return (req, res, next) => {
if (skipProxyPaths.some((regex) => regex.test(req.url || ''))) {
return next();
}
req.pause();
const options: RequestOptions = url.parse(req.url || '');
options.headers = req.headers;
options.method = req.method;
options.agent = false;
options.hostname = 'localhost';
options.port = 3000;
const connector = http.request(options, function (serverResponse) {
serverResponse.pause();
if (serverResponse.statusCode) {
res.writeHead(serverResponse.statusCode, serverResponse.headers);
}
serverResponse.pipe(res);
serverResponse.resume();
});
req.pipe(connector);
req.resume();
};
}

View File

@@ -0,0 +1,70 @@
import os from 'os';
import { api, getConnection, getTrashCollection } from '@rocket.chat/core-services';
import { InstanceStatus } from '@rocket.chat/instance-status';
import { registerServiceModels } from '@rocket.chat/models';
import { startBroker } from '@rocket.chat/network-broker';
import { startTracing } from '@rocket.chat/tracing';
void (async () => {
const { db, client } = await getConnection();
startTracing({ service: 'ddp-streamer', db: client });
registerServiceModels(db, await getTrashCollection());
api.setBroker(
startBroker({
nodeID: `${os.hostname().toLowerCase()}-${InstanceStatus.id()}`,
}),
);
// need to import service after models are registered
const { NotificationsModule } = await import('../../../../apps/meteor/server/modules/notifications/notifications.module');
const { DDPStreamer } = await import('./DDPStreamer');
const { Stream } = await import('./Streamer');
const notifications = new NotificationsModule(Stream);
notifications.configure();
api.registerService(new DDPStreamer(notifications));
await api.start();
})();
/**
* If some promise is rejected and doesn't have a catch (unhandledRejection) it may cause the process to exit.
*
* Since unhandled rejections are deprecated in NodeJS:
* (node:83382) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections
* that are not handled will terminate the Node.js process with a non-zero exit code.
* we will start respecting this and exit the process to prevent these kind of problems.
*/
process.on('unhandledRejection', (error) => {
console.error('=== UnHandledPromiseRejection ===');
console.error(error);
console.error('---------------------------------');
console.error(
'Setting EXIT_UNHANDLEDPROMISEREJECTION will cause the process to exit allowing your service to automatically restart the process',
);
console.error('Future node.js versions will automatically exit the process');
console.error('=================================');
if (process.env.TEST_MODE || process.env.NODE_ENV === 'development' || process.env.EXIT_UNHANDLEDPROMISEREJECTION) {
process.exit(1);
}
});
process.on('uncaughtException', async (error) => {
console.error('=== UnCaughtException ===');
console.error(error);
console.error('-------------------------');
console.error('Errors like this can cause oplog processing errors.');
console.error('===========================');
if (process.env.TEST_MODE || process.env.NODE_ENV === 'development' || process.env.EXIT_UNHANDLEDPROMISEREJECTION) {
process.exit(1);
}
});

View File

@@ -0,0 +1,9 @@
export interface IPacket {
name: string;
id: string;
method: string;
msg: string;
version: string;
support: string[];
params: any[];
}

View File

@@ -0,0 +1,20 @@
{
"extends": "@rocket.chat/tsconfig/server.json",
"compilerOptions": {
"strictPropertyInitialization": false, // TODO: Remove this line
"outDir": "./dist",
"types": ["jest", "node"],
/* This service typechecks sources from apps/meteor, which has
`installConfig.hoistingLimits: "workspaces"` and therefore its own nested
copy of mongodb. Without pinning, those sources resolve `mongodb` to
apps/meteor/node_modules while @rocket.chat/models resolves it to the root
copy, and the two identical-but-distinct instances fail to unify. */
"baseUrl": ".",
"paths": {
"mongodb": ["../../../node_modules/mongodb"]
}
},
"include": ["./src/**/*", "./definition"],
"exclude": ["./dist", "./ecosystem.config.js"]
}