snapshot of https://github.com/RocketChat/Rocket.Chat.git (develop, 2026-09-16; blobs > 131072B omitted)

ee/packages/network-broker/src/EnterpriseCheck.ts

@@ -0,0 +1,66 @@
+import type { ServiceSchema } from 'moleculer';
+
+const { LICENSE_CHECK_INTERVAL = '20', MAX_FAILS = '2' } = process.env;
+
+let checkFails = 0;
+const maxFails = parseInt(MAX_FAILS) || 2;
+
+export const EnterpriseCheck: ServiceSchema = {
+	name: 'EnterpriseCheck',
+	methods: {
+		/**
+		 * The main idea is if there is no scalability module enabled,
+		 * then we should not allow more than one service per environment.
+		 * So we list the services and nodes, and if there is more than
+		 * one, we say it should be shutdown.
+		 */
+		async shouldShutdown(): Promise<boolean> {
+			const services: {
+				name: string;
+				nodes: string[];
+			}[] = await this.broker.call('$node.services', { skipInternal: true });
+
+			const currentService = services.find((service) => {
+				return service.name === this.name;
+			});
+
+			// if current service is not on the list maybe it is already shut down?
+			if (!currentService) {
+				return true;
+			}
+
+			const { nodes } = currentService;
+
+			const firstNode = nodes.sort().shift();
+
+			// if the first node is the current node and there are others nodes running the same service or
+			// if this is the only one node online, then we should shutdown
+			return firstNode === this.broker.nodeID && (nodes.length > 0 || services.length === 1);
+		},
+	},
+	async started(): Promise<void> {
+		setInterval(
+			async () => {
+				try {
+					const hasLicense = await this.broker.call('license.hasValidLicense', ['scalability']);
+					if (hasLicense) {
+						checkFails = 0;
+						return;
+					}
+				} catch (e: unknown) {
+					// check failed, so continue
+				}
+
+				if (++checkFails < maxFails) {
+					return;
+				}
+
+				const shouldShutdown = await this.shouldShutdown();
+				if (shouldShutdown) {
+					this.broker.fatal('Enterprise license not found. Shutting down...');
+				}
+			},
+			(parseInt(LICENSE_CHECK_INTERVAL) || 20) * 1000,
+		);
+	},
+};

ee/packages/network-broker/src/NetworkBroker.test.ts

@@ -0,0 +1,39 @@
+import { ServiceClass } from '@rocket.chat/core-services';
+import { expect } from 'chai';
+import sinon from 'sinon';
+
+import { NetworkBroker } from './NetworkBroker';
+import { BrokerMocked } from '../../../../apps/meteor/tests/mocks/server/BrokerMocked';
+
+class DelayedStopBroker extends BrokerMocked {
+	async destroyService(name: string) {
+		const instance = this.services.get(name);
+
+		await new Promise((resolve) => setTimeout(resolve, 1000));
+
+		await instance.stopped();
+
+		await super.destroyService(name);
+	}
+}
+
+const broker = new NetworkBroker(new DelayedStopBroker() as any);
+
+describe('NetworkBroker', () => {
+	it('should wait services to be fully destroyed', async () => {
+		const stoppedStub = sinon.stub();
+
+		const instance = new (class extends ServiceClass {
+			name = 'test';
+
+			async stopped() {
+				stoppedStub();
+			}
+		})();
+
+		broker.createService(instance);
+		await broker.destroyService(instance);
+
+		expect(stoppedStub.called).to.be.true;
+	});
+});

ee/packages/network-broker/src/NetworkBroker.ts

@@ -0,0 +1,206 @@
+import Stream from 'node:stream';
+
+import { asyncLocalStorage } from '@rocket.chat/core-services';
+import type { CallingOptions, IBroker, IBrokerNode, IServiceMetrics, IServiceClass, EventSignatures } from '@rocket.chat/core-services';
+import { injectCurrentContext, tracerSpan } from '@rocket.chat/tracing';
+import type { ServiceBroker, Context, ServiceSchema } from 'moleculer';
+
+import { EnterpriseCheck } from './EnterpriseCheck';
+
+const events: { [k: string]: string } = {
+	onNodeConnected: '$node.connected',
+	onNodeUpdated: '$node.updated',
+	onNodeDisconnected: '$node.disconnected',
+};
+
+const lifecycle: { [k: string]: string } = {
+	created: 'created',
+	started: 'started',
+	stopped: 'stopped',
+};
+
+export class NetworkBroker implements IBroker {
+	private broker: ServiceBroker;
+
+	private started: Promise<boolean> = Promise.resolve(false);
+
+	private defaultDependencies = ['settings', 'license'];
+
+	metrics: IServiceMetrics;
+
+	constructor(broker: ServiceBroker) {
+		this.broker = broker;
+
+		this.metrics = broker.metrics;
+	}
+
+	async call(method: string, data: any, options?: CallingOptions): Promise<any> {
+		if (!(await this.started)) {
+			return;
+		}
+
+		const context = asyncLocalStorage.getStore();
+
+		const stream = data?.[0]?.streamParam;
+		const isStreamingCall = !!stream;
+
+		if (context?.ctx?.call) {
+			return context.ctx.call(method, isStreamingCall ? stream : data, {
+				...options,
+				...(isStreamingCall && { meta: { ...((options as any)?.meta || {}), details: data[0].details } }),
+			});
+		}
+
+		const services: { name: string }[] = await this.broker.call('$node.services', {
+			onlyAvailable: true,
+		});
+
+		if (!services.find((service) => service.name === method.split('.')[0])) {
+			return new Error('method-not-available');
+		}
+
+		return this.broker.call(method, isStreamingCall ? stream : data, {
+			...options,
+			meta: {
+				optl: injectCurrentContext(),
+				...(isStreamingCall && { details: data?.[0]?.details }),
+			},
+		});
+	}
+
+	async destroyService(instance: IServiceClass): Promise<void> {
+		const name = instance.getName();
+		if (!name) {
+			return;
+		}
+		await this.broker.destroyService(name);
+		instance.removeAllListeners();
+	}
+
+	createService(instance: IServiceClass, serviceDependencies: string[] = []): void {
+		const methods = (
+			instance.constructor?.name === 'Object'
+				? Object.getOwnPropertyNames(instance)
+				: Object.getOwnPropertyNames(Object.getPrototypeOf(instance))
+		).filter((name) => name !== 'constructor');
+
+		const serviceInstance = instance as any;
+
+		const name = instance.getName();
+		if (!name) {
+			return;
+		}
+
+		const instanceEvents = instance.getEvents();
+		if (!instanceEvents && !methods.length) {
+			return;
+		}
+
+		const dependencies = [...serviceDependencies, ...(name === 'settings' ? [] : this.defaultDependencies)].filter(
+			(dependency) => dependency !== name,
+		);
+
+		const service: ServiceSchema = {
+			name,
+			actions: {},
+			mixins: !instance.isInternal() ? [EnterpriseCheck] : [],
+			...(dependencies.length ? { dependencies } : {}),
+			events: instanceEvents.reduce<Record<string, (ctx: Context) => void>>((map, { eventName }) => {
+				map[eventName] = /^\$/.test(eventName)
+					? (ctx: Context): void => {
+							// internal events params are not an array
+							instance.emit(eventName, ctx.params as Parameters<EventSignatures[typeof eventName]>);
+						}
+					: (ctx: Context): void => {
+							instance.emit(eventName, ...(ctx.params as Parameters<EventSignatures[typeof eventName]>));
+						};
+				return map;
+			}, {}),
+		};
+
+		if (!service.events || !service.actions) {
+			return;
+		}
+
+		for (const method of methods) {
+			if (method.match(/^on[A-Z]/)) {
+				service.events[events[method]] = serviceInstance[method].bind(serviceInstance);
+				continue;
+			}
+
+			if (lifecycle[method]) {
+				service[method] = (): void => {
+					asyncLocalStorage.run(
+						{
+							id: '',
+							nodeID: this.broker.nodeID,
+							requestID: null,
+							broker: this,
+						},
+						serviceInstance[method].bind(serviceInstance),
+					);
+				};
+				continue;
+			}
+
+			service.actions[method] = async (ctx: Context<[], { optl?: unknown; details?: unknown }>): Promise<any> => {
+				const isStreamingCall = Stream.isReadable(ctx.params as any);
+				const params = isStreamingCall ? [{ streamParam: ctx.params, details: ctx.meta?.details }] : ctx.params;
+
+				return tracerSpan(
+					`action ${name}:${method}${isStreamingCall ? ' (streaming)' : ''}`,
+					{},
+					() => {
+						return asyncLocalStorage.run(
+							{
+								id: ctx.id,
+								nodeID: ctx.nodeID,
+								requestID: ctx.requestID,
+								broker: this,
+								ctx,
+							},
+							() => serviceInstance[method](...params),
+						);
+					},
+					ctx.meta?.optl,
+				);
+			};
+		}
+
+		this.broker.createService(service);
+	}
+
+	async broadcast<T extends keyof EventSignatures>(event: T, ...args: Parameters<EventSignatures[T]>): Promise<void> {
+		return this.broker.broadcast(event, args);
+	}
+
+	async broadcastLocal<T extends keyof EventSignatures>(event: T, ...args: Parameters<EventSignatures[T]>): Promise<void> {
+		void this.broker.broadcastLocal(event, args);
+	}
+
+	async broadcastToServices<T extends keyof EventSignatures>(
+		services: string[],
+		event: T,
+		...args: Parameters<EventSignatures[T]>
+	): Promise<void> {
+		void this.broker.broadcast(event, args, services);
+	}
+
+	async nodeList(): Promise<IBrokerNode[]> {
+		return this.broker.call('$node.list');
+	}
+
+	async start(): Promise<void> {
+		try {
+			// We have to set this before calling `.start` as some services might call another service (settings, likely) as part of
+			// their `started` lifecycle method. `.call` bails out early (and silently) if `started` is false, returning undefined
+			// which can affect the service startup.
+			this.started = Promise.resolve(true);
+
+			await this.broker.start();
+		} catch (e) {
+			this.started = Promise.resolve(false);
+			throw e;
+		}
+	}
+}

ee/packages/network-broker/src/index.ts

@@ -0,0 +1,155 @@
+import { isMeteorError, MeteorError } from '@rocket.chat/core-services';
+import EJSON from 'ejson';
+import type Moleculer from 'moleculer';
+import { Errors, Serializers, ServiceBroker } from 'moleculer';
+import { pino } from 'pino';
+
+import { NetworkBroker } from './NetworkBroker';
+
+const {
+	MS_NAMESPACE = '',
+	TRANSPORTER = '',
+	CACHE = 'Memory',
+	// SERIALIZER = 'MsgPack',
+	SERIALIZER = 'EJSON',
+	MOLECULER_LOG_LEVEL = 'warn',
+	BALANCE_STRATEGY = 'RoundRobin',
+	BALANCE_PREFER_LOCAL = 'true',
+	RETRY_FACTOR = '2',
+	RETRY_MAX_DELAY = '1000',
+	RETRY_DELAY = '100',
+	RETRY_RETRIES = '5',
+	RETRY_ENABLED = 'false',
+	REQUEST_TIMEOUT = '60',
+	HEARTBEAT_INTERVAL = '10',
+	HEARTBEAT_TIMEOUT = '30',
+	BULKHEAD_ENABLED = 'false',
+	BULKHEAD_CONCURRENCY = '10',
+	BULKHEAD_MAX_QUEUE_SIZE = '10000',
+	MS_METRICS = 'false',
+	MS_METRICS_PORT = '9458',
+	SKIP_PROCESS_EVENT_REGISTRATION = 'false',
+} = process.env;
+
+const { Base } = Serializers;
+
+class CustomRegenerator extends Errors.Regenerator {
+	override restoreCustomError(plainError: any): Error | undefined {
+		const { message, reason, details, errorType, isClientSafe } = plainError;
+
+		if (errorType === 'Meteor.Error') {
+			const error = new MeteorError(message, reason, details);
+			if (typeof isClientSafe !== 'undefined') {
+				error.isClientSafe = isClientSafe;
+			}
+			return error;
+		}
+
+		return undefined;
+	}
+
+	override extractPlainError(err: Error | MeteorError): Errors.PlainMoleculerError {
+		return {
+			...super.extractPlainError(err),
+			...(isMeteorError(err) && {
+				isClientSafe: err.isClientSafe,
+				errorType: err.errorType,
+				reason: err.reason,
+				details: err.details,
+			}),
+		};
+	}
+}
+
+class EJSONSerializer extends Base {
+	override serialize(obj: any): Buffer {
+		return Buffer.from(EJSON.stringify(obj));
+	}
+
+	override deserialize(buf: Buffer): any {
+		return EJSON.parse(buf.toString());
+	}
+}
+
+export function startBroker(options: Moleculer.BrokerOptions = {}): NetworkBroker {
+	const network = new ServiceBroker({
+		namespace: MS_NAMESPACE,
+		skipProcessEventRegistration: SKIP_PROCESS_EVENT_REGISTRATION === 'true',
+		transporter: TRANSPORTER,
+		metrics: {
+			enabled: MS_METRICS === 'true',
+			reporter: [
+				{
+					type: 'Prometheus',
+					options: {
+						port: MS_METRICS_PORT,
+					},
+				},
+			],
+		},
+		cacher: CACHE,
+		serializer: SERIALIZER === 'EJSON' ? new EJSONSerializer() : SERIALIZER,
+		logger: {
+			type: 'Pino',
+			options: {
+				level: MOLECULER_LOG_LEVEL,
+				pino: {
+					options: {
+						timestamp: pino.stdTimeFunctions.isoTime,
+						...(process.env.NODE_ENV !== 'production'
+							? {
+									transport: {
+										target: 'pino-pretty',
+										options: {
+											colorize: true,
+										},
+									},
+								}
+							: {}),
+					},
+				},
+			},
+		},
+		registry: {
+			strategy: BALANCE_STRATEGY,
+			preferLocal: BALANCE_PREFER_LOCAL !== 'false',
+		},
+
+		requestTimeout: parseInt(REQUEST_TIMEOUT) * 1000,
+		retryPolicy: {
+			enabled: RETRY_ENABLED === 'true',
+			retries: parseInt(RETRY_RETRIES),
+			delay: parseInt(RETRY_DELAY),
+			maxDelay: parseInt(RETRY_MAX_DELAY),
+			factor: parseInt(RETRY_FACTOR),
+			check: (err: any): boolean => err && !!err.retryable,
+		},
+
+		maxCallLevel: 100,
+		heartbeatInterval: parseInt(HEARTBEAT_INTERVAL),
+		heartbeatTimeout: parseInt(HEARTBEAT_TIMEOUT),
+
+		// circuitBreaker: {
+		// 	enabled: false,
+		// 	threshold: 0.5,
+		// 	windowTime: 60,
+		// 	minRequestCount: 20,
+		// 	halfOpenTime: 10 * 1000,
+		// 	check: (err: any): boolean => err && err.code >= 500,
+		// },
+
+		bulkhead: {
+			enabled: BULKHEAD_ENABLED === 'true',
+			concurrency: parseInt(BULKHEAD_CONCURRENCY),
+			maxQueueSize: parseInt(BULKHEAD_MAX_QUEUE_SIZE),
+		},
+
+		errorRegenerator: new CustomRegenerator(),
+		started(): void {
+			console.log('NetworkBroker started successfully.');
+		},
+		...options,
+	});
+
+	return new NetworkBroker(network);
+}