snapshot of https://github.com/toeverything/AFFiNE.git (canary, 2026-09-16; blobs > 131072B omitted)

packages/common/nbstore/src/storage/awareness.ts

@@ -0,0 +1,36 @@
+import type { Connection } from '../connection';
+import { type Storage } from './storage';
+
+export type AwarenessRecord = {
+  docId: string;
+  bin: Uint8Array;
+};
+
+export interface AwarenessStorage extends Storage {
+  readonly storageType: 'awareness';
+
+  /**
+   * Update the awareness record.
+   *
+   * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
+   */
+  update(record: AwarenessRecord, origin?: string): Promise<void>;
+  subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void;
+}
+
+export abstract class AwarenessStorageBase implements AwarenessStorage {
+  readonly storageType = 'awareness';
+  abstract readonly connection: Connection;
+
+  abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
+
+  abstract subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void;
+}

packages/common/nbstore/src/storage/blob-sync.ts

@@ -0,0 +1,30 @@
+import type { Connection } from '../connection';
+import type { Storage } from './storage';
+
+export interface BlobSyncStorage extends Storage {
+  readonly storageType: 'blobSync';
+
+  setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void>;
+
+  getBlobUploadedAt(peer: string, blobId: string): Promise<Date | null>;
+}
+
+export abstract class BlobSyncStorageBase implements BlobSyncStorage {
+  readonly storageType = 'blobSync';
+  abstract readonly connection: Connection;
+
+  abstract setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void>;
+
+  abstract getBlobUploadedAt(
+    peer: string,
+    blobId: string
+  ): Promise<Date | null>;
+}

packages/common/nbstore/src/storage/blob.ts

@@ -0,0 +1,78 @@
+import type { Connection } from '../connection';
+import { type Storage } from './storage';
+
+export interface BlobRecord {
+  key: string;
+  data: Uint8Array;
+  mime: string;
+  createdAt?: Date;
+}
+
+export interface ListedBlobRecord {
+  key: string;
+  mime: string;
+  size: number;
+  createdAt?: Date;
+}
+
+export type BlobSource =
+  | { type: 'currentDoc'; workspaceId: string; docId: string }
+  | {
+      type: 'history';
+      workspaceId: string;
+      docId: string;
+      timestampMs: number;
+    };
+
+export interface SourceBlobRecord extends ListedBlobRecord {
+  source: BlobSource;
+}
+
+export interface SourceScopedBlobStorage {
+  registerSource(source: BlobSource, signal?: AbortSignal): Promise<void>;
+  unregisterSource(source: BlobSource): Promise<void>;
+  readableSources(signal?: AbortSignal): AsyncIterable<SourceBlobRecord>;
+}
+
+export function isSourceScopedBlobStorage(
+  storage: BlobStorage
+): storage is BlobStorage & SourceScopedBlobStorage {
+  return 'registerSource' in storage && 'readableSources' in storage;
+}
+
+export interface BlobStorage extends Storage {
+  readonly storageType: 'blob';
+  readonly isReadonly: boolean;
+  get(
+    key: string,
+    signal?: AbortSignal,
+    source?: BlobSource
+  ): Promise<BlobRecord | null>;
+  set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
+  delete(
+    key: string,
+    permanently: boolean,
+    signal?: AbortSignal
+  ): Promise<void>;
+  release(signal?: AbortSignal): Promise<void>;
+  list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
+}
+
+export abstract class BlobStorageBase implements BlobStorage {
+  readonly storageType = 'blob';
+  abstract readonly connection: Connection;
+  abstract readonly isReadonly: boolean;
+  abstract get(
+    key: string,
+    signal?: AbortSignal,
+    source?: BlobSource
+  ): Promise<BlobRecord | null>;
+  abstract set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
+  abstract delete(
+    key: string,
+    permanently: boolean,
+    signal?: AbortSignal
+  ): Promise<void>;
+  abstract release(signal?: AbortSignal): Promise<void>;
+  abstract list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
+}

packages/common/nbstore/src/storage/doc-sync.ts

@@ -0,0 +1,49 @@
+import type { Connection } from '../connection';
+import type { DocClock, DocClocks } from './doc';
+import { type Storage } from './storage';
+
+export interface DocSyncStorage extends Storage {
+  readonly storageType: 'docSync';
+
+  getPeerRemoteClock(peer: string, docId: string): Promise<DocClock | null>;
+  getPeerRemoteClocks(peer: string): Promise<DocClocks>;
+  setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  getPeerPulledRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
+  setPeerPulledRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  getPeerPushedClock(peer: string, docId: string): Promise<DocClock | null>;
+  getPeerPushedClocks(peer: string): Promise<DocClocks>;
+  setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
+  clearClocks(): Promise<void>;
+}
+
+export abstract class DocSyncStorageBase implements DocSyncStorage {
+  readonly storageType = 'docSync';
+  abstract readonly connection: Connection;
+
+  abstract getPeerRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  abstract getPeerRemoteClocks(peer: string): Promise<DocClocks>;
+  abstract setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  abstract getPeerPulledRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  abstract getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
+  abstract setPeerPulledRemoteClock(
+    peer: string,
+    clock: DocClock
+  ): Promise<void>;
+  abstract getPeerPushedClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  abstract getPeerPushedClocks(peer: string): Promise<DocClocks>;
+  abstract setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
+  abstract clearClocks(): Promise<void>;
+}

packages/common/nbstore/src/storage/doc.ts

@@ -0,0 +1,319 @@
+import EventEmitter2 from 'eventemitter2';
+import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs';
+
+import type { Connection } from '../connection';
+import { isEmptyUpdate } from '../utils/is-empty-update';
+import type { Locker } from './lock';
+import { SingletonLocker } from './lock';
+import { type Storage } from './storage';
+
+export interface BlockInfo {
+  blockId: string;
+  flavour: string;
+  content?: string[];
+  blob?: string[];
+  refDocId?: string[];
+  refInfo?: string[];
+  parentFlavour?: string;
+  parentBlockId?: string;
+  additional?: string;
+}
+
+export interface CrawlResult {
+  blocks: BlockInfo[];
+  title: string;
+  summary: string;
+}
+
+export interface DocClock {
+  docId: string;
+  timestamp: Date;
+}
+
+export type DocClocks = Record<string, Date>;
+export interface DocRecord extends DocClock {
+  bin: Uint8Array;
+  editor?: string;
+}
+
+export interface DocDiff extends DocClock {
+  missing: Uint8Array;
+  state: Uint8Array;
+}
+
+export interface DocUpdate {
+  docId: string;
+  bin: Uint8Array;
+  editor?: string;
+}
+
+export type DocLifecycle = 'trash' | 'restore' | 'delete';
+
+export interface DocLifecycleResult {
+  rootUpdate: Uint8Array;
+  timestamp: Date;
+}
+
+export interface Editor {
+  name: string;
+  avatarUrl: string | null;
+}
+
+export interface DocStorageOptions {
+  mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
+  id: string;
+
+  /**
+   * open as readonly mode.
+   */
+  readonlyMode?: boolean;
+}
+
+export interface DocStorage extends Storage {
+  readonly storageType: 'doc';
+  readonly isReadonly: boolean;
+  readonly spaceId: string;
+  /**
+   * Get a doc record with latest binary.
+   */
+  getDoc(docId: string): Promise<DocRecord | null>;
+  /**
+   * Get a yjs binary diff with the given state vector.
+   */
+  getDocDiff(docId: string, state?: Uint8Array): Promise<DocDiff | null>;
+  /**
+   * Push updates into storage
+   *
+   * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
+   */
+  pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
+
+  /**
+   * Get the timestamp of the latest update of a doc.
+   */
+  getDocTimestamp(docId: string): Promise<DocClock | null>;
+
+  /**
+   * Get all docs timestamps info. especially for useful in sync process.
+   */
+  getDocTimestamps(after?: Date): Promise<DocClocks>;
+
+  /**
+   * Delete a specific doc data with all snapshots and updates
+   */
+  deleteDoc(docId: string): Promise<void>;
+
+  applyDocLifecycle?(
+    docId: string,
+    lifecycle: DocLifecycle
+  ): Promise<DocLifecycleResult>;
+
+  /**
+   * Subscribe on doc updates emitted from storage itself.
+   *
+   * NOTE:
+   *
+   *   There is not always update emitted from storage itself.
+   *
+   *   For example, in Sqlite storage, the update will only come from user's updating on docs,
+   *   in other words, the update will never somehow auto generated in storage internally.
+   *
+   *   But for Cloud storage, there will be updates broadcasted from other clients,
+   *   so the storage will emit updates to notify the client to integrate them.
+   */
+  subscribeDocUpdate(
+    callback: (update: DocRecord, origin?: string) => void
+  ): () => void;
+
+  crawlDocData?(docId: string): Promise<CrawlResult | null>;
+}
+
+export abstract class DocStorageBase<Opts = {}> implements DocStorage {
+  get isReadonly(): boolean {
+    return this.options.readonlyMode ?? false;
+  }
+  private readonly event = new EventEmitter2();
+  readonly storageType = 'doc';
+  abstract readonly connection: Connection;
+  protected readonly locker: Locker = new SingletonLocker();
+  readonly spaceId = this.options.id;
+
+  constructor(protected readonly options: Opts & DocStorageOptions) {}
+
+  async getDoc(docId: string) {
+    await using _lock = this.isReadonly
+      ? undefined
+      : await this.lockDocForUpdate(docId);
+
+    const snapshot = await this.getDocSnapshot(docId);
+    const updates = await this.getDocUpdates(docId);
+
+    if (updates.length) {
+      const { timestamp, bin, editor } = await this.squash(
+        snapshot ? [snapshot, ...updates] : updates
+      );
+
+      const newSnapshot = {
+        spaceId: this.spaceId,
+        docId,
+        bin,
+        timestamp,
+        editor,
+      };
+
+      // if is readonly, we will not set the new snapshot
+      if (!this.isReadonly) {
+        await this.setDocSnapshot(newSnapshot, snapshot);
+
+        // always mark updates as merged unless throws
+        await this.markUpdatesMerged(docId, updates);
+      }
+
+      return newSnapshot;
+    }
+
+    return snapshot;
+  }
+
+  async getDocDiff(docId: string, state?: Uint8Array) {
+    const doc = await this.getDoc(docId);
+
+    if (!doc) {
+      return null;
+    }
+
+    return {
+      docId,
+      missing: state && state.length > 0 ? diffUpdate(doc.bin, state) : doc.bin,
+      state: encodeStateVectorFromUpdate(doc.bin),
+      timestamp: doc.timestamp,
+    };
+  }
+
+  abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
+
+  abstract getDocTimestamp(docId: string): Promise<DocClock | null>;
+
+  abstract getDocTimestamps(after?: Date): Promise<DocClocks>;
+
+  abstract deleteDoc(docId: string): Promise<void>;
+
+  subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
+    this.event.on('update', callback);
+
+    return () => {
+      this.event.off('update', callback);
+    };
+  }
+
+  async crawlDocData(_docId: string): Promise<CrawlResult | null> {
+    return null;
+  }
+
+  // REGION: api for internal usage
+  protected on(
+    event: 'update',
+    callback: (update: DocRecord, origin: string) => void
+  ): () => void;
+  protected on(
+    event: 'snapshot',
+    callback: (snapshot: DocRecord, prevSnapshot: DocRecord | null) => void
+  ): () => void;
+  protected on(event: string, callback: (...args: any[]) => void): () => void {
+    this.event.on(event, callback);
+    return () => {
+      this.event.off(event, callback);
+    };
+  }
+
+  protected emit(event: 'update', update: DocRecord, origin?: string): void;
+  protected emit(
+    event: 'snapshot',
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): void;
+  protected emit(event: string, ...args: any[]): void {
+    this.event.emit(event, ...args);
+  }
+
+  protected off(event: string, callback: (...args: any[]) => void): void {
+    this.event.off(event, callback);
+  }
+
+  /**
+   * Get a doc snapshot from storage
+   */
+  protected abstract getDocSnapshot(docId: string): Promise<DocRecord | null>;
+  /**
+   * Set the doc snapshot into storage
+   *
+   * @safety
+   * be careful when implementing this method.
+   *
+   * It might be called with outdated snapshot when running in multi-thread environment.
+   *
+   * A common solution is update the snapshot record is DB only when the coming one's timestamp is newer.
+   *
+   * @example
+   * ```ts
+   * await using _lock = await this.lockDocForUpdate(docId);
+   * // set snapshot
+   *
+   * ```
+   */
+  protected abstract setDocSnapshot(
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): Promise<boolean>;
+
+  /**
+   * Get all updates of a doc that haven't been merged into snapshot.
+   *
+   * Updates queue design exists for a performace concern:
+   * A huge amount of write time will be saved if we don't merge updates into snapshot immediately.
+   * Updates will be merged into snapshot when the latest doc is requested.
+   */
+  protected abstract getDocUpdates(docId: string): Promise<DocRecord[]>;
+
+  /**
+   * Mark updates as merged into snapshot.
+   */
+  protected abstract markUpdatesMerged(
+    docId: string,
+    updates: DocRecord[]
+  ): Promise<number>;
+
+  /**
+   * Merge doc updates into a single update.
+   */
+  protected async squash(updates: DocRecord[]): Promise<DocRecord> {
+    const lastUpdate = updates.at(-1);
+    if (!lastUpdate) {
+      throw new Error('No updates to be squashed.');
+    }
+
+    // fast return
+    if (updates.length === 1) {
+      return lastUpdate;
+    }
+
+    const finalUpdate = await this.mergeUpdates(updates.map(u => u.bin));
+
+    return {
+      docId: lastUpdate.docId,
+      bin: finalUpdate,
+      timestamp: lastUpdate.timestamp,
+      editor: lastUpdate.editor,
+    };
+  }
+
+  protected mergeUpdates(updates: Uint8Array[]) {
+    const merge = this.options?.mergeUpdates ?? mergeUpdates;
+
+    return merge(updates.filter(bin => !isEmptyUpdate(bin)));
+  }
+
+  protected async lockDocForUpdate(docId: string): Promise<AsyncDisposable> {
+    return this.locker.lock(`workspace:${this.spaceId}:update`, docId);
+  }
+}

packages/common/nbstore/src/storage/dummy/awareness.ts

@@ -0,0 +1,16 @@
+import { DummyConnection } from '../../connection';
+import { type AwarenessRecord, AwarenessStorageBase } from '../awareness';
+
+export class DummyAwarenessStorage extends AwarenessStorageBase {
+  override update(_record: AwarenessRecord, _origin?: string): Promise<void> {
+    return Promise.resolve();
+  }
+  override subscribeUpdate(
+    _id: string,
+    _onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    _onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void {
+    return () => {};
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/blob-sync.ts

@@ -0,0 +1,14 @@
+import { type Connection, DummyConnection } from '../../connection';
+import type { BlobSyncStorage } from '../blob-sync';
+
+export class DummyBlobSyncStorage implements BlobSyncStorage {
+  storageType = 'blobSync' as const;
+  connection: Connection<any> = new DummyConnection();
+
+  setBlobUploadedAt(): Promise<void> {
+    return Promise.resolve();
+  }
+  getBlobUploadedAt(): Promise<Date | null> {
+    return Promise.resolve(new Date());
+  }
+}

packages/common/nbstore/src/storage/dummy/blob.ts

@@ -0,0 +1,33 @@
+import { DummyConnection } from '../../connection';
+import {
+  type BlobRecord,
+  BlobStorageBase,
+  type ListedBlobRecord,
+} from '../blob';
+
+export class DummyBlobStorage extends BlobStorageBase {
+  override readonly isReadonly = true;
+  override get(
+    _key: string,
+    _signal?: AbortSignal
+  ): Promise<BlobRecord | null> {
+    return Promise.resolve(null);
+  }
+  override set(_blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  override delete(
+    _key: string,
+    _permanently: boolean,
+    _signal?: AbortSignal
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override release(_signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  override list(_signal?: AbortSignal): Promise<ListedBlobRecord[]> {
+    return Promise.resolve([]);
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/doc-sync.ts

@@ -0,0 +1,49 @@
+import { DummyConnection } from '../../connection';
+import type { DocClock, DocClocks } from '../doc';
+import { DocSyncStorageBase } from '../doc-sync';
+
+export class DummyDocSyncStorage extends DocSyncStorageBase {
+  override getPeerRemoteClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerRemoteClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerRemoteClock(_peer: string, _clock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override getPeerPulledRemoteClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerPulledRemoteClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerPulledRemoteClock(
+    _peer: string,
+    _clock: DocClock
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override getPeerPushedClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerPushedClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerPushedClock(_peer: string, _clock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override clearClocks(): Promise<void> {
+    return Promise.resolve();
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/doc.ts

@@ -0,0 +1,42 @@
+import { DummyConnection } from '../../connection';
+import {
+  type DocClock,
+  type DocClocks,
+  type DocDiff,
+  type DocRecord,
+  type DocStorage,
+  type DocUpdate,
+} from '../doc';
+
+export class DummyDocStorage implements DocStorage {
+  spaceId = '';
+  readonly storageType = 'doc';
+  readonly isReadonly = true;
+  getDoc(_docId: string): Promise<DocRecord | null> {
+    return Promise.resolve(null);
+  }
+  getDocDiff(_docId: string, _state?: Uint8Array): Promise<DocDiff | null> {
+    return Promise.resolve(null);
+  }
+  pushDocUpdate(update: DocUpdate, _origin?: string): Promise<DocClock> {
+    return Promise.resolve({
+      docId: update.docId,
+      timestamp: new Date(),
+    });
+  }
+  getDocTimestamp(_docId: string): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  getDocTimestamps(_after?: Date): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  deleteDoc(_docId: string): Promise<void> {
+    return Promise.resolve();
+  }
+  subscribeDocUpdate(
+    _callback: (update: DocRecord, origin?: string) => void
+  ): () => void {
+    return () => {};
+  }
+  connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/indexer-sync.ts

@@ -0,0 +1,15 @@
+import { DummyConnection } from '../../connection';
+import { type DocIndexedClock, IndexerSyncStorageBase } from '../indexer-sync';
+
+export class DummyIndexerSyncStorage extends IndexerSyncStorageBase {
+  override connection = new DummyConnection();
+  override getDocIndexedClock(_docId: string): Promise<DocIndexedClock | null> {
+    return Promise.resolve(null);
+  }
+  override setDocIndexedClock(_docClock: DocIndexedClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override clearDocIndexedClock(_docId: string): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/common/nbstore/src/storage/dummy/indexer.ts

@@ -0,0 +1,94 @@
+import { NEVER, type Observable } from 'rxjs';
+
+import { DummyConnection } from '../../connection';
+import {
+  type AggregateOptions,
+  type AggregateResult,
+  type IndexerDocument,
+  type IndexerSchema,
+  IndexerStorageBase,
+  type Query,
+  type SearchOptions,
+  type SearchResult,
+} from '../indexer';
+
+export class DummyIndexerStorage extends IndexerStorageBase {
+  readonly isReadonly = true;
+  readonly connection = new DummyConnection();
+
+  override search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Promise<SearchResult<T, O>> {
+    return Promise.resolve({
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      nodes: [],
+    });
+  }
+  override aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Promise<AggregateResult<T, O>> {
+    return Promise.resolve({
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      buckets: [],
+    });
+  }
+  override search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Observable<SearchResult<T, O>> {
+    return NEVER;
+  }
+  override aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return NEVER;
+  }
+
+  override deleteByQuery<T extends keyof IndexerSchema>(
+    _table: T,
+    _query: Query<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override insert<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override delete<T extends keyof IndexerSchema>(
+    _table: T,
+    _id: string
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override update<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
+    return Promise.resolve();
+  }
+  override async refreshIfNeed(): Promise<void> {
+    return;
+  }
+  override async indexVersion(): Promise<number> {
+    return 0;
+  }
+}

packages/common/nbstore/src/storage/errors/index.ts

@@ -0,0 +1,2 @@
+export * from './over-capacity';
+export * from './over-size';

packages/common/nbstore/src/storage/errors/over-capacity.ts

@@ -0,0 +1,5 @@
+export class OverCapacityError extends Error {
+  constructor(public originError?: any) {
+    super('Storage over capacity.');
+  }
+}

packages/common/nbstore/src/storage/errors/over-size.ts

@@ -0,0 +1,10 @@
+export class OverSizeError extends Error {
+  constructor(limit: string | null, message?: string) {
+    if (message) {
+      super(message);
+    } else {
+      const formattedLimit = limit ? `${limit} ` : '';
+      super(`File size exceeds the ${formattedLimit}limit.`);
+    }
+  }
+}

packages/common/nbstore/src/storage/history.ts

@@ -0,0 +1,122 @@
+import { noop } from 'lodash-es';
+import {
+  applyUpdate,
+  Doc,
+  encodeStateAsUpdate,
+  encodeStateVector,
+  UndoManager,
+} from 'yjs';
+
+import { type DocRecord, DocStorageBase, type DocStorageOptions } from './doc';
+
+export interface HistoryFilter {
+  before?: Date;
+  limit?: Date;
+}
+
+export interface ListedHistory {
+  userId: string | null;
+  timestamp: Date;
+}
+
+export abstract class HistoricalDocStorage<
+  Options extends DocStorageOptions = DocStorageOptions,
+> extends DocStorageBase<Options> {
+  constructor(opts: Options) {
+    super(opts);
+
+    this.on('snapshot', snapshot => {
+      this.createHistory(snapshot.docId, snapshot).catch(noop);
+    });
+  }
+
+  override async setDocSnapshot(
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): Promise<boolean> {
+    const success = await this.upsertDocSnapshot(snapshot, prevSnapshot);
+    if (success) {
+      this.emit('snapshot', snapshot, prevSnapshot);
+    }
+    return success;
+  }
+
+  /**
+   * Update the doc snapshot in storage or create a new one if not exists.
+   *
+   * @safety
+   * be careful when implementing this method.
+   *
+   * It might be called with outdated snapshot when running in multi-thread environment.
+   *
+   * A common solution is update the snapshot record is DB only when the coming one's timestamp is newer.
+   *
+   * @example
+   * ```ts
+   * await using _lock = await this.lockDocForUpdate(docId);
+   * // set snapshot
+   *
+   * ```
+   */
+  abstract upsertDocSnapshot(
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): Promise<boolean>;
+
+  abstract listHistories(
+    docId: string,
+    filter?: HistoryFilter
+  ): Promise<ListedHistory[]>;
+  abstract getHistory(
+    docId: string,
+    timestamp: Date
+  ): Promise<DocRecord | null>;
+  abstract deleteHistory(docId: string, timestamp: Date): Promise<void>;
+
+  async rollbackDoc(docId: string, timestamp: Date, editor?: string) {
+    const toSnapshot = await this.getHistory(docId, timestamp);
+    if (!toSnapshot) {
+      throw new Error('Can not find the version to rollback to.');
+    }
+
+    const fromSnapshot = await this.getDoc(docId);
+
+    if (!fromSnapshot) {
+      throw new Error('Can not find the current version of the doc.');
+    }
+
+    const change = this.generateRevertUpdate(fromSnapshot.bin, toSnapshot.bin);
+    await this.pushDocUpdate({ docId, bin: change, editor }, 'rollback');
+    // force create a new history record after rollback
+    await this.createHistory(docId, fromSnapshot);
+  }
+
+  // history can only be created upon update pushing.
+  protected abstract createHistory(
+    docId: string,
+    snapshot: DocRecord
+  ): Promise<void>;
+
+  protected generateRevertUpdate(
+    fromNewerBin: Uint8Array,
+    toOlderBin: Uint8Array
+  ): Uint8Array {
+    const newerDoc = new Doc();
+    applyUpdate(newerDoc, fromNewerBin);
+    const olderDoc = new Doc();
+    applyUpdate(olderDoc, toOlderBin);
+
+    const newerState = encodeStateVector(newerDoc);
+    const olderState = encodeStateVector(olderDoc);
+
+    const diff = encodeStateAsUpdate(newerDoc, olderState);
+
+    const undoManager = new UndoManager(Array.from(olderDoc.share.values()));
+
+    applyUpdate(olderDoc, diff);
+
+    undoManager.undo();
+
+    return encodeStateAsUpdate(olderDoc, newerState);
+  }
+}

packages/common/nbstore/src/storage/index.ts

@@ -0,0 +1,96 @@
+import EventEmitter2 from 'eventemitter2';
+
+import type { AwarenessStorage } from './awareness';
+import type { BlobStorage } from './blob';
+import type { BlobSyncStorage } from './blob-sync';
+import type { DocStorage } from './doc';
+import type { DocSyncStorage } from './doc-sync';
+import { DummyAwarenessStorage } from './dummy/awareness';
+import { DummyBlobStorage } from './dummy/blob';
+import { DummyBlobSyncStorage } from './dummy/blob-sync';
+import { DummyDocStorage } from './dummy/doc';
+import { DummyDocSyncStorage } from './dummy/doc-sync';
+import { DummyIndexerStorage } from './dummy/indexer';
+import { DummyIndexerSyncStorage } from './dummy/indexer-sync';
+import type { IndexerStorage } from './indexer';
+import type { IndexerSyncStorage } from './indexer-sync';
+import type { StorageType } from './storage';
+
+type Storages =
+  | DocStorage
+  | BlobStorage
+  | BlobSyncStorage
+  | DocSyncStorage
+  | AwarenessStorage
+  | IndexerStorage
+  | IndexerSyncStorage;
+
+export type SpaceStorageOptions = {
+  [K in StorageType]?: Storages & { storageType: K };
+};
+
+export class SpaceStorage {
+  protected readonly storages: {
+    [K in StorageType]: Storages & { storageType: K };
+  };
+  private readonly event = new EventEmitter2();
+  private readonly disposables: Set<() => void> = new Set();
+
+  constructor(storages: SpaceStorageOptions) {
+    this.storages = {
+      awareness: storages.awareness ?? new DummyAwarenessStorage(),
+      blob: storages.blob ?? new DummyBlobStorage(),
+      blobSync: storages.blobSync ?? new DummyBlobSyncStorage(),
+      doc: storages.doc ?? new DummyDocStorage(),
+      docSync: storages.docSync ?? new DummyDocSyncStorage(),
+      indexer: storages.indexer ?? new DummyIndexerStorage(),
+      indexerSync: storages.indexerSync ?? new DummyIndexerSyncStorage(),
+    };
+  }
+
+  get<T extends StorageType>(type: T): Extract<Storages, { storageType: T }> {
+    const storage = this.storages[type];
+
+    if (!storage) {
+      throw new Error(`Storage ${type} not registered.`);
+    }
+
+    return storage as unknown as Extract<Storages, { storageType: T }>;
+  }
+
+  connect() {
+    Object.values(this.storages).forEach(storage => {
+      storage.connection.connect();
+    });
+  }
+
+  disconnect() {
+    Object.values(this.storages).forEach(storage => {
+      storage.connection.disconnect();
+    });
+  }
+
+  async waitForConnected(signal?: AbortSignal) {
+    await Promise.all(
+      Object.values(this.storages).map(storage =>
+        storage.connection.waitForConnected(signal)
+      )
+    );
+  }
+
+  async destroy() {
+    this.disposables.forEach(disposable => disposable());
+    this.event.removeAllListeners();
+  }
+}
+
+export * from './awareness';
+export * from './blob';
+export * from './blob-sync';
+export * from './doc';
+export * from './doc-sync';
+export * from './errors';
+export * from './history';
+export * from './indexer';
+export * from './indexer-sync';
+export * from './storage';

packages/common/nbstore/src/storage/indexer-sync.ts

@@ -0,0 +1,31 @@
+import type { Connection } from '../connection';
+import type { DocClock } from './doc';
+import type { Storage } from './storage';
+
+export interface DocIndexedClock extends DocClock {
+  indexerVersion: number;
+}
+
+export interface IndexerSyncStorage extends Storage {
+  readonly storageType: 'indexerSync';
+  readonly commitsIndexAtomically: boolean;
+
+  getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
+
+  setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
+  setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise<void>;
+
+  clearDocIndexedClock(docId: string): Promise<void>;
+}
+
+export abstract class IndexerSyncStorageBase implements IndexerSyncStorage {
+  readonly storageType = 'indexerSync';
+  readonly commitsIndexAtomically: boolean = false;
+  abstract connection: Connection<any>;
+  abstract getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
+  abstract setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
+  async setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise<void> {
+    for (const clock of docClocks) await this.setDocIndexedClock(clock);
+  }
+  abstract clearDocIndexedClock(docId: string): Promise<void>;
+}

packages/common/nbstore/src/storage/indexer.ts

@@ -0,0 +1,184 @@
+export * from './indexer/document';
+export * from './indexer/field-type';
+export * from './indexer/query';
+export * from './indexer/schema';
+
+import type { Observable } from 'rxjs';
+
+import type { Connection } from '../connection';
+import type { IndexerDocument } from './indexer/document';
+import type { Query } from './indexer/query';
+import type { IndexerSchema } from './indexer/schema';
+import type { Storage } from './storage';
+
+export interface IndexerStorage extends Storage {
+  readonly storageType: 'indexer';
+  readonly isReadonly: boolean;
+  readonly recommendRefreshInterval: number; // 100ms
+
+  search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Promise<SearchResult<T, O>>;
+
+  aggregate<T extends keyof IndexerSchema, const O extends AggregateOptions<T>>(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>>;
+
+  search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Observable<SearchResult<T, O>>;
+
+  aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>>;
+
+  deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void>;
+
+  insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  delete<T extends keyof IndexerSchema>(table: T, id: string): Promise<void>;
+
+  update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
+  refreshIfNeed(): Promise<void>;
+  indexVersion(): Promise<number>;
+}
+
+type ResultPagination = {
+  count: number;
+  limit: number;
+  skip: number;
+  hasMore: boolean;
+};
+
+type PaginationOption = { limit?: number; skip?: number };
+
+type HighlightAbleField<T extends keyof IndexerSchema> = {
+  [K in keyof IndexerSchema[T]]: IndexerSchema[T][K] extends 'FullText'
+    ? K
+    : never;
+}[keyof IndexerSchema[T]];
+
+export type SearchOptions<T extends keyof IndexerSchema> = {
+  pagination?: PaginationOption;
+  highlights?: { field: HighlightAbleField<T>; before: string; end: string }[];
+  fields?: (keyof IndexerSchema[T])[];
+};
+
+export type SearchResult<
+  T extends keyof IndexerSchema,
+  O extends SearchOptions<T>,
+> = {
+  pagination: ResultPagination;
+  nodes: ({ id: string; score: number } & (O['fields'] extends any[]
+    ? { fields: { [key in O['fields'][number]]: string | string[] } }
+    : unknown) &
+    (O['highlights'] extends any[]
+      ? { highlights: { [key in O['highlights'][number]['field']]: string[] } }
+      : unknown))[];
+};
+
+export interface AggregateOptions<T extends keyof IndexerSchema> {
+  pagination?: PaginationOption;
+  hits?: SearchOptions<T>;
+}
+
+export type AggregateResult<
+  T extends keyof IndexerSchema,
+  O extends AggregateOptions<T>,
+> = {
+  pagination: ResultPagination;
+  buckets: ({
+    key: string;
+    score: number;
+    count: number;
+  } & (O['hits'] extends object
+    ? { hits: SearchResult<T, O['hits']> }
+    : unknown))[];
+};
+
+export abstract class IndexerStorageBase implements IndexerStorage {
+  readonly storageType = 'indexer';
+  readonly recommendRefreshInterval: number = 100; // 100ms
+  abstract readonly connection: Connection;
+  abstract readonly isReadonly: boolean;
+
+  abstract search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>>;
+
+  abstract aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>>;
+
+  abstract search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>>;
+
+  abstract aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>>;
+
+  abstract deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void>;
+
+  abstract insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  abstract delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void>;
+
+  abstract update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  abstract refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
+
+  abstract refreshIfNeed(): Promise<void>;
+
+  abstract indexVersion(): Promise<number>;
+}

packages/common/nbstore/src/storage/indexer/document.ts

@@ -0,0 +1,58 @@
+import type { IndexerSchema } from './schema';
+
+export class IndexerDocument<
+  S extends keyof IndexerSchema = keyof IndexerSchema,
+> {
+  constructor(public readonly id: string) {}
+
+  fields = new Map<keyof IndexerSchema[S], string[]>();
+
+  public insert<F extends keyof IndexerSchema[S]>(
+    field: F,
+    value: string | string[]
+  ) {
+    const values = this.fields.get(field) ?? [];
+    if (Array.isArray(value)) {
+      values.push(...value);
+    } else {
+      values.push(value);
+    }
+    this.fields.set(field, values);
+  }
+
+  get<F extends keyof IndexerSchema[S]>(
+    field: F
+  ): string[] | string | undefined {
+    const values = this.fields.get(field);
+    if (values === undefined) {
+      return undefined;
+    } else if (values.length === 1) {
+      return values[0];
+    } else {
+      return values;
+    }
+  }
+
+  static from<S extends keyof IndexerSchema>(
+    id: string,
+    map:
+      | Partial<Record<keyof IndexerSchema[S], string | string[]>>
+      | Map<keyof IndexerSchema[S], string | string[]>
+  ): IndexerDocument<S> {
+    const doc = new IndexerDocument<S>(id);
+
+    if (map instanceof Map) {
+      for (const [key, value] of map) {
+        doc.insert(key, value);
+      }
+    } else {
+      for (const key in map) {
+        if (map[key] === undefined || map[key] === null) {
+          continue;
+        }
+        doc.insert(key, map[key]);
+      }
+    }
+    return doc;
+  }
+}

packages/common/nbstore/src/storage/indexer/field-type.ts

@@ -0,0 +1 @@
+export type IndexFieldType = 'Integer' | 'FullText' | 'String' | 'Boolean';

packages/common/nbstore/src/storage/indexer/query.ts

@@ -0,0 +1,35 @@
+import type { IndexerSchema } from './schema';
+
+export type MatchQuery<T extends keyof IndexerSchema> = {
+  type: 'match';
+  field: keyof IndexerSchema[T];
+  match: string;
+};
+
+export type BoostQuery = {
+  type: 'boost';
+  query: Query<any>;
+  boost: number;
+};
+
+export type BooleanQuery<T extends keyof IndexerSchema> = {
+  type: 'boolean';
+  occur: 'should' | 'must' | 'must_not';
+  queries: Query<T>[];
+};
+
+export type ExistsQuery<T extends keyof IndexerSchema> = {
+  type: 'exists';
+  field: keyof IndexerSchema[T];
+};
+
+export type AllQuery = {
+  type: 'all';
+};
+
+export type Query<T extends keyof IndexerSchema> =
+  | BooleanQuery<T>
+  | MatchQuery<T>
+  | AllQuery
+  | ExistsQuery<T>
+  | BoostQuery;

packages/common/nbstore/src/storage/indexer/schema.ts

@@ -0,0 +1,51 @@
+import type { IndexFieldType } from './field-type';
+
+export const IndexerSchema = {
+  doc: {
+    docId: 'String',
+    title: 'FullText',
+    // summary of the doc, used for preview
+    summary: { type: 'String', index: false },
+  },
+  block: {
+    docId: 'String',
+    blockId: 'String',
+    content: 'FullText',
+    flavour: 'String',
+    blob: 'String',
+    // reference doc id
+    // ['xxx','yyy']
+    refDocId: 'String',
+    // reference info, used for backlink to specific block
+    // [{"docId":"xxx","mode":"page","blockIds":["gt5Yfq1maYvgNgpi13rIq"]},{"docId":"yyy","mode":"edgeless","blockIds":["k5prpOlDF-9CzfatmO0W7"]}]
+    ref: { type: 'String', index: false },
+    // parent block flavour
+    parentFlavour: 'String',
+    // parent block id
+    parentBlockId: 'String',
+    // additional info
+    // { "databaseName": "xxx", "displayMode": "page/edgeless", "noteBlockId": "xxx" }
+    additional: { type: 'String', index: false },
+    markdownPreview: { type: 'String', index: false },
+  },
+} satisfies Record<string, Record<string, IndexerFieldSchema>>;
+
+export type IndexerFieldSchema =
+  | IndexFieldType
+  | {
+      type: IndexFieldType;
+      /**
+       * If false, the field will not be indexed, and thus not searchable.
+       *
+       * default: true
+       */
+      index?: boolean;
+      /**
+       * If false, the field will not be stored, and not included in the search result.
+       *
+       * default: true
+       */
+      store?: boolean;
+    };
+
+export type IndexerSchema = typeof IndexerSchema;

packages/common/nbstore/src/storage/lock.ts

@@ -0,0 +1,44 @@
+export interface Locker {
+  lock(domain: string, resource: string): Promise<AsyncDisposable>;
+}
+
+export class SingletonLocker implements Locker {
+  lockedResource = new Map<string, Lock>();
+  constructor() {}
+
+  async lock(domain: string, resource: string) {
+    const key = `${domain}:${resource}`;
+    let lock = this.lockedResource.get(key);
+
+    if (!lock) {
+      lock = new Lock();
+      this.lockedResource.set(key, lock);
+    }
+
+    await lock.acquire();
+
+    return lock;
+  }
+}
+
+export class Lock {
+  private inner: Promise<void> = Promise.resolve();
+  private release: () => void = () => {};
+
+  async acquire() {
+    // oxlint-disable-next-line typescript/no-non-null-assertion
+    let release: () => void = null!;
+    const nextLock = new Promise<void>(resolve => {
+      release = resolve;
+    });
+
+    await this.inner;
+    this.inner = nextLock;
+    this.release = release;
+  }
+
+  [Symbol.asyncDispose]() {
+    this.release();
+    return Promise.resolve();
+  }
+}

packages/common/nbstore/src/storage/storage.ts

@@ -0,0 +1,15 @@
+import type { Connection } from '../connection';
+
+export type StorageType =
+  | 'blob'
+  | 'blobSync'
+  | 'doc'
+  | 'docSync'
+  | 'awareness'
+  | 'indexer'
+  | 'indexerSync';
+
+export interface Storage {
+  readonly storageType: StorageType;
+  readonly connection: Connection;
+}