snapshot of https://github.com/toeverything/AFFiNE.git (canary, 2026-09-16; blobs > 131072B omitted)

packages/common/nbstore/README.md

@@ -0,0 +1,80 @@
+# Space Storage
+
+## Usage
+
+### Independent Storage usage
+
+```ts
+import type { ConnectionStatus } from '@affine/nbstore';
+import { IndexedDBDocStorage } from '@affine/nbstore/idb';
+
+const storage = new IndexedDBDocStorage({
+  peer: 'local'
+  spaceId: 'my-new-workspace',
+});
+
+await storage.connect();
+storage.connection.onStatusChange((status: ConnectionStatus, error?: Error) => {
+  ui.show(status, error);
+});
+
+// { docId: string, bin: Uint8Array, timestamp: Date, editor?: string } | null
+const doc = await storage.getDoc('my-first-doc');
+```
+
+### Use All storages together
+
+```ts
+import { SpaceStorage } from '@affine/nbstore';
+import type { ConnectionStatus } from '@affine/nbstore';
+import { IndexedDBDocStorage } from '@affine/nbstore/idb';
+import { SqliteBlobStorage } from '@affine/nbstore/sqlite';
+
+const storage = new SpaceStorage([
+  new IndexedDBDocStorage({}),
+  new SqliteBlobStorage({}),
+]);
+
+await storage.connect();
+storage.on('connection', ({ storage, status, error }) => {
+  ui.show(storage, status, error);
+});
+
+await storage.get('doc').pushDocUpdate({
+  docId: 'my-first-doc',
+  bin: new Uint8Array(),
+  editor: 'me',
+});
+await storage.tryGet('blob')?.get('img');
+```
+
+### Put Storage behind Worker
+
+```ts
+import { SpaceStorageWorkerClient } from '@affine/nbstore/op';
+import type { ConnectionStatus } from '@affine/nbstore';
+import { IndexedDBDocStorage } from '@affine/nbstore/idb';
+
+const client = new SpaceStorageWorkerClient();
+client.addStorage(IndexedDBDocStorage, {
+  // options can only be structure-cloneable type
+  peer: 'local',
+  spaceType: 'workspace',
+  spaceId: 'my-new-workspace',
+});
+
+await client.connect();
+client.ob$('connection', ({ storage, status, error }) => {
+  ui.show(storage, status, error);
+});
+
+await client.call('pushDocUpdate', {
+  docId: 'my-first-doc',
+  bin: new Uint8Array(),
+  editor: 'me',
+});
+
+// call unregistered op will leads to Error
+// Error { message: 'Handler for operation [listHistory] is not registered.' }
+await client.call('listHistories', { docId: 'my-first-doc' });
+```

packages/common/nbstore/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@affine/nbstore",
+  "type": "module",
+  "version": "0.27.5",
+  "private": true,
+  "sideEffects": false,
+  "exports": {
+    ".": "./src/index.ts",
+    "./worker/client": "./src/worker/client.ts",
+    "./worker/consumer": "./src/worker/consumer.ts",
+    "./idb": "./src/impls/idb/index.ts",
+    "./broadcast-channel": "./src/impls/broadcast-channel/index.ts",
+    "./idb/v1": "./src/impls/idb/v1/index.ts",
+    "./cloud": "./src/impls/cloud/index.ts",
+    "./sqlite": "./src/impls/sqlite/index.ts",
+    "./sqlite/v1": "./src/impls/sqlite/v1/index.ts",
+    "./sync": "./src/sync/index.ts",
+    "./frontend": "./src/frontend/index.ts"
+  },
+  "dependencies": {
+    "@affine/reader": "workspace:*",
+    "@affine/realtime": "workspace:*",
+    "@toeverything/infra": "workspace:*",
+    "eventemitter2": "^6.4.9",
+    "graphemer": "^1.4.0",
+    "lodash-es": "^4.17.23",
+    "nanoid": "^5.1.6",
+    "rxjs": "^7.8.2",
+    "uuid": "^14.0.0",
+    "y-protocols": "^1.0.6",
+    "yjs": "^13.6.27"
+  },
+  "devDependencies": {
+    "@affine/error": "workspace:*",
+    "@affine/graphql": "workspace:*",
+    "@affine/realtime": "workspace:*",
+    "@blocksuite/affine": "workspace:*",
+    "fake-indexeddb": "^6.0.0",
+    "idb": "^8.0.0",
+    "socket.io-client": "^4.8.3",
+    "vitest": "^4.1.8"
+  },
+  "peerDependencies": {
+    "@affine/error": "workspace:*",
+    "@affine/graphql": "workspace:*",
+    "@affine/realtime": "workspace:*",
+    "@blocksuite/affine": "workspace:*",
+    "idb": "^8.0.0",
+    "socket.io-client": "^4.8.3"
+  }
+}

packages/common/nbstore/src/__tests__/__snapshots__/cloud-doc-updates.spec.ts.snap

@@ -0,0 +1,224 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`CloudDocStorage broadcast updates > awareness joins active documents through the batch route 1`] = `
+[
+  {
+    "event": "space:join-batch",
+    "payload": {
+      "clientVersion": "0.27.5",
+      "spaces": [
+        {
+          "docId": "doc-a",
+          "spaceId": "space-1",
+          "spaceType": "workspace",
+        },
+        {
+          "docId": "doc-b",
+          "spaceId": "space-1",
+          "spaceType": "workspace",
+        },
+      ],
+    },
+  },
+  {
+    "event": "space:load-awarenesses",
+    "payload": {
+      "docId": "doc-a",
+      "spaceId": "space-1",
+      "spaceType": "workspace",
+    },
+  },
+  {
+    "event": "space:load-awarenesses",
+    "payload": {
+      "docId": "doc-b",
+      "spaceId": "space-1",
+      "spaceType": "workspace",
+    },
+  },
+]
+`;
+
+exports[`CloudDocStorage broadcast updates > batch route joins the workspace and applies lifecycle commands 1`] = `
+{
+  "emitted": [
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "space-1",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:load-doc",
+      "payload": {
+        "docId": "space-1",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+      },
+    },
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "doc-1",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:load-doc",
+      "payload": {
+        "docId": "doc-1",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+      },
+    },
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "doc-2",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:load-doc",
+      "payload": {
+        "docId": "doc-2",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+        "stateVector": undefined,
+      },
+    },
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "doc-3",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:load-doc",
+      "payload": {
+        "docId": "doc-3",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+      },
+    },
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "doc-4",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:push-doc-update",
+      "payload": {
+        "docId": "doc-4",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+        "update": "AAA=",
+      },
+    },
+    {
+      "event": "space:doc-lifecycle",
+      "payload": {
+        "docId": "doc-1",
+        "lifecycle": "trash",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+      },
+    },
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "denied",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:join-batch",
+      "payload": {
+        "clientVersion": "0.27.5",
+        "spaces": [
+          {
+            "docId": "doc-1",
+            "spaceId": "space-1",
+            "spaceType": "workspace",
+          },
+        ],
+      },
+    },
+    {
+      "event": "space:load-doc",
+      "payload": {
+        "docId": "doc-1",
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+        "stateVector": undefined,
+      },
+    },
+    {
+      "event": "space:leave-batch",
+      "payload": {
+        "docIds": [
+          "space-1",
+          "doc-1",
+          "doc-2",
+          "doc-3",
+          "doc-4",
+        ],
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+      },
+    },
+    {
+      "event": "space:leave-batch",
+      "payload": {
+        "docIds": [],
+        "spaceId": "space-1",
+        "spaceType": "workspace",
+      },
+    },
+  ],
+  "lifecycle": {
+    "rootUpdate": Any<Uint8Array>,
+    "timestamp": Any<Date>,
+  },
+}
+`;

packages/common/nbstore/src/__tests__/base64.bench.ts

@@ -0,0 +1,23 @@
+import { bench, describe } from 'vitest';
+
+import { base64ToUint8Array, uint8ArrayToBase64 } from '../impls/cloud/socket';
+
+const data = new Uint8Array(1024 * 256);
+for (let i = 0; i < data.length; i++) {
+  data[i] = i % 251;
+}
+let encoded = '';
+
+await uint8ArrayToBase64(data).then(result => {
+  encoded = result;
+});
+
+describe('base64 helpers', () => {
+  bench('encode Uint8Array to base64', async () => {
+    await uint8ArrayToBase64(data);
+  });
+
+  bench('decode base64 to Uint8Array', () => {
+    base64ToUint8Array(encoded);
+  });
+});

packages/common/nbstore/src/__tests__/base64.spec.ts

@@ -0,0 +1,27 @@
+import { describe, expect, test } from 'vitest';
+
+import { base64ToUint8Array, uint8ArrayToBase64 } from '../impls/cloud/socket';
+
+function makeSample(size: number) {
+  const data = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    data[i] = i % 251;
+  }
+  return data;
+}
+
+describe('base64 helpers', () => {
+  test('roundtrip preserves data', async () => {
+    const input = makeSample(1024);
+    const encoded = await uint8ArrayToBase64(input);
+    const decoded = base64ToUint8Array(encoded);
+    expect(decoded).toEqual(input);
+  });
+
+  test('handles large payloads', async () => {
+    const input = makeSample(256 * 1024);
+    const encoded = await uint8ArrayToBase64(input);
+    const decoded = base64ToUint8Array(encoded);
+    expect(decoded).toEqual(input);
+  });
+});

packages/common/nbstore/src/__tests__/cloud-blob.spec.ts

@@ -0,0 +1,943 @@
+import { getEventListeners } from 'node:events';
+
+import { UserFriendlyError } from '@affine/error';
+import {
+  abortBlobUploadMutation,
+  BlobUploadMethod,
+  completeBlobUploadMutation,
+  createBlobUploadMutation,
+  getBlobUploadPartUrlQuery,
+  listBlobsQuery,
+  setBlobMutation,
+  workspaceBlobQuotaQuery,
+} from '@affine/graphql';
+import { afterEach, expect, test, vi } from 'vitest';
+
+import { CloudBlobStorage } from '../impls/cloud/blob';
+import { BlobSourceRegistry } from '../impls/cloud/blob-source-registry';
+import { OverSizeError } from '../storage';
+
+const originalBuildConfig = globalThis.BUILD_CONFIG;
+const quotaResponse = {
+  workspace: {
+    quota: {
+      humanReadable: {
+        blobLimit: '1 MB',
+      },
+      blobLimit: 1024 * 1024,
+    },
+  },
+};
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.restoreAllMocks();
+  vi.unstubAllGlobals();
+  globalThis.BUILD_CONFIG = originalBuildConfig;
+});
+
+function createStorage() {
+  return new CloudBlobStorage({
+    serverBaseUrl: 'https://example.com',
+    id: 'workspace-1',
+  });
+}
+
+test('uses graphql upload when server returns GRAPHQL method', async () => {
+  const storage = createStorage();
+  const gqlMock = vi.fn(async ({ query }) => {
+    if (query === workspaceBlobQuotaQuery) {
+      return quotaResponse;
+    }
+    if (query === createBlobUploadMutation) {
+      return {
+        createBlobUpload: {
+          method: BlobUploadMethod.GRAPHQL,
+          blobKey: 'blob-key',
+          alreadyUploaded: false,
+        },
+      };
+    }
+    if (query === setBlobMutation) {
+      return { setBlob: 'blob-key' };
+    }
+    throw new Error('Unexpected query');
+  });
+
+  vi.spyOn(storage.connection, 'gql').mockImplementation(
+    gqlMock as typeof storage.connection.gql
+  );
+
+  await storage.set({
+    key: 'blob-key',
+    data: new Uint8Array([1, 2, 3]),
+    mime: 'text/plain',
+  });
+
+  const queries = gqlMock.mock.calls.map(call => call[0].query);
+  expect(queries).toContain(createBlobUploadMutation);
+  expect(queries).toContain(setBlobMutation);
+});
+
+test.each(
+  [BlobUploadMethod.PRESIGNED, BlobUploadMethod.MULTIPART].flatMap(method =>
+    ['500', '413', 'complete-413', 'abort', 'abort-string', 'pre-abort'].map(
+      failure => ({ method, failure })
+    )
+  )
+)(
+  'handles $method upload failure $failure without retrying terminal errors',
+  async ({ method, failure }) => {
+    const storage = createStorage();
+    const controller = new AbortController();
+    const reason =
+      failure === 'abort-string'
+        ? 'caller canceled'
+        : new DOMException('caller canceled', 'AbortError');
+    const gqlMock = vi.fn(async ({ query, context }) => {
+      if (query === workspaceBlobQuotaQuery) return quotaResponse;
+      if (query === createBlobUploadMutation) {
+        return {
+          createBlobUpload: {
+            method,
+            blobKey: 'blob-key',
+            alreadyUploaded: false,
+            uploadUrl: 'https://upload.example.com/blob',
+            uploadId: 'upload-1',
+            partSize: 2,
+            uploadedParts: [],
+          },
+        };
+      }
+      if (query === getBlobUploadPartUrlQuery) {
+        return {
+          workspace: {
+            blobUploadPartUrl: { uploadUrl: 'https://upload.example.com/part' },
+          },
+        };
+      }
+      if (query === abortBlobUploadMutation) {
+        expect(context?.signal?.aborted).not.toBe(true);
+        if (failure === '413') throw new Error('cleanup failed');
+        return { abortBlobUpload: true };
+      }
+      if (query === setBlobMutation) return { setBlob: 'blob-key' };
+      if (query === completeBlobUploadMutation) {
+        throw new UserFriendlyError({
+          status: 413,
+          code: 'CONTENT_TOO_LARGE',
+          type: 'CONTENT_TOO_LARGE',
+          name: 'CONTENT_TOO_LARGE',
+          message: 'Content too large',
+        });
+      }
+      throw new Error('Unexpected query');
+    });
+    vi.spyOn(storage.connection, 'gql').mockImplementation(
+      gqlMock as typeof storage.connection.gql
+    );
+    const fetchMock = vi.fn(async (_input: string, init?: RequestInit) => {
+      expect(init?.signal).toBe(controller.signal);
+      if (failure.startsWith('abort')) {
+        controller.abort(reason);
+        throw init?.signal?.reason;
+      }
+      return new Response('', {
+        status: failure === 'complete-413' ? 200 : Number(failure),
+        headers: { etag: 'part-etag' },
+      });
+    });
+    vi.stubGlobal('fetch', fetchMock);
+    if (failure === 'pre-abort') controller.abort(reason);
+
+    const uploading = storage.set(
+      {
+        key: 'blob-key',
+        data: new Uint8Array([1, 2, 3]),
+        mime: 'text/plain',
+      },
+      controller.signal
+    );
+    if (failure === '500') {
+      await expect(uploading).resolves.toBeUndefined();
+    } else if (failure.includes('413')) {
+      await expect(uploading).rejects.toBeInstanceOf(OverSizeError);
+    } else {
+      await expect(uploading).rejects.toBe(reason);
+    }
+
+    const queries = new Set(gqlMock.mock.calls.map(call => call[0].query));
+    expect(queries.has(setBlobMutation)).toBe(failure === '500');
+    expect(queries.has(abortBlobUploadMutation)).toBe(
+      method === BlobUploadMethod.MULTIPART && failure !== 'pre-abort'
+    );
+    expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0);
+    if (failure === 'pre-abort') {
+      expect(gqlMock).not.toHaveBeenCalled();
+      expect(fetchMock).not.toHaveBeenCalled();
+    }
+  }
+);
+
+test('uses presigned upload and completes without graphql fallback', async () => {
+  const storage = createStorage();
+  const gqlMock = vi.fn(async ({ query }) => {
+    if (query === workspaceBlobQuotaQuery) {
+      return quotaResponse;
+    }
+    if (query === createBlobUploadMutation) {
+      return {
+        createBlobUpload: {
+          method: BlobUploadMethod.PRESIGNED,
+          blobKey: 'blob-key',
+          alreadyUploaded: false,
+          uploadUrl: 'https://upload.example.com/blob',
+        },
+      };
+    }
+    if (query === completeBlobUploadMutation) {
+      return { completeBlobUpload: 'blob-key' };
+    }
+    throw new Error('Unexpected query');
+  });
+
+  vi.spyOn(storage.connection, 'gql').mockImplementation(
+    gqlMock as typeof storage.connection.gql
+  );
+  const fetchMock = vi.fn(async () => new Response('', { status: 200 }));
+  vi.stubGlobal('fetch', fetchMock);
+
+  await storage.set({
+    key: 'blob-key',
+    data: new Uint8Array([1, 2, 3]),
+    mime: 'text/plain',
+  });
+
+  const queries = gqlMock.mock.calls.map(call => call[0].query);
+  expect(queries).toContain(completeBlobUploadMutation);
+  expect(queries).not.toContain(setBlobMutation);
+  expect(fetchMock).toHaveBeenCalledWith(
+    'https://upload.example.com/blob',
+    expect.objectContaining({
+      method: 'PUT',
+    })
+  );
+});
+
+test('uses multipart upload and completes without graphql fallback', async () => {
+  const storage = createStorage();
+  const controller = new AbortController();
+  const gqlMock = vi.fn(async ({ query, variables }) => {
+    if (query === workspaceBlobQuotaQuery) {
+      return quotaResponse;
+    }
+    if (query === createBlobUploadMutation) {
+      return {
+        createBlobUpload: {
+          method: BlobUploadMethod.MULTIPART,
+          blobKey: 'blob-key',
+          alreadyUploaded: false,
+          uploadId: 'upload-1',
+          partSize: 2,
+          uploadedParts: [],
+        },
+      };
+    }
+    if (query === getBlobUploadPartUrlQuery) {
+      return {
+        workspace: {
+          blobUploadPartUrl: {
+            uploadUrl: `https://upload.example.com/part/${variables.partNumber}`,
+          },
+        },
+      };
+    }
+    if (query === completeBlobUploadMutation) {
+      return { completeBlobUpload: 'blob-key' };
+    }
+    throw new Error('Unexpected query');
+  });
+
+  vi.spyOn(storage.connection, 'gql').mockImplementation(
+    gqlMock as typeof storage.connection.gql
+  );
+  const fetchMock = vi.fn(async (_input: string, init?: RequestInit) => {
+    expect(init?.signal).toBe(controller.signal);
+    const body = init?.body as ArrayBuffer;
+    const length = body.byteLength;
+    return new Response('', {
+      status: 200,
+      headers: {
+        etag: `etag-${length}`,
+      },
+    });
+  });
+  vi.stubGlobal('fetch', fetchMock);
+
+  await storage.set(
+    {
+      key: 'blob-key',
+      data: new Uint8Array([1, 2, 3]),
+      mime: 'text/plain',
+    },
+    controller.signal
+  );
+
+  const queries = gqlMock.mock.calls.map(call => call[0].query);
+  expect(queries).toContain(getBlobUploadPartUrlQuery);
+  expect(queries).toContain(completeBlobUploadMutation);
+  expect(queries).not.toContain(setBlobMutation);
+  expect(fetchMock).toHaveBeenCalledTimes(2);
+  expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0);
+});
+
+test.each(['success', 'failure', 'abort', 'pre-abort', 'timeout'])(
+  'blob HTTP transport preserves cancellation and releases timers on %s',
+  async outcome => {
+    vi.useFakeTimers();
+    const storage = createStorage();
+    const controller = new AbortController();
+    const reason = new DOMException('caller canceled', 'AbortError');
+    const fetchMock = vi.fn(async (_input: URL, init?: RequestInit) => {
+      if (outcome === 'success') return new Response('ok');
+      if (outcome === 'failure') throw new Error('offline');
+      const signal = init!.signal!;
+      return new Promise<Response>((_resolve, reject) => {
+        signal.addEventListener('abort', () => reject(signal.reason), {
+          once: true,
+        });
+      });
+    });
+    vi.stubGlobal('fetch', fetchMock);
+    if (outcome === 'pre-abort') controller.abort(reason);
+    const pending = storage.connection.fetch('/blob', {
+      signal: controller.signal,
+      timeout: 1000,
+    });
+    const result = pending.then(
+      value => value,
+      error => error
+    );
+    if (outcome === 'abort') controller.abort(reason);
+    if (outcome === 'timeout') await vi.advanceTimersByTimeAsync(1000);
+    const settled = await result;
+    if (outcome === 'success') {
+      expect(settled).toBeInstanceOf(Response);
+    } else if (outcome === 'abort' || outcome === 'pre-abort') {
+      expect(settled).toBe(reason);
+    } else {
+      expect(UserFriendlyError.fromAny(settled).is('NETWORK_ERROR')).toBe(true);
+    }
+    expect(vi.getTimerCount()).toBe(0);
+    expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0);
+    if (outcome === 'pre-abort') expect(fetchMock).not.toHaveBeenCalled();
+  }
+);
+
+test('downloads only through registered source-scoped V1 entries', async () => {
+  const storage = createStorage();
+  await expect(storage.get('blob-key')).rejects.toThrow(
+    'Blob source context is required'
+  );
+  const first = {
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId: 'doc-1',
+  };
+  const second = {
+    type: 'history' as const,
+    workspaceId: 'workspace-1',
+    docId: 'doc-2',
+    timestampMs: 1720000000000,
+  };
+  const fetchMock = vi
+    .fn()
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [
+            { key: 'blob-key', mime: 'text/plain', size: 9, source: first },
+          ],
+        }),
+        {
+          status: 200,
+          headers: { 'content-type': 'application/json' },
+        }
+      )
+    )
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [
+            { key: 'blob-key', mime: 'text/plain', size: 9, source: second },
+          ],
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          status: 403,
+          code: 'ACTION_FORBIDDEN',
+          type: 'ACTION_FORBIDDEN',
+          name: 'DOC_ACTION_DENIED',
+          message: 'denied',
+        }),
+        { status: 403, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(
+      new Response('blob-data', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    )
+    .mockResolvedValueOnce(
+      new Response('blob-data', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    );
+  vi.stubGlobal('fetch', fetchMock);
+
+  await storage.registerSource(first);
+  await storage.registerSource(second);
+  const blob = await storage.get('blob-key');
+  const retried = await storage.get('blob-key');
+
+  expect(blob?.data).toEqual(new TextEncoder().encode('blob-data'));
+  expect(retried?.data).toEqual(new TextEncoder().encode('blob-data'));
+  const urls = fetchMock.mock.calls.map(call => call[0]?.toString());
+  expect(urls).toMatchInlineSnapshot(`
+    [
+      "https://example.com/api/workspaces/workspace-1/blob-manifest/v1?sourceType=currentDoc&docId=doc-1",
+      "https://example.com/api/workspaces/workspace-1/blob-manifest/v1?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+      "https://example.com/api/workspaces/workspace-1/blobs/v1/blob-key?sourceType=currentDoc&docId=doc-1",
+      "https://example.com/api/workspaces/workspace-1/blobs/v1/blob-key?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+      "https://example.com/api/workspaces/workspace-1/blobs/v1/blob-key?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+    ]
+  `);
+  expect(urls.some(url => url?.includes('redirect=manual'))).toBe(false);
+  await Promise.all([
+    storage.unregisterSource(first),
+    storage.unregisterSource(second),
+  ]);
+  await expect(storage.get('blob-key', undefined, second)).rejects.toThrow(
+    'Blob source context is required'
+  );
+  expect(fetchMock).toHaveBeenCalledTimes(5);
+
+  fetchMock
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [
+            { key: 'failed-blob', mime: 'text/plain', size: 1, source: second },
+          ],
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(new Response('', { status: 500 }));
+  await expect(
+    (async () => {
+      await storage.registerSource(second);
+      try {
+        return await storage.get('failed-blob', undefined, second);
+      } finally {
+        await storage.unregisterSource(second);
+      }
+    })()
+  ).rejects.toBeDefined();
+  await expect(storage.get('failed-blob', undefined, second)).rejects.toThrow(
+    'Blob source context is required'
+  );
+  expect(fetchMock.mock.calls.slice(-2).map(call => call[0]?.toString()))
+    .toMatchInlineSnapshot(`
+      [
+        "https://example.com/api/workspaces/workspace-1/blob-manifest/v1?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+        "https://example.com/api/workspaces/workspace-1/blobs/v1/failed-blob?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+      ]
+    `);
+
+  fetchMock
+    .mockResolvedValueOnce(new Response('', { status: 500 }))
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [
+            {
+              key: 'recovered-blob',
+              mime: 'text/plain',
+              size: 9,
+              source: second,
+            },
+          ],
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(
+      new Response('recovered', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    );
+  await expect(storage.registerSource(second)).rejects.toBeDefined();
+  const recovered = await storage.get('recovered-blob', undefined, second);
+  expect({
+    data: recovered && new TextDecoder().decode(recovered.data),
+    urls: fetchMock.mock.calls.slice(-3).map(call => call[0]?.toString()),
+  }).toMatchInlineSnapshot(`
+    {
+      "data": "recovered",
+      "urls": [
+        "https://example.com/api/workspaces/workspace-1/blob-manifest/v1?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+        "https://example.com/api/workspaces/workspace-1/blob-manifest/v1?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+        "https://example.com/api/workspaces/workspace-1/blobs/v1/recovered-blob?sourceType=history&docId=doc-2&timestampMs=1720000000000",
+      ],
+    }
+  `);
+  await storage.unregisterSource(second);
+
+  fetchMock.mockImplementation(
+    async () =>
+      new Response(JSON.stringify({ version: 1, entries: [] }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      })
+  );
+  await storage.registerSource(first);
+  await expect(storage.get('absent')).resolves.toBeNull();
+  await expect(storage.get('absent', undefined, first)).resolves.toBeNull();
+  await expect(storage.get('absent', undefined, second)).rejects.toThrow(
+    'Blob source context is required'
+  );
+  fetchMock.mockRejectedValueOnce(new Error('manifest offline'));
+  await expect(storage.get('absent', undefined, first)).rejects.toThrow(
+    'manifest offline'
+  );
+  const canceled = new AbortController();
+  canceled.abort(new DOMException('canceled', 'AbortError'));
+  const requests = fetchMock.mock.calls.length;
+  await expect(storage.get('absent', canceled.signal, first)).rejects.toBe(
+    canceled.signal.reason
+  );
+  expect(fetchMock).toHaveBeenCalledTimes(requests);
+  await storage.unregisterSource(first);
+});
+
+test('keeps an unregistered source denied across an in-flight download', async () => {
+  const storage = createStorage();
+  const source = {
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId: 'doc-1',
+  };
+  let releaseDownload!: (response: Response) => void;
+  const download = new Promise<Response>(resolve => {
+    releaseDownload = resolve;
+  });
+  const fetchMock = vi
+    .fn()
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [{ key: 'stale', mime: 'text/plain', size: 1, source }],
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockReturnValueOnce(download);
+  vi.stubGlobal('fetch', fetchMock);
+
+  await storage.registerSource(source);
+  const reading = storage.get('stale');
+  await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
+  await storage.unregisterSource(source);
+  releaseDownload(
+    new Response('stale-bytes', {
+      status: 200,
+      headers: { 'content-type': 'text/plain' },
+    })
+  );
+  await expect(reading).resolves.toBeNull();
+  await expect(storage.get('stale')).rejects.toThrow(
+    'Blob source context is required'
+  );
+  expect(fetchMock).toHaveBeenCalledTimes(2);
+
+  let releaseRefresh!: (response: Response) => void;
+  const refreshingManifest = new Promise<Response>(resolve => {
+    releaseRefresh = resolve;
+  });
+  fetchMock
+    .mockReturnValueOnce(refreshingManifest)
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [{ key: 'fresh-key', mime: 'text/plain', size: 5, source }],
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(
+      new Response('fresh', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    );
+  const refresh = storage.registerSource(source);
+  const dirtyRefresh = storage.registerSource(source);
+  releaseRefresh(
+    new Response(
+      JSON.stringify({
+        version: 1,
+        entries: [{ key: 'stale', mime: 'text/plain', size: 1, source }],
+      }),
+      { status: 200, headers: { 'content-type': 'application/json' } }
+    )
+  );
+  await Promise.all([refresh, dirtyRefresh]);
+  await expect(storage.get('fresh-key')).resolves.toMatchObject({
+    data: new TextEncoder().encode('fresh'),
+  });
+  expect(fetchMock).toHaveBeenCalledTimes(5);
+});
+
+test('pages readable workspace sources without workspace inventory fallback', async () => {
+  const storage = createStorage();
+  const source = (docId: string) => ({
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId,
+  });
+  const fetchMock = vi
+    .fn()
+    .mockResolvedValueOnce(
+      new Response(JSON.stringify({ version: 1, entries: [] }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      })
+    )
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [
+            { key: 'one', mime: 'text/plain', size: 1, source: source('a') },
+          ],
+          nextCursor: 'a',
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(
+      new Response('1', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    )
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [
+            { key: 'two', mime: 'text/plain', size: 1, source: source('b') },
+          ],
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    );
+  vi.stubGlobal('fetch', fetchMock);
+
+  await storage.registerSource(source('a'));
+
+  const entries = storage.readableSources()[Symbol.asyncIterator]();
+  await expect(entries.next()).resolves.toMatchObject({
+    value: { key: 'one' },
+    done: false,
+  });
+  await storage.unregisterSource(source('a'));
+  expect(fetchMock).toHaveBeenCalledTimes(2);
+  await expect(
+    storage.get('one', undefined, source('a'))
+  ).resolves.toMatchObject({ key: 'one' });
+  await expect(entries.next()).resolves.toMatchObject({
+    value: { key: 'two' },
+    done: false,
+  });
+  expect(fetchMock).toHaveBeenCalledTimes(4);
+  await expect(entries.next()).resolves.toEqual({
+    value: undefined,
+    done: true,
+  });
+  expect(fetchMock.mock.calls[3]?.[0]?.toString()).toContain('cursor=a');
+  await expect(storage.get('two')).rejects.toThrow(
+    'Blob source context is required'
+  );
+  await expect(storage.list()).rejects.toThrow(
+    'Workspace blob inventory is unavailable'
+  );
+});
+
+test('uses the managed workspace inventory only through its typed query', async () => {
+  const storage = createStorage();
+  const gqlMock = vi.fn(async ({ query }) => {
+    expect(query).toBe(listBlobsQuery);
+    return {
+      workspace: {
+        blobs: [
+          {
+            key: 'managed',
+            mime: 'text/plain',
+            size: 7,
+            createdAt: '2026-08-29T00:00:00.000Z',
+          },
+        ],
+      },
+    };
+  });
+  vi.spyOn(storage.connection, 'gql').mockImplementation(
+    gqlMock as typeof storage.connection.gql
+  );
+
+  expect(await storage.listManageable()).toEqual([
+    {
+      key: 'managed',
+      mime: 'text/plain',
+      size: 7,
+      createdAt: new Date('2026-08-29T00:00:00.000Z'),
+    },
+  ]);
+  await expect(storage.list()).rejects.toThrow(
+    'Workspace blob inventory is unavailable'
+  );
+});
+
+test('releases transient workspace sources and preserves owned registrations', async () => {
+  const storage = createStorage();
+  const source = {
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId: 'doc-1',
+  };
+  const entry = { key: 'stable', mime: 'text/plain', size: 4, source };
+  const fetchMock = vi
+    .fn()
+    .mockResolvedValueOnce(
+      new Response(JSON.stringify({ version: 1, entries: [entry] }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      })
+    )
+    .mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          version: 1,
+          entries: [{ key: 'partial', mime: 'text/plain', size: 1, source }],
+          nextCursor: 'next',
+        }),
+        { status: 200, headers: { 'content-type': 'application/json' } }
+      )
+    )
+    .mockResolvedValueOnce(new Response('', { status: 500 }))
+    .mockResolvedValueOnce(
+      new Response('keep', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    )
+    .mockResolvedValueOnce(
+      new Response(JSON.stringify({ version: 1, entries: [] }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      })
+    )
+    .mockResolvedValueOnce(
+      new Response('keep', {
+        status: 200,
+        headers: { 'content-type': 'text/plain' },
+      })
+    );
+  vi.stubGlobal('fetch', fetchMock);
+
+  await storage.registerSource(source);
+  await expect(async () => {
+    for await (const _entry of storage.readableSources()) {
+      // Consume until the next page fails.
+    }
+  }).rejects.toThrow();
+  expect((await storage.get('stable'))?.data).toEqual(
+    new TextEncoder().encode('keep')
+  );
+
+  for await (const _entry of storage.readableSources()) {
+    // A complete empty workspace generation must also release its page state.
+  }
+  expect((await storage.get('stable'))?.data).toEqual(
+    new TextEncoder().encode('keep')
+  );
+  await storage.unregisterSource(source);
+  await expect(storage.get('stable')).rejects.toThrow(
+    'Blob source context is required'
+  );
+
+  const registry = new BlobSourceRegistry();
+  const sources = ['one', 'two', 'three'].map(docId => ({
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId,
+  }));
+  const record = (source: (typeof sources)[number]) => ({
+    key: source.docId,
+    mime: 'text/plain',
+    size: 1,
+    source,
+  });
+  registry.replaceOwned(sources[0], [record(sources[0])], {});
+  registry.replaceOwned(sources[1], [record(sources[1])], {});
+  registry.replaceOwned(sources[2], [record(sources[2])], {});
+  expect(
+    ['one', 'two', 'three'].map(key =>
+      registry.candidates(key).map(candidate => candidate.source.docId)
+    )
+  ).toEqual([['one'], ['two'], ['three']]);
+  for (let index = 0; index < 256; index++) {
+    const source = {
+      type: 'currentDoc' as const,
+      workspaceId: 'workspace-1',
+      docId: `owned-${index}`,
+    };
+    registry.replaceOwned(source, [record(source)], {});
+  }
+  expect(registry.candidates('one')).toHaveLength(1);
+  registry.removeOwned(sources[1]);
+  expect(registry.candidates('two')).toEqual([]);
+  const released = new BlobSourceRegistry();
+  for (let index = 0; index < 1000; index++) {
+    const source = {
+      type: 'currentDoc' as const,
+      workspaceId: 'workspace-1',
+      docId: `removed-${index}`,
+    };
+    released.replaceOwned(source, [record(source)], {});
+    released.removeOwned(source);
+  }
+  expect(
+    ['removed-0', 'removed-999'].flatMap(key => released.candidates(key))
+  ).toEqual([]);
+
+  const pendingStorage = createStorage();
+  const pendingSources = Array.from({ length: 33 }, (_, index) => ({
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId: `pending-${index}`,
+  }));
+  const pendingFetch = vi.fn((input: RequestInfo | URL, init?: RequestInit) => {
+    if (input.toString().includes('docId=pending-32')) {
+      return Promise.resolve(
+        new Response(JSON.stringify({ version: 1, entries: [] }), {
+          status: 200,
+          headers: { 'content-type': 'application/json' },
+        })
+      );
+    }
+    return new Promise<Response>((_resolve, reject) => {
+      init?.signal?.addEventListener(
+        'abort',
+        () => reject(new DOMException('Aborted', 'AbortError')),
+        { once: true }
+      );
+    });
+  });
+  vi.stubGlobal('fetch', pendingFetch);
+  const pending = pendingSources
+    .slice(0, 32)
+    .map(source =>
+      pendingStorage.registerSource(source).catch((error: unknown) => error)
+    );
+  const dirty = pendingStorage
+    .registerSource(pendingSources[0])
+    .catch((error: unknown) => error);
+  const unregistering = pendingStorage.unregisterSource(pendingSources[0]);
+  await expect(
+    pendingStorage.registerSource(pendingSources[32])
+  ).rejects.toThrow('Blob source registration budget exceeded');
+  await unregistering;
+  await pending[0];
+  await pendingStorage.registerSource(pendingSources[32]);
+  await dirty;
+  const fetchCount = pendingFetch.mock.calls.length;
+  await expect(
+    pendingStorage.get('unregistered', undefined, pendingSources[32])
+  ).resolves.toBeNull();
+  expect(pendingFetch).toHaveBeenCalledTimes(fetchCount + 1);
+  await pendingStorage.unregisterSource(pendingSources[32]);
+  await Promise.all(
+    pendingSources
+      .slice(1, 32)
+      .map(source => pendingStorage.unregisterSource(source))
+  );
+  await Promise.all(pending);
+
+  const contended = createStorage();
+  const registered = Array.from({ length: 64 }, (_, index) => ({
+    type: 'currentDoc' as const,
+    workspaceId: 'workspace-1',
+    docId: `contended-${index}`,
+  }));
+  const releases: Array<() => void> = [];
+  let holdRefresh = false;
+  vi.stubGlobal(
+    'fetch',
+    vi.fn(async (input: RequestInfo | URL) => {
+      const url = new URL(input.toString());
+      if (url.pathname.includes('/blobs/v1/')) return new Response('found');
+      const docId = url.searchParams.get('docId');
+      if (holdRefresh) {
+        await new Promise<void>(resolve => releases.push(resolve));
+      }
+      return new Response(
+        JSON.stringify({
+          version: 1,
+          entries:
+            holdRefresh && docId === registered[63].docId
+              ? [
+                  {
+                    key: 'after-budget',
+                    mime: 'text/plain',
+                    size: 5,
+                    source: registered[63],
+                  },
+                ]
+              : [],
+        }),
+        { headers: { 'content-type': 'application/json' } }
+      );
+    })
+  );
+  for (const source of registered) await contended.registerSource(source);
+  holdRefresh = true;
+  const refreshing = registered
+    .slice(32)
+    .map(source => contended.registerSource(source));
+  expect(releases).toHaveLength(32);
+  const reading = contended.get('after-budget').then(
+    value => value,
+    error => error
+  );
+  for (const release of releases) release();
+  await Promise.all(refreshing);
+  await expect(reading).resolves.toMatchObject({ key: 'after-budget' });
+  await Promise.all(
+    registered.map(source => contended.unregisterSource(source))
+  );
+});

packages/common/nbstore/src/__tests__/cloud-doc-updates.spec.ts

@@ -0,0 +1,224 @@
+import { describe, expect, test, vi } from 'vitest';
+
+import { CloudAwarenessStorage } from '../impls/cloud/awareness';
+import { CloudDocStorage } from '../impls/cloud/doc';
+
+const base64UpdateA = 'AQID';
+const base64UpdateB = 'BAUG';
+
+class FakeSocket {
+  connected = true;
+  readonly joinedDocs = new Set<string>();
+  readonly emitted: Array<{ event: string; payload: unknown }> = [];
+  readonly handlers = new Map<string, (...args: unknown[]) => void>();
+
+  on(event: string, handler: (...args: unknown[]) => void) {
+    this.handlers.set(event, handler);
+    return this;
+  }
+
+  once(event: string, handler: (...args: unknown[]) => void) {
+    this.handlers.set(event, handler);
+    return this;
+  }
+
+  off(event: string, handler?: (...args: unknown[]) => void) {
+    if (!handler || this.handlers.get(event) === handler) {
+      this.handlers.delete(event);
+    }
+    return this;
+  }
+
+  emit(event: string, payload?: unknown) {
+    this.emitted.push({ event, payload });
+    return true;
+  }
+
+  async emitWithAck(
+    event: string,
+    payload: { docId?: string; spaces?: { docId?: string }[] }
+  ) {
+    this.emitted.push({ event, payload });
+    if (event === 'space:join-batch') {
+      for (const space of payload.spaces ?? []) {
+        if (space.docId === 'denied') {
+          return { error: { name: 'DOC_ACTION_DENIED', message: 'denied' } };
+        }
+        if (space.docId) this.joinedDocs.add(space.docId);
+      }
+    }
+    if (event === 'space:load-doc' || event === 'space:push-doc-update') {
+      expect(this.joinedDocs.has(payload.docId!)).toBe(true);
+      return { data: { missing: 'AAA=', state: 'AA==', timestamp: 1_000 } };
+    }
+    if (event === 'space:doc-lifecycle') {
+      return { data: { rootUpdate: base64UpdateA, timestamp: 1_000 } };
+    }
+    return { data: { clientId: 'client-1', success: true } };
+  }
+}
+
+describe('CloudDocStorage broadcast updates', () => {
+  test('emits updates from batch payload', () => {
+    const storage = new CloudDocStorage({
+      id: 'space-1',
+      serverBaseUrl: 'http://localhost',
+      isSelfHosted: true,
+      type: 'workspace',
+      readonlyMode: true,
+    });
+
+    (storage as any).connection.idConverter = {
+      oldIdToNewId: (id: string) => id,
+      newIdToOldId: (id: string) => id,
+    };
+
+    const received: Uint8Array[] = [];
+    storage.subscribeDocUpdate(update => {
+      received.push(update.bin);
+    });
+
+    storage.onServerUpdates({
+      spaceType: 'workspace',
+      spaceId: 'space-1',
+      docId: 'doc-1',
+      updates: [base64UpdateA, base64UpdateB],
+      timestamp: Date.now(),
+    });
+
+    expect(received).toEqual([
+      new Uint8Array([1, 2, 3]),
+      new Uint8Array([4, 5, 6]),
+    ]);
+  });
+
+  test('repairs strict invalidation through readable timestamps', async () => {
+    const storage = new CloudDocStorage({
+      id: 'space-1',
+      serverBaseUrl: 'http://localhost',
+      isSelfHosted: true,
+      type: 'workspace',
+      readonlyMode: true,
+    });
+
+    (storage as any).connection.idConverter = {
+      oldIdToNewId: (id: string) => id,
+      newIdToOldId: (id: string) => id,
+    };
+
+    const getDocTimestamps = vi
+      .spyOn(storage, 'getDocTimestamps')
+      .mockResolvedValue({ 'doc-a': new Date(1_000) });
+    const received: Array<{ docId: string; bin: Uint8Array }> = [];
+    storage.subscribeDocUpdate(update => {
+      received.push({ docId: update.docId, bin: update.bin });
+    });
+
+    storage.onServerInvalidation({
+      spaceType: 'workspace',
+      spaceId: 'space-1',
+      timestamp: 1_000,
+    });
+    storage.onServerInvalidation({
+      spaceType: 'workspace',
+      spaceId: 'space-1',
+      timestamp: 1_001,
+    });
+
+    await vi.waitFor(() => expect(received).toHaveLength(1));
+    expect(getDocTimestamps).toHaveBeenCalledOnce();
+    expect(received[0]).toMatchObject({ docId: 'doc-a' });
+    expect(received[0]?.bin).toEqual(new Uint8Array());
+  });
+
+  test('batch route joins the workspace and applies lifecycle commands', async () => {
+    vi.stubGlobal('BUILD_CONFIG', { appVersion: '0.27.5' });
+    const fakeSocket = new FakeSocket();
+    const disconnect = vi.fn();
+    const storage = new CloudDocStorage({
+      id: 'space-1',
+      serverBaseUrl: 'http://localhost',
+      isSelfHosted: true,
+      type: 'workspace',
+      readonlyMode: true,
+    });
+    const connection = storage.connection as any;
+
+    Object.defineProperty(connection, 'manager', {
+      configurable: true,
+      value: {
+        connect: () => ({ socket: fakeSocket, disconnect }),
+      },
+    });
+    const inner = await connection.doConnect();
+    connection._inner = inner;
+    await storage.getDocSnapshot('doc-1');
+    await storage.getDocDiff('doc-2');
+    await storage.getDocTimestamp('doc-3');
+    await storage.pushDocUpdate({
+      docId: 'doc-4',
+      bin: new Uint8Array([0, 0]),
+    });
+    const lifecycle = await storage.applyDocLifecycle('doc-1', 'trash');
+    await expect(storage.getDocSnapshot('denied')).rejects.toMatchObject({
+      name: 'DOC_ACTION_DENIED',
+    });
+    fakeSocket.joinedDocs.delete('doc-1');
+    await storage.getDocDiff('doc-1');
+    connection.doDisconnect(inner);
+    expect({ emitted: fakeSocket.emitted, lifecycle }).toMatchSnapshot({
+      lifecycle: {
+        rootUpdate: expect.any(Uint8Array),
+        timestamp: expect.any(Date),
+      },
+    });
+
+    expect(disconnect).toHaveBeenCalledOnce();
+    vi.unstubAllGlobals();
+  });
+
+  test('awareness joins active documents through the batch route', async () => {
+    vi.stubGlobal('BUILD_CONFIG', { appVersion: '0.27.5' });
+    const fakeSocket = new FakeSocket();
+    const storage = new CloudAwarenessStorage({
+      id: 'space-1',
+      serverBaseUrl: 'http://localhost',
+      isSelfHosted: true,
+      type: 'workspace',
+    });
+
+    Object.defineProperty(storage, 'connection', {
+      configurable: true,
+      value: {
+        status: 'connected',
+        inner: { socket: fakeSocket },
+        onStatusChanged: () => () => {},
+      },
+    });
+
+    const unsubscribeA = storage.subscribeUpdate(
+      'doc-a',
+      () => {},
+      async () => null
+    );
+    const unsubscribeB = storage.subscribeUpdate(
+      'doc-b',
+      () => {},
+      async () => null
+    );
+
+    await vi.waitFor(() => {
+      expect(
+        fakeSocket.emitted.filter(
+          ({ event: emittedEvent }) => emittedEvent === 'space:join-batch'
+        )
+      ).toHaveLength(1);
+    });
+
+    expect(fakeSocket.emitted).toMatchSnapshot();
+
+    unsubscribeA();
+    unsubscribeB();
+    vi.unstubAllGlobals();
+  });
+});

packages/common/nbstore/src/__tests__/frontend.spec.ts

@@ -0,0 +1,192 @@
+import 'fake-indexeddb/auto';
+
+import { expect, test, vitest } from 'vitest';
+import { Awareness } from 'y-protocols/awareness.js';
+import { Doc as YDoc } from 'yjs';
+
+import { AwarenessFrontend } from '../frontend/awareness';
+import { DocFrontend } from '../frontend/doc';
+import { BroadcastChannelAwarenessStorage } from '../impls/broadcast-channel/awareness';
+import { IndexedDBDocStorage } from '../impls/idb';
+import { AwarenessSyncImpl } from '../sync/awareness';
+import { DocSyncImpl } from '../sync/doc';
+import { expectYjsEqual } from './utils';
+
+test('doc', async () => {
+  const doc1 = new YDoc({
+    guid: 'test-doc',
+  });
+  doc1.getMap('test').set('hello', 'world');
+
+  const docStorage = new IndexedDBDocStorage({
+    id: 'ws1',
+    flavour: 'a',
+    type: 'workspace',
+  });
+
+  docStorage.connection.connect();
+
+  await docStorage.connection.waitForConnected();
+
+  const frontend1 = new DocFrontend(docStorage, DocSyncImpl.dummy);
+  frontend1.start();
+  frontend1.connectDoc(doc1);
+  await vitest.waitFor(async () => {
+    const doc = await docStorage.getDoc('test-doc');
+    expectYjsEqual(doc!.bin, {
+      test: {
+        hello: 'world',
+      },
+    });
+  });
+
+  const doc2 = new YDoc({
+    guid: 'test-doc',
+  });
+  const frontend2 = new DocFrontend(docStorage, DocSyncImpl.dummy);
+  frontend2.start();
+  frontend2.connectDoc(doc2);
+
+  await vitest.waitFor(async () => {
+    expectYjsEqual(doc2, {
+      test: {
+        hello: 'world',
+      },
+    });
+  });
+});
+
+test('awareness', async () => {
+  const storage1 = new BroadcastChannelAwarenessStorage({
+    id: 'ws1:a',
+  });
+
+  const storage2 = new BroadcastChannelAwarenessStorage({
+    id: 'ws1:b',
+  });
+
+  storage1.connection.connect();
+  storage2.connection.connect();
+
+  await storage1.connection.waitForConnected();
+  await storage2.connection.waitForConnected();
+
+  // peer a
+  const docA = new YDoc({ guid: 'test-doc' });
+  docA.clientID = 1;
+  const awarenessA = new Awareness(docA);
+
+  // peer b
+  const docB = new YDoc({ guid: 'test-doc' });
+  docB.clientID = 2;
+  const awarenessB = new Awareness(docB);
+
+  // peer c
+  const docC = new YDoc({ guid: 'test-doc' });
+  docC.clientID = 3;
+  const awarenessC = new Awareness(docC);
+
+  {
+    const sync = new AwarenessSyncImpl({
+      local: storage1,
+      remotes: {
+        b: storage2,
+      },
+    });
+    const frontend = new AwarenessFrontend(sync);
+    frontend.connectAwareness(awarenessA);
+    frontend.connectAwareness(awarenessB);
+  }
+  {
+    const sync = new AwarenessSyncImpl({
+      local: storage2,
+      remotes: {
+        a: storage1,
+      },
+    });
+    const frontend = new AwarenessFrontend(sync);
+    frontend.connectAwareness(awarenessC);
+  }
+
+  awarenessA.setLocalState({
+    hello: 'world',
+  });
+
+  await vitest.waitFor(() => {
+    expect(awarenessB.getStates().get(1)).toEqual({
+      hello: 'world',
+    });
+    expect(awarenessC.getStates().get(1)).toEqual({
+      hello: 'world',
+    });
+  });
+
+  awarenessB.setLocalState({
+    foo: 'bar',
+  });
+
+  await vitest.waitFor(() => {
+    expect(awarenessA.getStates().get(2)).toEqual({
+      foo: 'bar',
+    });
+    expect(awarenessC.getStates().get(2)).toEqual({
+      foo: 'bar',
+    });
+  });
+
+  awarenessC.setLocalState({
+    baz: 'qux',
+  });
+
+  await vitest.waitFor(() => {
+    expect(awarenessA.getStates().get(3)).toEqual({
+      baz: 'qux',
+    });
+    expect(awarenessB.getStates().get(3)).toEqual({
+      baz: 'qux',
+    });
+  });
+});
+
+test('doc priority requests accumulate', async () => {
+  const docStorage = new IndexedDBDocStorage({
+    id: 'ws-priority',
+    flavour: 'a',
+    type: 'workspace',
+  });
+
+  docStorage.connection.connect();
+
+  await docStorage.connection.waitForConnected();
+
+  const frontend = new DocFrontend(docStorage, DocSyncImpl.dummy);
+
+  // two holders prioritize the same doc, then one of them goes away
+  frontend.addPriority('high', 10);
+  const releaseSecondHolder = frontend.addPriority('high', 10);
+  releaseSecondHolder();
+
+  frontend.addPriority('low', 5);
+
+  const loadOrder: string[] = [];
+  vitest.spyOn(docStorage, 'getDoc').mockImplementation(async docId => {
+    loadOrder.push(docId);
+    return null;
+  });
+
+  // both load jobs are queued before the main loop starts, so the queue
+  // priority alone decides which doc is loaded first
+  frontend.connectDoc(new YDoc({ guid: 'low' }));
+  frontend.connectDoc(new YDoc({ guid: 'high' }));
+
+  frontend.start();
+
+  await vitest.waitFor(
+    () => {
+      expect(loadOrder).toEqual(['high', 'low']);
+    },
+    { timeout: 2000 }
+  );
+
+  frontend.stop();
+});

packages/common/nbstore/src/__tests__/sync.spec.ts

@@ -0,0 +1,988 @@
+import 'fake-indexeddb/auto';
+
+import * as reader from '@affine/reader';
+import { firstValueFrom, NEVER } from 'rxjs';
+import { afterEach, expect, test, vi } from 'vitest';
+import { Doc as YDoc, encodeStateAsUpdate } from 'yjs';
+
+import { DummyConnection } from '../connection';
+import {
+  IndexedDBBlobStorage,
+  IndexedDBBlobSyncStorage,
+  IndexedDBDocStorage,
+  IndexedDBDocSyncStorage,
+} from '../impls/idb';
+import {
+  type AggregateOptions,
+  type AggregateResult,
+  type CrawlResult,
+  type DocClock,
+  type DocClocks,
+  type DocDiff,
+  type DocIndexedClock,
+  type DocRecord,
+  type DocStorage,
+  type DocUpdate,
+  type IndexerDocument,
+  type IndexerSchema,
+  IndexerStorageBase,
+  IndexerSyncStorageBase,
+  type Query,
+  type SearchOptions,
+  type SearchResult,
+  SpaceStorage,
+} from '../storage';
+import { Sync } from '../sync';
+import { BlobSyncPeer } from '../sync/blob/peer';
+import { DocSyncPeer } from '../sync/doc/peer';
+import { IndexerSyncImpl } from '../sync/indexer';
+import { expectYjsEqual } from './utils';
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+function deferred<T = void>() {
+  let resolve!: (value: T | PromiseLike<T>) => void;
+  let reject!: (reason?: unknown) => void;
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return { promise, resolve, reject };
+}
+
+class TestDocStorage implements DocStorage {
+  readonly storageType = 'doc' as const;
+  readonly connection = new DummyConnection();
+  readonly isReadonly = false;
+  private readonly subscribers = new Set<
+    (update: DocRecord, origin?: string) => void
+  >();
+
+  constructor(
+    readonly spaceId: string,
+    private readonly timestamps: Map<string, Date>,
+    private readonly crawlDocDataImpl: (
+      docId: string
+    ) => Promise<CrawlResult | null>
+  ) {}
+
+  async getDoc(_docId: string): Promise<DocRecord | null> {
+    return null;
+  }
+
+  async getDocDiff(
+    _docId: string,
+    _state?: Uint8Array
+  ): Promise<DocDiff | null> {
+    return null;
+  }
+
+  async pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock> {
+    const timestamp = this.timestamps.get(update.docId) ?? new Date();
+    const record = { ...update, timestamp };
+    this.timestamps.set(update.docId, timestamp);
+    for (const subscriber of this.subscribers) {
+      subscriber(record, origin);
+    }
+    return { docId: update.docId, timestamp };
+  }
+
+  async getDocTimestamp(docId: string): Promise<DocClock | null> {
+    const timestamp = this.timestamps.get(docId);
+    return timestamp ? { docId, timestamp } : null;
+  }
+
+  async getDocTimestamps(): Promise<DocClocks> {
+    return Object.fromEntries(this.timestamps);
+  }
+
+  async deleteDoc(docId: string): Promise<void> {
+    this.timestamps.delete(docId);
+  }
+
+  subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
+    this.subscribers.add(callback);
+    return () => {
+      this.subscribers.delete(callback);
+    };
+  }
+
+  async crawlDocData(docId: string): Promise<CrawlResult | null> {
+    return this.crawlDocDataImpl(docId);
+  }
+}
+
+class TimestampBlindDocStorage extends IndexedDBDocStorage {
+  override async getDocTimestamps(): Promise<DocClocks> {
+    return {};
+  }
+}
+
+class PermissionDeniedRemoteDocStorage implements DocStorage {
+  readonly storageType = 'doc' as const;
+  readonly connection = new DummyConnection();
+  readonly isReadonly = false;
+  pushCount = 0;
+
+  constructor(readonly spaceId: string) {}
+
+  async getDoc(_docId: string): Promise<DocRecord | null> {
+    return null;
+  }
+
+  async getDocDiff(
+    _docId: string,
+    _state?: Uint8Array
+  ): Promise<DocDiff | null> {
+    return null;
+  }
+
+  async pushDocUpdate(_update: DocUpdate): Promise<DocClock> {
+    this.pushCount++;
+    const error = new Error('No permission to update doc');
+    error.name = 'DOC_ACTION_DENIED';
+    throw error;
+  }
+
+  async getDocTimestamp(_docId: string): Promise<DocClock | null> {
+    return null;
+  }
+
+  async getDocTimestamps(): Promise<DocClocks> {
+    return {};
+  }
+
+  async deleteDoc(_docId: string): Promise<void> {
+    return;
+  }
+
+  subscribeDocUpdate(_callback: (update: DocRecord, origin?: string) => void) {
+    return () => {};
+  }
+}
+
+class PermissionDeniedConnection extends DummyConnection {
+  waitCount = 0;
+
+  override async waitForConnected(_signal?: AbortSignal): Promise<void> {
+    this.waitCount++;
+    const error = new Error('No permission to access space');
+    error.name = 'SPACE_ACCESS_DENIED';
+    throw error;
+  }
+}
+
+class PermissionDeniedConnectionDocStorage extends PermissionDeniedRemoteDocStorage {
+  override readonly connection = new PermissionDeniedConnection();
+}
+
+class TrackingIndexerStorage extends IndexerStorageBase {
+  override readonly connection = new DummyConnection();
+  override readonly isReadonly = false;
+
+  constructor(
+    private readonly calls: string[],
+    override readonly recommendRefreshInterval: number
+  ) {
+    super();
+  }
+
+  override async search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Promise<SearchResult<T, O>> {
+    return {
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      nodes: [],
+    } as SearchResult<T, O>;
+  }
+
+  override async aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Promise<AggregateResult<T, O>> {
+    return {
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      buckets: [],
+    } as AggregateResult<T, O>;
+  }
+
+  override search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O) {
+    return NEVER;
+  }
+
+  override aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(_table: T, _query: Query<T>, _field: keyof IndexerSchema[T], _options?: O) {
+    return NEVER;
+  }
+
+  override async deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    _query: Query<T>
+  ): Promise<void> {
+    this.calls.push(`deleteByQuery:${String(table)}`);
+  }
+
+  override async insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void> {
+    this.calls.push(`insert:${String(table)}:${document.id}`);
+  }
+
+  override async delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void> {
+    this.calls.push(`delete:${String(table)}:${id}`);
+  }
+
+  override async update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void> {
+    this.calls.push(`update:${String(table)}:${document.id}`);
+  }
+
+  override async refresh<T extends keyof IndexerSchema>(
+    _table: T
+  ): Promise<void> {
+    return;
+  }
+
+  override async refreshIfNeed(): Promise<void> {
+    this.calls.push('refresh');
+  }
+
+  override async indexVersion(): Promise<number> {
+    return 1;
+  }
+}
+
+class TrackingIndexerSyncStorage extends IndexerSyncStorageBase {
+  override readonly connection = new DummyConnection();
+  private readonly clocks = new Map<string, DocIndexedClock>();
+
+  constructor(private readonly calls: string[]) {
+    super();
+  }
+
+  override async getDocIndexedClock(
+    docId: string
+  ): Promise<DocIndexedClock | null> {
+    return this.clocks.get(docId) ?? null;
+  }
+
+  override async setDocIndexedClock(clock: DocIndexedClock): Promise<void> {
+    this.calls.push(`setClock:${clock.docId}`);
+    this.clocks.set(clock.docId, clock);
+  }
+
+  override async clearDocIndexedClock(docId: string): Promise<void> {
+    this.calls.push(`clearClock:${docId}`);
+    this.clocks.delete(docId);
+  }
+}
+
+test('doc', async () => {
+  const doc = new YDoc();
+  doc.getMap('test').set('hello', 'world');
+  const update = encodeStateAsUpdate(doc);
+
+  const peerADoc = new IndexedDBDocStorage({
+    id: 'ws1',
+    flavour: 'a',
+    type: 'workspace',
+  });
+
+  const peerASync = new IndexedDBDocSyncStorage({
+    id: 'ws1',
+    flavour: 'a',
+    type: 'workspace',
+  });
+
+  const peerBDoc = new TimestampBlindDocStorage({
+    id: 'ws1',
+    flavour: 'b',
+    type: 'workspace',
+  });
+  const peerCDoc = new IndexedDBDocStorage({
+    id: 'ws1',
+    flavour: 'c',
+    type: 'workspace',
+  });
+
+  const peerA = new SpaceStorage({
+    doc: peerADoc,
+    docSync: peerASync,
+  });
+  const peerB = new SpaceStorage({
+    doc: peerBDoc,
+  });
+  const peerC = new SpaceStorage({
+    doc: peerCDoc,
+  });
+
+  peerA.connect();
+  peerB.connect();
+  peerC.connect();
+
+  await peerA.waitForConnected();
+  await peerB.waitForConnected();
+  await peerC.waitForConnected();
+
+  await peerA.get('doc').pushDocUpdate({
+    docId: 'doc1',
+    bin: update,
+  });
+  const prioritizedDocId = 'prioritized-doc';
+  const localPrioritizedDoc = new YDoc();
+  localPrioritizedDoc.getMap('test').set('local', true);
+  const localPrioritizedClock = await peerA.get('doc').pushDocUpdate({
+    docId: prioritizedDocId,
+    bin: encodeStateAsUpdate(localPrioritizedDoc),
+  });
+  await peerASync.setPeerPushedClock('b', localPrioritizedClock);
+  const remotePrioritizedDoc = new YDoc();
+  remotePrioritizedDoc.getMap('test').set('remote', true);
+  await peerB.get('doc').pushDocUpdate({
+    docId: prioritizedDocId,
+    bin: encodeStateAsUpdate(remotePrioritizedDoc),
+  });
+  const rootDoc = new YDoc();
+  rootDoc.getMap('meta').set('name', 'Self-host workspace');
+  await peerB.get('doc').pushDocUpdate({
+    docId: 'ws1',
+    bin: encodeStateAsUpdate(rootDoc),
+  });
+
+  const sync = new Sync({
+    local: peerA,
+    remotes: {
+      b: peerB,
+      c: peerC,
+    },
+  });
+  const removeRootPriority = sync.doc.addPriority('ws1', 100);
+  const removeForegroundPriority = sync.doc.addPriority('doc1', 200);
+  const remoteDiff = vi.spyOn(peerBDoc, 'getDocDiff');
+  expect(await firstValueFrom(sync.doc.docState$('doc1'))).toMatchObject({
+    synced: false,
+  });
+  sync.start();
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  {
+    expect(remoteDiff.mock.calls[0]?.[0]).toBe('ws1');
+    const b = await peerB.get('doc').getDoc('doc1');
+    expectYjsEqual(b!.bin, {
+      test: {
+        hello: 'world',
+      },
+    });
+
+    const c = await peerC.get('doc').getDoc('doc1');
+    expectYjsEqual(c!.bin, {
+      test: {
+        hello: 'world',
+      },
+    });
+
+    const root = await peerA.get('doc').getDoc('ws1');
+    expectYjsEqual(root!.bin, {
+      meta: {
+        name: 'Self-host workspace',
+      },
+    });
+
+    const prioritized = await peerA.get('doc').getDoc(prioritizedDocId);
+    expectYjsEqual(prioritized!.bin, {
+      test: {
+        local: true,
+      },
+    });
+  }
+
+  const removeDocPriority = sync.doc.addPriority(prioritizedDocId, 100);
+  await vi.waitFor(async () => {
+    const prioritized = await peerA.get('doc').getDoc(prioritizedDocId);
+    expectYjsEqual(prioritized!.bin, {
+      test: {
+        local: true,
+        remote: true,
+      },
+    });
+  });
+
+  doc.getMap('test').set('foo', 'bar');
+  const update2 = encodeStateAsUpdate(doc);
+  await peerC.get('doc').pushDocUpdate({
+    docId: 'doc1',
+    bin: update2,
+  });
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  {
+    const a = await peerA.get('doc').getDoc('doc1');
+    expectYjsEqual(a!.bin, {
+      test: {
+        hello: 'world',
+        foo: 'bar',
+      },
+    });
+
+    const c = await peerC.get('doc').getDoc('doc1');
+    expectYjsEqual(c!.bin, {
+      test: {
+        hello: 'world',
+        foo: 'bar',
+      },
+    });
+  }
+
+  removeDocPriority();
+  removeForegroundPriority();
+  removeRootPriority();
+  sync.stop();
+  peerA.disconnect();
+  peerB.disconnect();
+  peerC.disconnect();
+});
+
+test('blob', async () => {
+  const a = new IndexedDBBlobStorage({
+    id: 'ws1',
+    flavour: 'a',
+    type: 'workspace',
+  });
+
+  const b = new IndexedDBBlobStorage({
+    id: 'ws1',
+    flavour: 'b',
+    type: 'workspace',
+  });
+
+  const c = new IndexedDBBlobStorage({
+    id: 'ws1',
+    flavour: 'c',
+    type: 'workspace',
+  });
+
+  const blobSync = new IndexedDBBlobSyncStorage({
+    id: 'ws1',
+    flavour: 'a',
+    type: 'workspace',
+  });
+
+  const peerA = new SpaceStorage({
+    blob: a,
+    blobSync,
+  });
+  const peerB = new SpaceStorage({
+    blob: b,
+  });
+  const peerC = new SpaceStorage({
+    blob: c,
+  });
+
+  peerA.connect();
+  peerB.connect();
+  peerC.connect();
+
+  await peerA.waitForConnected();
+  await peerB.waitForConnected();
+  await peerC.waitForConnected();
+
+  await a.set({
+    key: 'test',
+    data: new Uint8Array([1, 2, 3, 4]),
+    mime: 'text/plain',
+    createdAt: new Date(100),
+  });
+
+  await c.set({
+    key: 'test2',
+    data: new Uint8Array([4, 3, 2, 1]),
+    mime: 'text/plain',
+    createdAt: new Date(100),
+  });
+
+  const sync = new Sync({
+    local: peerA,
+    remotes: {
+      b: peerB,
+      c: peerC,
+    },
+  });
+  sync.start();
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  {
+    const a = await peerA.get('blob').get('test');
+    expect(a).not.toBeNull();
+    expect(a?.data).toEqual(new Uint8Array([1, 2, 3, 4]));
+  }
+
+  {
+    const b = await peerB.get('blob').get('test');
+    expect(b).not.toBeNull();
+    expect(b?.data).toEqual(new Uint8Array([1, 2, 3, 4]));
+  }
+
+  {
+    const c = await peerC.get('blob').get('test2');
+    expect(c).not.toBeNull();
+    expect(c?.data).toEqual(new Uint8Array([4, 3, 2, 1]));
+  }
+  sync.stop();
+
+  const localReads = vi.spyOn(a, 'get');
+  const localLists = vi.spyOn(a, 'list');
+  const remoteReads = vi.spyOn(c, 'get');
+  await new BlobSyncPeer('c', a, c, blobSync).fullDownload();
+  expect(localReads).not.toHaveBeenCalled();
+  expect(localLists).toHaveBeenCalledTimes(1);
+  expect(remoteReads.mock.calls.map(([key]) => key)).toEqual(['test2']);
+  remoteReads.mockRestore();
+  localLists.mockClear();
+
+  for (const key of ['fresh', 'retry']) {
+    await c.set({ key, data: new Uint8Array([1, 2, 3]), mime: 'text/plain' });
+  }
+  const entries = await c.list();
+  const scoped = Object.assign(c, {
+    registerSource: vi.fn(),
+    unregisterSource: vi.fn(),
+    async *readableSources() {
+      for (const entry of entries) {
+        for (const docId of ['first', 'second']) {
+          yield {
+            ...entry,
+            source: { type: 'currentDoc' as const, workspaceId: 'ws1', docId },
+          };
+        }
+      }
+    },
+  });
+  const get = c.get.bind(c);
+  let retryAttempts = 0;
+  const reads = vi.spyOn(c, 'get').mockImplementation(async key => {
+    if (key === 'retry' && retryAttempts++ === 0)
+      throw new Error('source denied');
+    return get(key);
+  });
+  const inventory = vi
+    .spyOn(c, 'list')
+    .mockRejectedValue(new Error('cloud inventory forbidden'));
+  await new BlobSyncPeer('scoped', a, scoped, blobSync).fullDownload();
+  expect(localReads).not.toHaveBeenCalled();
+  expect(localLists).toHaveBeenCalledTimes(1);
+  expect(inventory).not.toHaveBeenCalled();
+  expect(reads.mock.calls.map(([key]) => key)).toEqual([
+    'fresh',
+    'retry',
+    'retry',
+  ]);
+  localReads.mockRestore();
+  expect((await a.get('fresh'))?.data).toEqual(new Uint8Array([1, 2, 3]));
+  expect((await a.get('retry'))?.data).toEqual(new Uint8Array([1, 2, 3]));
+});
+
+test('doc sync peer stops retrying a doc when remote denies permission', async () => {
+  const local = new IndexedDBDocStorage({
+    id: 'ws-denied',
+    flavour: 'local-denied',
+    type: 'workspace',
+  });
+  const syncMetadata = new IndexedDBDocSyncStorage({
+    id: 'ws-denied',
+    flavour: 'local-denied',
+    type: 'workspace',
+  });
+  const remote = new PermissionDeniedRemoteDocStorage('ws-denied');
+  const peer = new DocSyncPeer('remote-denied', local, syncMetadata, remote);
+  const abort = new AbortController();
+
+  local.connection.connect();
+  syncMetadata.connection.connect();
+  await local.connection.waitForConnected();
+  await syncMetadata.connection.waitForConnected();
+
+  const doc = new YDoc();
+  doc.getMap('test').set('hello', 'world');
+  await local.pushDocUpdate({
+    docId: 'doc-denied',
+    bin: encodeStateAsUpdate(doc),
+  });
+
+  try {
+    void peer.mainLoop(abort.signal);
+
+    await vi.waitFor(() => {
+      expect(remote.pushCount).toBe(1);
+    });
+
+    await vi.waitFor(() => {
+      let state:
+        | {
+            syncing: boolean;
+            synced: boolean;
+            retrying: boolean;
+            errorMessage: string | null;
+          }
+        | undefined;
+      const dispose = peer.docState$('doc-denied').subscribe(next => {
+        state = next;
+      });
+      dispose.unsubscribe();
+
+      expect(state).toMatchObject({
+        syncing: false,
+        synced: false,
+        retrying: false,
+        errorMessage: expect.stringContaining('No permission'),
+      });
+    });
+
+    await vi.waitFor(() => {
+      let state:
+        | {
+            synced: boolean;
+            errorMessage: string | null;
+          }
+        | undefined;
+      const dispose = peer.peerState$.subscribe(next => {
+        state = next;
+      });
+      dispose.unsubscribe();
+
+      expect(state).toMatchObject({
+        synced: false,
+        errorMessage: expect.stringContaining('No permission'),
+      });
+    });
+
+    await new Promise(resolve => setTimeout(resolve, 1200));
+    expect(remote.pushCount).toBe(1);
+  } finally {
+    abort.abort();
+    local.connection.disconnect();
+    syncMetadata.connection.disconnect();
+  }
+});
+
+test('doc sync peer stops retrying when remote connection denies permission', async () => {
+  const local = new IndexedDBDocStorage({
+    id: 'ws-connection-denied',
+    flavour: 'local-connection-denied',
+    type: 'workspace',
+  });
+  const syncMetadata = new IndexedDBDocSyncStorage({
+    id: 'ws-connection-denied',
+    flavour: 'local-connection-denied',
+    type: 'workspace',
+  });
+  const remote = new PermissionDeniedConnectionDocStorage(
+    'ws-connection-denied'
+  );
+  const peer = new DocSyncPeer(
+    'remote-connection-denied',
+    local,
+    syncMetadata,
+    remote
+  );
+  const abort = new AbortController();
+
+  local.connection.connect();
+  syncMetadata.connection.connect();
+  await local.connection.waitForConnected();
+  await syncMetadata.connection.waitForConnected();
+
+  try {
+    void peer.mainLoop(abort.signal);
+
+    await vi.waitFor(() => {
+      expect(remote.connection.waitCount).toBe(1);
+    });
+
+    await vi.waitFor(() => {
+      let state:
+        | {
+            retrying: boolean;
+            errorMessage: string | null;
+          }
+        | undefined;
+      const dispose = peer.peerState$.subscribe(next => {
+        state = next;
+      });
+      dispose.unsubscribe();
+
+      expect(state).toMatchObject({
+        retrying: false,
+        errorMessage: expect.stringContaining('No permission'),
+      });
+    });
+
+    await new Promise(resolve => setTimeout(resolve, 1200));
+    expect(remote.connection.waitCount).toBe(1);
+  } finally {
+    abort.abort();
+    local.connection.disconnect();
+    syncMetadata.connection.disconnect();
+  }
+});
+
+test('doc sync peer resolves on terminal permission error without abort signal', async () => {
+  const local = new IndexedDBDocStorage({
+    id: 'ws-connection-denied-no-signal',
+    flavour: 'local-connection-denied-no-signal',
+    type: 'workspace',
+  });
+  const syncMetadata = new IndexedDBDocSyncStorage({
+    id: 'ws-connection-denied-no-signal',
+    flavour: 'local-connection-denied-no-signal',
+    type: 'workspace',
+  });
+  const remote = new PermissionDeniedConnectionDocStorage(
+    'ws-connection-denied-no-signal'
+  );
+  const peer = new DocSyncPeer(
+    'remote-connection-denied-no-signal',
+    local,
+    syncMetadata,
+    remote
+  );
+
+  local.connection.connect();
+  syncMetadata.connection.connect();
+  await local.connection.waitForConnected();
+  await syncMetadata.connection.waitForConnected();
+
+  try {
+    await expect(peer.mainLoop()).resolves.toBeUndefined();
+    expect(remote.connection.waitCount).toBe(1);
+
+    let state:
+      | {
+          retrying: boolean;
+          errorMessage: string | null;
+        }
+      | undefined;
+    const dispose = peer.peerState$.subscribe(next => {
+      state = next;
+    });
+    dispose.unsubscribe();
+
+    expect(state).toMatchObject({
+      retrying: false,
+      errorMessage: expect.stringContaining('No permission'),
+    });
+  } finally {
+    local.connection.disconnect();
+    syncMetadata.connection.disconnect();
+  }
+});
+
+test('indexer defers indexed clock persistence until a refresh happens on delayed refresh storages', async () => {
+  const calls: string[] = [];
+  const docsInRootDoc = new Map([['doc1', { title: 'Doc 1' }]]);
+  const docStorage = new TestDocStorage(
+    'workspace-id',
+    new Map([['doc1', new Date('2026-01-01T00:00:00.000Z')]]),
+    async () => ({
+      title: 'Doc 1',
+      summary: 'summary',
+      blocks: [
+        { blockId: 'block-1', flavour: 'affine:image', blob: ['blob-1'] },
+      ],
+    })
+  );
+  const indexer = new TrackingIndexerStorage(calls, 30_000);
+  const update = vi.spyOn(indexer, 'update');
+  const indexerSyncStorage = new TrackingIndexerSyncStorage(calls);
+  const sync = new IndexerSyncImpl(
+    docStorage,
+    {
+      local: indexer,
+      remotes: {},
+    },
+    indexerSyncStorage
+  );
+
+  vi.spyOn(reader, 'readAllDocsFromRootDoc').mockImplementation(
+    () => new Map(docsInRootDoc)
+  );
+
+  try {
+    sync.start();
+    await sync.waitForCompleted();
+
+    const docUpdate = update.mock.calls.find(([table]) => table === 'doc');
+    expect(docUpdate).toBeDefined();
+    expect([...(docUpdate?.[1].fields ?? [])]).toEqual(
+      expect.arrayContaining([
+        ['docId', ['doc1']],
+        ['title', ['Doc 1']],
+        ['summary', ['summary']],
+      ])
+    );
+    expect(calls).not.toContain('setClock:doc1');
+
+    sync.stop();
+
+    await vi.waitFor(() => {
+      expect(calls).toContain('setClock:doc1');
+    });
+
+    const lastRefreshIndex = calls.lastIndexOf('refresh');
+    const setClockIndex = calls.indexOf('setClock:doc1');
+
+    expect(lastRefreshIndex).toBeGreaterThanOrEqual(0);
+    expect(setClockIndex).toBeGreaterThan(lastRefreshIndex);
+  } finally {
+    sync.stop();
+  }
+});
+
+test('indexer completion waits for the current job to finish', async () => {
+  const docsInRootDoc = new Map([['doc1', { title: 'Doc 1' }]]);
+  const crawlStarted = deferred<void>();
+  const releaseCrawl = deferred<void>();
+  const docStorage = new TestDocStorage(
+    'workspace-id',
+    new Map([['doc1', new Date('2026-01-01T00:00:00.000Z')]]),
+    async () => {
+      crawlStarted.resolve();
+      await releaseCrawl.promise;
+      return {
+        title: 'Doc 1',
+        summary: 'summary',
+        blocks: [
+          { blockId: 'block-1', flavour: 'affine:image', blob: ['blob-1'] },
+        ],
+      };
+    }
+  );
+  const sync = new IndexerSyncImpl(
+    docStorage,
+    {
+      local: new TrackingIndexerStorage([], 30_000),
+      remotes: {},
+    },
+    new TrackingIndexerSyncStorage([])
+  );
+
+  vi.spyOn(reader, 'readAllDocsFromRootDoc').mockImplementation(
+    () => new Map(docsInRootDoc)
+  );
+
+  try {
+    sync.start();
+    await crawlStarted.promise;
+
+    let completed = false;
+    let docCompleted = false;
+
+    const waitForCompleted = sync.waitForCompleted().then(() => {
+      completed = true;
+    });
+    const waitForDocCompleted = sync.waitForDocCompleted('doc1').then(() => {
+      docCompleted = true;
+    });
+
+    await new Promise(resolve => setTimeout(resolve, 20));
+
+    expect(completed).toBe(false);
+    expect(docCompleted).toBe(false);
+
+    releaseCrawl.resolve();
+
+    await waitForCompleted;
+    await waitForDocCompleted;
+  } finally {
+    sync.stop();
+  }
+});
+
+test('indexer priority requests accumulate', async () => {
+  const docsInRootDoc = new Map([
+    ['doc-low', { title: 'Doc Low' }],
+    ['doc-high', { title: 'Doc High' }],
+  ]);
+  const crawled: string[] = [];
+  const rootDocCrawlStarted = deferred<void>();
+  const releaseRootDocCrawl = deferred<void>();
+  const docStorage = new TestDocStorage(
+    'workspace-id',
+    new Map([
+      ['doc-low', new Date('2026-01-01T00:00:00.000Z')],
+      ['doc-high', new Date('2026-01-01T00:00:00.000Z')],
+    ]),
+    async docId => {
+      crawled.push(docId);
+      return { title: docId, summary: 'summary', blocks: [] };
+    }
+  );
+  const indexer = new TrackingIndexerStorage([], 30_000);
+  // hold the loop inside the root doc crawl, so both docs stay queued while
+  // their priorities are changed
+  let holding = false;
+  vi.spyOn(indexer, 'insert').mockImplementation(async () => {
+    if (!holding) {
+      holding = true;
+      rootDocCrawlStarted.resolve();
+      await releaseRootDocCrawl.promise;
+    }
+  });
+  const sync = new IndexerSyncImpl(
+    docStorage,
+    {
+      local: indexer,
+      remotes: {},
+    },
+    new TrackingIndexerSyncStorage([])
+  );
+
+  vi.spyOn(reader, 'readAllDocsFromRootDoc').mockImplementation(
+    () => new Map(docsInRootDoc)
+  );
+
+  try {
+    sync.start();
+    await rootDocCrawlStarted.promise;
+
+    sync.addPriority('doc-low', 5);
+    // two holders on the same doc, one of them goes away
+    sync.addPriority('doc-high', 10);
+    const releaseSecondHolder = sync.addPriority('doc-high', 10);
+    releaseSecondHolder();
+
+    releaseRootDocCrawl.resolve();
+
+    await vi.waitFor(() => {
+      expect(crawled).toHaveLength(2);
+    });
+
+    // the remaining holder still asked for +10, so `doc-high` must outrank the
+    // +5 of `doc-low`
+    expect(crawled).toEqual(['doc-high', 'doc-low']);
+  } finally {
+    releaseRootDocCrawl.resolve();
+    sync.stop();
+  }
+});

packages/common/nbstore/src/__tests__/utils.ts

@@ -0,0 +1,30 @@
+import { expect } from 'vitest';
+import { applyUpdate, Doc as YDoc } from 'yjs';
+
+export function expectYjsEqual(
+  doc: Uint8Array | YDoc,
+  match: Record<string, any>
+) {
+  let ydoc: YDoc;
+  if (doc instanceof Uint8Array) {
+    ydoc = new YDoc();
+    applyUpdate(ydoc, doc);
+  } else {
+    ydoc = doc;
+  }
+
+  for (const key in match) {
+    const value = match[key];
+    if (Array.isArray(value)) {
+      const actual = ydoc.getArray(key).toJSON();
+      expect(actual).toEqual(value);
+    } else if (typeof value === 'string') {
+      const actual = ydoc.getText(key).toJSON();
+      expect(actual).toEqual(value);
+    } else {
+      const actual = ydoc.getMap(key).toJSON();
+      expect(actual).toEqual(value);
+    }
+  }
+  return doc;
+}

packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts

@@ -0,0 +1,318 @@
+import { expect, test, vitest } from 'vitest';
+
+import { AutoReconnectConnection } from '../connection';
+
+test('connect and disconnect', async () => {
+  class TestConnection extends AutoReconnectConnection<{
+    disconnect: () => void;
+  }> {
+    connectCount = 0;
+    abortCount = 0;
+    disconnectCount = 0;
+    notListenAbort = false;
+    override async doConnect(signal?: AbortSignal) {
+      this.connectCount++;
+      return new Promise<{ disconnect: () => void }>((resolve, reject) => {
+        setTimeout(() => {
+          resolve({
+            disconnect: () => {
+              this.disconnectCount++;
+            },
+          });
+        }, 300);
+        if (!this.notListenAbort) {
+          signal?.addEventListener('abort', reason => {
+            reject(reason);
+          });
+        }
+      }).catch(err => {
+        this.abortCount++;
+        throw err;
+      });
+    }
+    override doDisconnect(t: { disconnect: () => void }) {
+      return t.disconnect();
+    }
+  }
+
+  const connection = new TestConnection();
+  connection.connect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(1);
+    expect(connection.disconnectCount).toBe(0);
+    expect(connection.abortCount).toBe(0);
+    expect(connection.status).toBe('connected');
+  });
+
+  connection.disconnect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(1);
+    expect(connection.disconnectCount).toBe(1);
+    expect(connection.abortCount).toBe(0);
+    expect(connection.status).toBe('closed');
+  });
+
+  // connect twice
+  connection.connect();
+  connection.connect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(2);
+    expect(connection.disconnectCount).toBe(1);
+    expect(connection.abortCount).toBe(0);
+    expect(connection.status).toBe('connected');
+  });
+
+  connection.disconnect();
+  connection.disconnect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(2);
+    expect(connection.disconnectCount).toBe(2);
+    expect(connection.abortCount).toBe(0);
+    expect(connection.status).toBe('closed');
+  });
+
+  // calling connect disconnect consecutively, the previous connect call will be aborted.
+  connection.connect();
+  connection.disconnect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(3);
+    expect(connection.disconnectCount).toBe(2);
+    expect(connection.abortCount).toBe(1);
+    expect(connection.status).toBe('closed');
+  });
+
+  connection.connect();
+  connection.disconnect();
+  connection.connect();
+  connection.disconnect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(5);
+    expect(connection.disconnectCount).toBe(2);
+    expect(connection.abortCount).toBe(3);
+    expect(connection.status).toBe('closed');
+  });
+
+  // if connection is not listening to abort event, disconnect will be called
+  connection.notListenAbort = true;
+  connection.connect();
+  connection.disconnect();
+  connection.connect();
+  connection.disconnect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(7);
+    expect(connection.disconnectCount).toBe(4);
+    expect(connection.abortCount).toBe(3);
+    expect(connection.status).toBe('closed');
+  });
+});
+
+test('retry when connect failed', async () => {
+  class TestConnection extends AutoReconnectConnection {
+    override retryDelay = 300;
+    connectCount = 0;
+    retryDelayFor(retryCount: number) {
+      return this.getRetryDelay(retryCount);
+    }
+    override async doConnect() {
+      this.connectCount++;
+      if (this.connectCount >= 3) {
+        return { hello: 'world' };
+      }
+      throw new Error('not connected, count: ' + this.connectCount);
+    }
+    override doDisconnect() {
+      return Promise.resolve();
+    }
+    triggerError(error: Error) {
+      this.error = error;
+    }
+  }
+
+  const connection = new TestConnection();
+  expect([0, 1, 2, 8].map(count => connection.retryDelayFor(count))).toEqual([
+    300, 600, 1200, 60000,
+  ]);
+  connection.connect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(1);
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toContain('not connected, count: 1');
+  });
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(2);
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toBe('not connected, count: 2');
+  });
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(3);
+    expect(connection.status).toBe('connected');
+    expect(connection.error).toBeUndefined();
+  });
+
+  connection.triggerError(new Error('disconnected'));
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(4);
+    expect(connection.status).toBe('connected');
+  });
+});
+
+test('retry when error', async () => {
+  class TestConnection extends AutoReconnectConnection {
+    override retryDelay = 300;
+    connectCount = 0;
+    disconnectCount = 0;
+    override async doConnect() {
+      this.connectCount++;
+      return {
+        hello: 'world',
+      };
+    }
+    override doDisconnect(conn: any) {
+      this.disconnectCount++;
+      expect(conn).toEqual({
+        hello: 'world',
+      });
+    }
+    triggerError(error: Error) {
+      this.error = error;
+    }
+  }
+
+  const connection = new TestConnection();
+  connection.connect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(1);
+    expect(connection.status).toBe('connected');
+  });
+
+  connection.triggerError(new Error('test error'));
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(1);
+    expect(connection.disconnectCount).toBe(1);
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toBe('test error');
+  });
+
+  // waitfor reconnect
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(2);
+    expect(connection.disconnectCount).toBe(1);
+    expect(connection.status).toBe('connected');
+    expect(connection.error).toBeUndefined();
+  });
+
+  // do not reconnect if the connection is closed
+  connection.disconnect();
+  connection.triggerError(new Error('test error2'));
+  await new Promise(resolve => setTimeout(resolve, 1000));
+  expect(connection.connectCount).toBe(2);
+  expect(connection.status).toBe('closed');
+});
+
+test('connecting timeout', async () => {
+  class TestConnection extends AutoReconnectConnection {
+    override connectingTimeout = 150;
+    override retryDelay = 150;
+    connectCount = 0;
+    disconnectCount = 0;
+    override async doConnect() {
+      this.connectCount++;
+      if (this.connectCount === 3) {
+        return { foo: 'bar' };
+      }
+      await new Promise(resolve => setTimeout(resolve, 300));
+      throw new Error('not connected, count: ' + this.connectCount);
+    }
+    override doDisconnect(conn: any) {
+      this.disconnectCount++;
+      expect(conn).toEqual({
+        foo: 'bar',
+      });
+    }
+    triggerError(error: Error) {
+      this.error = error;
+    }
+  }
+
+  const connection = new TestConnection();
+  connection.connect();
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(1);
+    expect(connection.disconnectCount).toBe(0);
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toBe('connecting timeout');
+  });
+
+  // wait for reconnect
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(2);
+    expect(connection.disconnectCount).toBe(0);
+    expect(connection.status).toBe('connecting');
+    expect(connection.error?.message).toBe('connecting timeout');
+  });
+
+  // trigger error while connecting
+  connection.triggerError(new Error('test error2'));
+  connection.triggerError(new Error('test error2'));
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(2);
+    expect(connection.disconnectCount).toBe(0);
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toBe('test error2');
+  });
+
+  // wait for reconnect
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(3);
+    expect(connection.disconnectCount).toBe(0);
+    expect(connection.status).toBe('connected');
+    expect(connection.error).toBeUndefined();
+  });
+
+  // trigger error after connected
+  connection.triggerError(new Error('test error3'));
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(3);
+    expect(connection.disconnectCount).toBe(1); // previous connect is disconnected
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toBe('test error3');
+  });
+
+  // reconnect and timeout again
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(4);
+  });
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(4);
+    expect(connection.disconnectCount).toBe(1);
+    expect(connection.status).toBe('error');
+    expect(connection.error?.message).toBe('connecting timeout');
+  });
+
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(5);
+  });
+
+  connection.disconnect();
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  // no reconnect after disconnect
+  expect(connection.connectCount).toBe(5);
+  expect(connection.status).toBe('closed');
+});

packages/common/nbstore/src/connection/connection.ts

@@ -0,0 +1,257 @@
+import EventEmitter2 from 'eventemitter2';
+
+import { MANUALLY_STOP } from '../utils/throw-if-aborted';
+
+export type ConnectionStatus =
+  | 'idle'
+  | 'connecting'
+  | 'connected'
+  | 'error'
+  | 'closed';
+
+export interface Connection<T = any> {
+  readonly status: ConnectionStatus;
+  readonly error?: Error;
+  readonly inner: T;
+  connect(): void;
+  disconnect(): void;
+  waitForConnected(signal?: AbortSignal): Promise<void>;
+  onStatusChanged(
+    cb: (status: ConnectionStatus, error?: Error) => void
+  ): () => void;
+}
+
+export abstract class AutoReconnectConnection<
+  T = any,
+> implements Connection<T> {
+  private readonly event = new EventEmitter2({
+    maxListeners: 100,
+  });
+  private _inner: T | undefined = undefined;
+  private _status: ConnectionStatus = 'idle';
+  private _error: Error | undefined = undefined;
+  retryDelay = 3000;
+  maxRetryDelay = 60000;
+  connectingTimeout = 15000;
+  private retryCount = 0;
+  private refCount = 0;
+  private connectingAbort?: AbortController;
+  private reconnectingAbort?: AbortController;
+
+  constructor() {}
+
+  get shareId(): string | undefined {
+    return undefined;
+  }
+
+  get maybeConnection() {
+    return this._inner;
+  }
+
+  get inner(): T {
+    if (this._inner === undefined) {
+      throw new Error(
+        `Connection ${this.constructor.name} has not been established.`
+      );
+    }
+
+    return this._inner;
+  }
+
+  private set inner(inner: T | undefined) {
+    this._inner = inner;
+  }
+
+  get status() {
+    return this._status;
+  }
+
+  get error() {
+    return this._error;
+  }
+
+  protected set error(error: Error | undefined) {
+    this.handleError(error);
+  }
+
+  private setStatus(status: ConnectionStatus, error?: Error) {
+    const shouldEmit = status !== this._status || error !== this._error;
+    this._status = status;
+    // we only clear-up error when status is connected
+    if (error || status === 'connected') {
+      this._error = error;
+    }
+    if (shouldEmit) {
+      this.emitStatusChanged(status, this._error);
+    }
+  }
+
+  protected abstract doConnect(signal?: AbortSignal): Promise<T>;
+  protected abstract doDisconnect(conn: T): void;
+
+  private innerConnect() {
+    if (this.status !== 'connecting') {
+      this.setStatus('connecting');
+      const connectingAbort = new AbortController();
+      this.connectingAbort = connectingAbort;
+      const signal = connectingAbort.signal;
+      const timeout = setTimeout(() => {
+        if (!signal.aborted) {
+          this.handleError(new Error('connecting timeout'));
+        }
+      }, this.connectingTimeout);
+      this.doConnect(signal)
+        .then(value => {
+          clearTimeout(timeout);
+          if (!signal.aborted) {
+            this._inner = value;
+            this.retryCount = 0;
+            this.setStatus('connected');
+          } else {
+            try {
+              this.doDisconnect(value);
+            } catch (error) {
+              console.error('failed to disconnect', error);
+            }
+          }
+        })
+        .catch(error => {
+          if (!signal.aborted) {
+            clearTimeout(timeout);
+            console.error('failed to connect', error);
+            this.handleError(error as any);
+          }
+        });
+    }
+  }
+
+  private innerDisconnect() {
+    this.connectingAbort?.abort(MANUALLY_STOP);
+    this.reconnectingAbort?.abort(MANUALLY_STOP);
+    try {
+      if (this._inner) {
+        this.doDisconnect(this._inner);
+      }
+    } catch (error) {
+      console.error('failed to disconnect', error);
+    }
+    this.reconnectingAbort = undefined;
+    this.connectingAbort = undefined;
+    this._inner = undefined;
+  }
+
+  private handleError(reason?: Error) {
+    // on error
+    console.error('connection error, will reconnect', reason);
+    this.innerDisconnect();
+    // if the connection is closed, do not reconnect
+    if (this.status === 'closed') {
+      return;
+    }
+    this.setStatus('error', reason);
+    // reconnect
+
+    this.reconnectingAbort = new AbortController();
+    const signal = this.reconnectingAbort.signal;
+    const retryDelay = this.getRetryDelay(this.retryCount++);
+    const timeout = setTimeout(() => {
+      if (!signal.aborted) {
+        this.innerConnect();
+      }
+    }, retryDelay);
+    signal.addEventListener('abort', () => {
+      clearTimeout(timeout);
+    });
+  }
+
+  protected getRetryDelay(retryCount: number) {
+    return Math.min(this.retryDelay * 2 ** retryCount, this.maxRetryDelay);
+  }
+
+  connect() {
+    this.refCount++;
+    if (this.refCount === 1) {
+      this.innerConnect();
+    }
+  }
+
+  disconnect(force?: boolean) {
+    if (force) {
+      this.refCount = 0;
+    } else {
+      this.refCount = Math.max(this.refCount - 1, 0);
+    }
+    if (this.refCount === 0) {
+      this.innerDisconnect();
+      this.retryCount = 0;
+      this.setStatus('closed');
+    }
+  }
+
+  waitForConnected(signal?: AbortSignal) {
+    return new Promise<void>((resolve, reject) => {
+      if (this.status === 'connected') {
+        resolve();
+        return;
+      }
+
+      if (signal?.aborted) {
+        reject(signal.reason);
+        return;
+      }
+
+      const off = this.onStatusChanged(status => {
+        if (status === 'connected') {
+          resolve();
+          cleanup();
+        }
+      });
+
+      const onAbort = () => {
+        reject(signal?.reason);
+        cleanup();
+      };
+      const cleanup = () => {
+        off();
+        signal?.removeEventListener('abort', onAbort);
+      };
+      signal?.addEventListener('abort', onAbort, { once: true });
+    });
+  }
+
+  onStatusChanged(
+    cb: (status: ConnectionStatus, error?: Error) => void
+  ): () => void {
+    this.event.on('statusChanged', cb);
+    return () => {
+      this.event.off('statusChanged', cb);
+    };
+  }
+
+  private readonly emitStatusChanged = (
+    status: ConnectionStatus,
+    error?: Error
+  ) => {
+    this.event.emit('statusChanged', status, error);
+  };
+}
+
+export class DummyConnection implements Connection<undefined> {
+  readonly status: ConnectionStatus = 'connected';
+  readonly inner: undefined;
+
+  connect(): void {
+    return;
+  }
+  disconnect(): void {
+    return;
+  }
+  waitForConnected(_signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  onStatusChanged(
+    _cb: (status: ConnectionStatus, error?: Error) => void
+  ): () => void {
+    return () => {};
+  }
+}

packages/common/nbstore/src/connection/index.ts

@@ -0,0 +1,2 @@
+export * from './connection';
+export * from './shared-connection';

packages/common/nbstore/src/connection/shared-connection.ts

@@ -0,0 +1,20 @@
+import type { AutoReconnectConnection } from './connection';
+
+const CONNECTIONS: Map<string, AutoReconnectConnection<any>> = new Map();
+export function share<T extends AutoReconnectConnection<any>>(conn: T): T {
+  if (!conn.shareId) {
+    throw new Error(
+      `Connection ${conn.constructor.name} is not shareable.\nIf you want to make it shareable, please override [shareId].`
+    );
+  }
+
+  const existing = CONNECTIONS.get(conn.shareId);
+
+  if (existing) {
+    return existing as T;
+  }
+
+  CONNECTIONS.set(conn.shareId, conn);
+
+  return conn;
+}

packages/common/nbstore/src/frontend/awareness.ts

@@ -0,0 +1,74 @@
+import { nanoid } from 'nanoid';
+import {
+  applyAwarenessUpdate,
+  type Awareness,
+  encodeAwarenessUpdate,
+} from 'y-protocols/awareness';
+
+import type { AwarenessRecord } from '../storage/awareness';
+import type { AwarenessSync } from '../sync/awareness';
+
+type AwarenessChanges = Record<'added' | 'updated' | 'removed', number[]>;
+
+export class AwarenessFrontend {
+  constructor(private readonly sync: AwarenessSync) {}
+
+  connectAwareness(awareness: Awareness) {
+    const uniqueId = nanoid();
+    const handleAwarenessUpdate = (
+      changes: AwarenessChanges,
+      origin: string
+    ) => {
+      if (origin === uniqueId) {
+        return;
+      }
+      const changedClients = Object.values(changes).reduce((res, cur) =>
+        res.concat(cur)
+      );
+
+      const update = encodeAwarenessUpdate(awareness, changedClients);
+      this.sync
+        .update(
+          {
+            docId: awareness.doc.guid,
+            bin: update,
+          },
+          uniqueId
+        )
+        .catch(error => {
+          console.error('update awareness error', error);
+        });
+    };
+
+    awareness.on('change', handleAwarenessUpdate);
+    const handleSyncUpdate = (update: AwarenessRecord, origin?: string) => {
+      if (origin === uniqueId) {
+        // skip self update
+        return;
+      }
+
+      applyAwarenessUpdate(awareness, update.bin, uniqueId);
+    };
+    const handleSyncCollect = () => {
+      return Promise.resolve({
+        docId: awareness.doc.guid,
+        bin: encodeAwarenessUpdate(awareness, [awareness.clientID]),
+      });
+    };
+    const unsubscribe = this.sync.subscribeUpdate(
+      awareness.doc.guid,
+      handleSyncUpdate,
+      handleSyncCollect
+    );
+
+    awareness.once('destroy', () => {
+      awareness.off('change', handleAwarenessUpdate);
+      unsubscribe();
+    });
+
+    return () => {
+      awareness.off('change', handleAwarenessUpdate);
+      unsubscribe();
+    };
+  }
+}

packages/common/nbstore/src/frontend/blob.ts

@@ -0,0 +1,86 @@
+import type { BlobRecord, BlobSource, BlobStorage } from '../storage';
+import { SingletonLocker } from '../storage/lock';
+import type { BlobSync } from '../sync/blob';
+
+export class BlobFrontend {
+  // Since 'set' and 'get' operations may be called in rapid succession, we use a lock mechanism
+  // to ensure that 'get' requests for the same blob are paused when a 'set' operation is in progress.
+  private readonly lock = new SingletonLocker();
+  constructor(
+    readonly storage: BlobStorage,
+    private readonly sync: BlobSync
+  ) {}
+
+  get state$() {
+    return this.sync.state$;
+  }
+
+  blobState$(blobId: string) {
+    return this.sync.blobState$(blobId);
+  }
+
+  async get(blobId: string) {
+    await this.waitForConnected();
+    await using lock = await this.lock.lock('blob', blobId);
+    const local = await this.storage.get(blobId);
+    if (local) {
+      return local;
+    }
+    await lock[Symbol.asyncDispose]();
+
+    await this.sync.downloadBlob(blobId).catch(() => {
+      // ignore the error as it has already been recorded in the sync status
+    });
+    return await this.storage.get(blobId);
+  }
+
+  async set(blob: BlobRecord) {
+    await this.waitForConnected();
+    await using lock = await this.lock.lock('blob', blob.key);
+    await this.storage.set(blob);
+    await lock[Symbol.asyncDispose]();
+
+    // We don't wait for the upload to complete,
+    // as the upload process runs asynchronously in the background
+    this.sync.uploadBlob(blob, true /* force upload */).catch(() => {
+      // ignore the error as it has already been recorded in the sync status
+    });
+
+    return;
+  }
+
+  /**
+   * Uploads a blob to the peer. Do nothing if the blob has already been uploaded.
+   *
+   * @returns Always resolves to true when successful
+   *
+   * @throws This method will throw an error if the blob is not found locally, if the upload is aborted, or if it fails due to storage limitations.
+   */
+  async upload(blobIdOrRecord: string | BlobRecord): Promise<boolean> {
+    await this.waitForConnected();
+    const blob =
+      typeof blobIdOrRecord === 'string'
+        ? await this.storage.get(blobIdOrRecord)
+        : blobIdOrRecord;
+    if (!blob) {
+      throw new Error(`Blob ${blobIdOrRecord} not found`);
+    }
+    return this.sync.uploadBlob(blob, false);
+  }
+
+  registerSource(source: BlobSource) {
+    return this.sync.registerSource(source);
+  }
+
+  unregisterSource(source: BlobSource) {
+    return this.sync.unregisterSource(source);
+  }
+
+  fullDownload(peerId?: string, signal?: AbortSignal) {
+    return this.sync.fullDownload(peerId, signal);
+  }
+
+  private waitForConnected(signal?: AbortSignal) {
+    return this.storage.connection.waitForConnected(signal);
+  }
+}

packages/common/nbstore/src/frontend/doc.ts

@@ -0,0 +1,546 @@
+import { groupBy } from 'lodash-es';
+import { nanoid } from 'nanoid';
+import {
+  combineLatest,
+  filter,
+  first,
+  lastValueFrom,
+  map,
+  Observable,
+  ReplaySubject,
+  share,
+  Subject,
+  throttleTime,
+} from 'rxjs';
+import {
+  applyUpdate,
+  type Doc as YDoc,
+  encodeStateAsUpdate,
+  Map as YMap,
+  mergeUpdates,
+  type Transaction as YTransaction,
+} from 'yjs';
+
+import type { DocRecord, DocStorage } from '../storage';
+import type { DocSync } from '../sync/doc';
+import { AsyncPriorityQueue } from '../utils/async-priority-queue';
+import { isEmptyUpdate } from '../utils/is-empty-update';
+import { takeUntilAbort } from '../utils/take-until-abort';
+import { MANUALLY_STOP, throwIfAborted } from '../utils/throw-if-aborted';
+
+const NBSTORE_ORIGIN = 'nbstore-frontend';
+
+type Job =
+  | {
+      type: 'load';
+      docId: string;
+    }
+  | {
+      type: 'save';
+      docId: string;
+      update: Uint8Array;
+    }
+  | {
+      type: 'apply';
+      docId: string;
+      update: Uint8Array;
+    };
+
+interface DocFrontendOptions {
+  mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
+}
+
+export type DocFrontendDocState = {
+  /**
+   * some data is available in yjs doc instance
+   */
+  ready: boolean;
+  /**
+   * data is loaded from local doc storage and applied to yjs doc instance
+   */
+  loaded: boolean;
+  /**
+   * some data is being applied to yjs doc instance, or some data is being saved to local doc storage
+   */
+  updating: boolean;
+  /**
+   * the doc is syncing with remote peers
+   */
+  syncing: boolean;
+  /**
+   * the doc is synced with remote peers
+   */
+  synced: boolean;
+  /**
+   * the doc is retrying to sync with remote peers
+   */
+  syncRetrying: boolean;
+  /**
+   * the error message when syncing with remote peers
+   */
+  syncErrorMessage: string | null;
+};
+
+export type DocFrontendState = {
+  /**
+   * total number of docs
+   */
+  total: number;
+  /**
+   * number of docs that have been loaded to yjs doc instance
+   */
+  loaded: number;
+  /**
+   * some data is being applied to yjs doc instance, or some data is being saved to local doc storage
+   */
+  updating: boolean;
+  /**
+   * number of docs that are syncing with remote peers
+   */
+  syncing: number;
+  /**
+   * whether all docs are synced with remote peers
+   */
+  synced: boolean;
+  /**
+   * whether the doc is retrying to sync with remote peers
+   */
+  syncRetrying: boolean;
+  /**
+   * the error message when syncing with remote peers
+   */
+  syncErrorMessage: string | null;
+};
+
+export class DocFrontend {
+  private readonly uniqueId = `frontend:${nanoid()}`;
+
+  private readonly prioritySettings = new Map<string, number>();
+
+  private readonly status = {
+    docs: new Map<string, YDoc>(),
+    connectedDocs: new Set<string>(),
+    readyDocs: new Set<string>(),
+    jobDocQueue: new AsyncPriorityQueue(),
+    jobMap: new Map<string, Job[]>(),
+    currentJob: null as { docId: string; jobs: Job[] } | null,
+  };
+
+  private readonly statusUpdatedSubject$ = new Subject<string>();
+
+  private readonly abort = new AbortController();
+
+  constructor(
+    public readonly storage: DocStorage,
+    private readonly sync: DocSync,
+    readonly options: DocFrontendOptions = {}
+  ) {}
+
+  applyDocLifecycle(docId: string, lifecycle: 'trash' | 'restore' | 'delete') {
+    if (!this.storage.applyDocLifecycle) {
+      throw new Error('Document lifecycle is unavailable');
+    }
+    return this.storage.applyDocLifecycle(docId, lifecycle);
+  }
+
+  private _docState$(docId: string): Observable<DocFrontendDocState> {
+    const frontendState$ = new Observable<{
+      ready: boolean;
+      loaded: boolean;
+      updating: boolean;
+    }>(subscribe => {
+      const next = () => {
+        subscribe.next({
+          ready: this.status.readyDocs.has(docId),
+          loaded: this.status.connectedDocs.has(docId),
+          updating:
+            (this.status.jobMap.get(docId)?.length ?? 0) > 0 ||
+            this.status.currentJob?.docId === docId,
+        });
+      };
+      next();
+      return this.statusUpdatedSubject$.subscribe(updatedId => {
+        if (updatedId === docId) next();
+      });
+    });
+    const syncState$ = this.sync.docState$(docId);
+    return combineLatest([frontendState$, syncState$]).pipe(
+      map(([frontend, sync]) => ({
+        ...frontend,
+        synced: sync.synced,
+        syncing: sync.syncing,
+        syncRetrying: sync.retrying,
+        syncErrorMessage: sync.errorMessage,
+      }))
+    );
+  }
+
+  docState$(docId: string): Observable<DocFrontendDocState> {
+    return this._docState$(docId).pipe(
+      throttleTime(1000, undefined, {
+        trailing: true,
+        leading: true,
+      })
+    );
+  }
+
+  private readonly _state$ = combineLatest([
+    new Observable<{ total: number; loaded: number; updating: boolean }>(
+      subscriber => {
+        const next = () => {
+          subscriber.next({
+            total: this.status.docs.size,
+            loaded: this.status.connectedDocs.size,
+            updating:
+              this.status.jobMap.size > 0 || this.status.currentJob !== null,
+          });
+        };
+        next();
+        return this.statusUpdatedSubject$.subscribe(() => {
+          next();
+        });
+      }
+    ),
+    this.sync.state$,
+  ]).pipe(
+    map(([frontend, sync]) => ({
+      total: sync.total ?? frontend.total,
+      loaded: frontend.loaded,
+      updating: frontend.updating,
+      syncing: sync.syncing,
+      synced: sync.synced,
+      syncRetrying: sync.retrying,
+      syncErrorMessage: sync.errorMessage,
+    })),
+    share({
+      connector: () => new ReplaySubject(1),
+    })
+  ) satisfies Observable<DocFrontendState>;
+
+  state$ = this._state$.pipe(
+    throttleTime(1000, undefined, {
+      leading: true,
+      trailing: true,
+    })
+  );
+
+  start() {
+    if (this.abort.signal.aborted) {
+      throw new Error('doc frontend can only start once');
+    }
+    this.mainLoop(this.abort.signal).catch(error => {
+      console.error(error);
+    });
+  }
+
+  stop() {
+    this.abort.abort(MANUALLY_STOP);
+  }
+
+  private async mainLoop(signal?: AbortSignal) {
+    await this.storage.connection.waitForConnected(signal);
+    const dispose = this.storage.subscribeDocUpdate((record, origin) => {
+      this.event.onStorageUpdate(record, origin);
+    });
+    try {
+      // wait for storage to connect
+      await Promise.race([
+        this.storage.connection.waitForConnected(signal),
+        new Promise((_, reject) => {
+          signal?.addEventListener('abort', reason => {
+            reject(reason);
+          });
+        }),
+      ]);
+
+      while (true) {
+        throwIfAborted(signal);
+
+        const docId = await this.status.jobDocQueue.asyncPop(undefined, signal);
+        const jobs = this.status.jobMap.get(docId);
+        this.status.jobMap.delete(docId);
+
+        if (!jobs) {
+          this.statusUpdatedSubject$.next(docId);
+          continue;
+        }
+
+        this.status.currentJob = { docId, jobs };
+        this.statusUpdatedSubject$.next(docId);
+
+        const { apply, load, save } = groupBy(jobs, job => job.type) as {
+          [key in Job['type']]?: Job[];
+        };
+
+        if (load?.length) {
+          await this.jobs.load(load[0] as any, signal);
+        }
+
+        for (const applyJob of apply ?? []) {
+          await this.jobs.apply(applyJob as any, signal);
+        }
+
+        if (save?.length) {
+          await this.jobs.save(docId, save as any, signal);
+        }
+
+        this.status.currentJob = null;
+        this.statusUpdatedSubject$.next(docId);
+      }
+    } finally {
+      dispose();
+    }
+  }
+
+  /**
+   * Connect a doc to the frontend, the doc will sync with the doc storage.
+   * @param doc - The doc to connect
+   */
+  connectDoc(doc: YDoc) {
+    this._connectDoc(doc);
+  }
+
+  readonly jobs = {
+    load: async (job: Job & { type: 'load' }, signal?: AbortSignal) => {
+      const doc = this.status.docs.get(job.docId);
+      if (!doc) {
+        return;
+      }
+      const existingData = encodeStateAsUpdate(doc);
+
+      if (!isEmptyUpdate(existingData)) {
+        this.schedule({
+          type: 'save',
+          docId: doc.guid,
+          update: existingData,
+        });
+      }
+
+      // mark doc as loaded
+      doc.emit('sync', [true, doc]);
+
+      const docRecord = await this.storage.getDoc(job.docId);
+      throwIfAborted(signal);
+
+      if (docRecord && !isEmptyUpdate(docRecord.bin)) {
+        this.applyUpdate(job.docId, docRecord.bin);
+
+        this.status.readyDocs.add(job.docId);
+      }
+
+      this.status.connectedDocs.add(job.docId);
+      this.statusUpdatedSubject$.next(job.docId);
+    },
+    save: async (
+      docId: string,
+      jobs: (Job & { type: 'save' })[],
+      signal?: AbortSignal
+    ) => {
+      if (!this.status.docs.has(docId)) {
+        return;
+      }
+      if (this.status.connectedDocs.has(docId)) {
+        const merged = await this.mergeUpdates(
+          jobs.map(j => j.update).filter(update => !isEmptyUpdate(update))
+        );
+        throwIfAborted(signal);
+        await this.storage.pushDocUpdate(
+          {
+            docId,
+            bin: merged,
+          },
+          this.uniqueId
+        );
+      }
+    },
+    apply: async (job: Job & { type: 'apply' }, signal?: AbortSignal) => {
+      throwIfAborted(signal);
+      if (!this.status.docs.has(job.docId)) {
+        return;
+      }
+      if (this.status.connectedDocs.has(job.docId)) {
+        this.applyUpdate(job.docId, job.update);
+      }
+      if (!isEmptyUpdate(job.update)) {
+        this.status.readyDocs.add(job.docId);
+        this.statusUpdatedSubject$.next(job.docId);
+      }
+    },
+  };
+
+  event = {
+    onStorageUpdate: (update: DocRecord, origin?: string) => {
+      if (origin !== this.uniqueId) {
+        this.schedule({
+          type: 'apply',
+          docId: update.docId,
+          update: update.bin,
+        });
+      }
+    },
+  };
+
+  /**
+   * Disconnect a doc from the frontend, the doc will stop syncing with the doc storage.
+   * It's not recommended to use this method directly, better to use `doc.destroy()`.
+   *
+   * @param doc - The doc to disconnect
+   */
+  disconnectDoc(doc: YDoc) {
+    this.status.docs.delete(doc.guid);
+    this.status.connectedDocs.delete(doc.guid);
+    this.status.readyDocs.delete(doc.guid);
+    this.status.jobDocQueue.remove(doc.guid);
+    this.status.jobMap.delete(doc.guid);
+    this.statusUpdatedSubject$.next(doc.guid);
+    doc.off('update', this.handleDocUpdate);
+  }
+
+  addPriority(id: string, priority: number) {
+    const undoSyncPriority = this.sync?.addPriority(id, priority);
+    const oldPriority = this.prioritySettings.get(id) ?? 0;
+    const newPriority = oldPriority + priority;
+
+    this.prioritySettings.set(id, newPriority);
+    this.status.jobDocQueue.setPriority(id, newPriority);
+
+    return () => {
+      const currentPriority = this.prioritySettings.get(id) ?? 0;
+      const restoredPriority = currentPriority - priority;
+      this.prioritySettings.set(id, restoredPriority);
+      this.status.jobDocQueue.setPriority(id, restoredPriority);
+
+      undoSyncPriority?.();
+    };
+  }
+
+  private _connectDoc(doc: YDoc) {
+    if (this.status.docs.has(doc.guid)) {
+      throw new Error('doc already connected');
+    }
+    this.schedule({
+      type: 'load',
+      docId: doc.guid,
+    });
+
+    this.status.docs.set(doc.guid, doc);
+    this.statusUpdatedSubject$.next(doc.guid);
+
+    doc.on('update', this.handleDocUpdate);
+
+    doc.on('destroy', () => {
+      this.disconnectDoc(doc);
+    });
+  }
+
+  private schedule(job: Job) {
+    const priority = this.prioritySettings.get(job.docId) ?? 0;
+    this.status.jobDocQueue.push(job.docId, priority);
+
+    const existingJobs = this.status.jobMap.get(job.docId) ?? [];
+    existingJobs.push(job);
+    this.status.jobMap.set(job.docId, existingJobs);
+    this.statusUpdatedSubject$.next(job.docId);
+  }
+
+  private isApplyingUpdate = false;
+
+  applyUpdate(docId: string, update: Uint8Array) {
+    const doc = this.status.docs.get(docId);
+    if (doc && !isEmptyUpdate(update)) {
+      try {
+        this.isApplyingUpdate = true;
+        applyUpdate(doc, update, NBSTORE_ORIGIN);
+      } catch (err) {
+        console.error('failed to apply update yjs doc', err);
+      } finally {
+        this.isApplyingUpdate = false;
+      }
+    }
+  }
+
+  private readonly handleDocUpdate = (
+    update: Uint8Array,
+    origin: any,
+    doc: YDoc,
+    transaction: YTransaction
+  ) => {
+    if (origin === NBSTORE_ORIGIN) {
+      return;
+    }
+    if (this.isApplyingUpdate && BUILD_CONFIG.debug) {
+      let changedList = '';
+      for (const [changed, keys] of transaction.changed) {
+        for (const key of keys) {
+          if (changed instanceof YMap && key) {
+            changedList += `${key} => ${changed.get(key)}\n`;
+          }
+        }
+      }
+      console.warn(`⚠️ When nbstore applies a remote update, some code triggers a local change to the doc.
+This will causes the document's 'edited by' to become the current user, even if the user has not actually modified the document.
+This is usually caused by a coding error and needs to be fixed by the developer.
+Changed:
+${changedList}
+`);
+    }
+    if (!this.status.docs.has(doc.guid)) {
+      return;
+    }
+
+    this.schedule({
+      type: 'save',
+      docId: doc.guid,
+      update,
+    });
+  };
+
+  protected mergeUpdates(updates: Uint8Array[]) {
+    const merge = this.options?.mergeUpdates ?? mergeUpdates;
+
+    return merge(updates.filter(bin => !isEmptyUpdate(bin)));
+  }
+
+  async waitForUpdated(docId?: string, abort?: AbortSignal) {
+    const source$: Observable<DocFrontendDocState | DocFrontendState> = docId
+      ? this._docState$(docId)
+      : this._state$;
+    await lastValueFrom(
+      source$.pipe(
+        filter(status => !status.updating),
+        takeUntilAbort(abort),
+        first()
+      )
+    );
+    return;
+  }
+
+  async waitForDocLoaded(docId: string, abort?: AbortSignal) {
+    await lastValueFrom(
+      this._docState$(docId).pipe(
+        filter(state => state.loaded),
+        takeUntilAbort(abort),
+        first()
+      )
+    );
+  }
+
+  async waitForSynced(docId?: string, abort?: AbortSignal) {
+    await this.waitForUpdated(docId, abort);
+    await this.sync.waitForSynced(docId, abort);
+  }
+
+  async waitForDocReady(docId: string, abort?: AbortSignal) {
+    await lastValueFrom(
+      this._docState$(docId).pipe(
+        filter(state => state.ready),
+        takeUntilAbort(abort),
+        first()
+      )
+    );
+  }
+
+  async resetSync() {
+    await this.sync.resetSync();
+  }
+}

packages/common/nbstore/src/frontend/index.ts

@@ -0,0 +1,4 @@
+export * from './awareness';
+export * from './blob';
+export * from './doc';
+export * from './indexer';

packages/common/nbstore/src/frontend/indexer.ts

@@ -0,0 +1,80 @@
+import type {
+  AggregateOptions,
+  IndexerSchema,
+  Query,
+  SearchOptions,
+} from '../storage';
+import type { IndexerPreferOptions, IndexerSync } from '../sync/indexer';
+
+export class IndexerFrontend {
+  constructor(public readonly sync: IndexerSync) {}
+
+  get state$() {
+    return this.sync.state$;
+  }
+
+  docState$(docId: string) {
+    return this.sync.docState$(docId);
+  }
+
+  async search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O & { prefer?: IndexerPreferOptions }
+  ) {
+    return this.sync.search(table, query, options);
+  }
+
+  async aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O & { prefer?: IndexerPreferOptions }
+  ) {
+    return this.sync.aggregate(table, query, field, options);
+  }
+
+  search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O & { prefer?: IndexerPreferOptions }
+  ) {
+    return this.sync.search$(table, query, options);
+  }
+
+  aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O & { prefer?: IndexerPreferOptions }
+  ) {
+    return this.sync.aggregate$(table, query, field, options);
+  }
+
+  addPriority(docId: string, priority: number) {
+    return this.sync.addPriority(docId, priority);
+  }
+
+  waitForCompleted(signal?: AbortSignal) {
+    return this.sync.waitForCompleted(signal);
+  }
+
+  waitForDocCompleted(docId: string, signal?: AbortSignal) {
+    return this.sync.waitForDocCompleted(docId, signal);
+  }
+
+  waitForDocCompletedWithPriority(
+    docId: string,
+    priority: number,
+    signal?: AbortSignal
+  ) {
+    const undo = this.addPriority(docId, priority);
+    return this.sync.waitForDocCompleted(docId, signal).finally(() => undo());
+  }
+}

packages/common/nbstore/src/impls/broadcast-channel/awareness.ts

@@ -0,0 +1,152 @@
+import { nanoid } from 'nanoid';
+
+import { type AwarenessRecord, AwarenessStorageBase } from '../../storage';
+import { BroadcastChannelConnection } from './channel';
+
+type ChannelMessage =
+  | {
+      type: 'awareness-update';
+      docId: string;
+      bin: Uint8Array;
+      origin?: string;
+    }
+  | {
+      type: 'awareness-collect';
+      docId: string;
+      collectId: string;
+    }
+  | {
+      type: 'awareness-collect-feedback';
+      docId: string;
+      bin: Uint8Array;
+      collectId: string;
+    };
+
+interface BroadcastChannelAwarenessStorageOptions {
+  id: string;
+}
+
+export class BroadcastChannelAwarenessStorage extends AwarenessStorageBase {
+  static readonly identifier = 'BroadcastChannelAwarenessStorage';
+
+  override readonly storageType = 'awareness';
+  override readonly connection = new BroadcastChannelConnection({
+    id: this.options.id,
+  });
+  get channel() {
+    return this.connection.inner;
+  }
+
+  constructor(
+    private readonly options: BroadcastChannelAwarenessStorageOptions
+  ) {
+    super();
+  }
+
+  private readonly subscriptions = new Map<
+    string,
+    Set<{
+      onUpdate: (update: AwarenessRecord, origin?: string) => void;
+      onCollect: () => Promise<AwarenessRecord | null>;
+    }>
+  >();
+
+  override update(record: AwarenessRecord, origin?: string): Promise<void> {
+    const subscribers = this.subscriptions.get(record.docId);
+    if (subscribers) {
+      subscribers.forEach(subscriber => subscriber.onUpdate(record, origin));
+    }
+    this.channel.postMessage({
+      type: 'awareness-update',
+      docId: record.docId,
+      bin: record.bin,
+      origin,
+    } satisfies ChannelMessage);
+    return Promise.resolve();
+  }
+
+  override subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void {
+    const subscribers = this.subscriptions.get(id) ?? new Set();
+    subscribers.forEach(subscriber => {
+      subscriber
+        .onCollect()
+        .then(awareness => {
+          if (awareness) {
+            onUpdate(awareness);
+          }
+        })
+        .catch(error => {
+          console.error('error in on collect awareness', error);
+        });
+    });
+
+    const collectUniqueId = nanoid();
+
+    const onChannelMessage = (message: MessageEvent<ChannelMessage>) => {
+      if (
+        message.data.type === 'awareness-update' &&
+        message.data.docId === id
+      ) {
+        onUpdate(
+          {
+            docId: message.data.docId,
+            bin: message.data.bin,
+          },
+          message.data.origin
+        );
+      }
+      if (
+        message.data.type === 'awareness-collect' &&
+        message.data.docId === id
+      ) {
+        onCollect()
+          .then(awareness => {
+            if (awareness) {
+              this.channel.postMessage({
+                type: 'awareness-collect-feedback',
+                docId: message.data.docId,
+                bin: awareness.bin,
+                collectId: collectUniqueId,
+              } satisfies ChannelMessage);
+            }
+          })
+          .catch(error => {
+            console.error('error in on collect awareness', error);
+          });
+      }
+      if (
+        message.data.type === 'awareness-collect-feedback' &&
+        message.data.docId === id &&
+        message.data.collectId === collectUniqueId
+      ) {
+        onUpdate({
+          docId: message.data.docId,
+          bin: message.data.bin,
+        });
+      }
+    };
+
+    this.channel.addEventListener('message', onChannelMessage);
+    this.channel.postMessage({
+      type: 'awareness-collect',
+      docId: id,
+      collectId: collectUniqueId,
+    } satisfies ChannelMessage);
+
+    const subscriber = {
+      onUpdate,
+      onCollect,
+    };
+    subscribers.add(subscriber);
+    this.subscriptions.set(id, subscribers);
+
+    return () => {
+      subscribers.delete(subscriber);
+      this.channel.removeEventListener('message', onChannelMessage);
+    };
+  }
+}

packages/common/nbstore/src/impls/broadcast-channel/channel.ts

@@ -0,0 +1,21 @@
+import { AutoReconnectConnection } from '../../connection';
+
+export interface BroadcastChannelConnectionOptions {
+  id: string;
+}
+
+export class BroadcastChannelConnection extends AutoReconnectConnection<BroadcastChannel> {
+  readonly channelName = `channel:${this.opts.id}`;
+
+  constructor(private readonly opts: BroadcastChannelConnectionOptions) {
+    super();
+  }
+
+  override async doConnect() {
+    return new BroadcastChannel(this.channelName);
+  }
+
+  override doDisconnect(channel: BroadcastChannel) {
+    channel.close();
+  }
+}

packages/common/nbstore/src/impls/broadcast-channel/index.ts

@@ -0,0 +1,6 @@
+import type { StorageConstructor } from '..';
+import { BroadcastChannelAwarenessStorage } from './awareness';
+
+export const broadcastChannelStorages = [
+  BroadcastChannelAwarenessStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/cloud/awareness.ts

@@ -0,0 +1,231 @@
+import {
+  type AwarenessRecord,
+  AwarenessStorageBase,
+} from '../../storage/awareness';
+import type { SpaceType } from '../../utils/universal-id';
+import {
+  base64ToUint8Array,
+  SocketConnection,
+  SPACE_JOIN_BATCH_LIMIT,
+  uint8ArrayToBase64,
+} from './socket';
+
+interface CloudAwarenessStorageOptions {
+  isSelfHosted: boolean;
+  serverBaseUrl: string;
+  type: SpaceType;
+  id: string;
+}
+
+export class CloudAwarenessStorage extends AwarenessStorageBase {
+  static readonly identifier = 'CloudAwarenessStorage';
+
+  constructor(private readonly options: CloudAwarenessStorageOptions) {
+    super();
+  }
+
+  connection = new SocketConnection(
+    this.options.serverBaseUrl,
+    this.options.isSelfHosted
+  );
+
+  private get socket() {
+    return this.connection.inner.socket;
+  }
+
+  private readonly activeAwarenessIds = new Set<string>();
+  private readonly joinedAwarenessIds = new Set<string>();
+  private joinPromise: Promise<void> | undefined;
+
+  private joinActiveAwareness(): Promise<void> {
+    if (
+      this.connection.status !== 'connected' ||
+      this.activeAwarenessIds.size === 0
+    ) {
+      return Promise.resolve();
+    }
+
+    if (this.joinPromise) {
+      return this.joinPromise;
+    }
+
+    const batchPromise = (async () => {
+      while (this.connection.status === 'connected') {
+        await Promise.resolve();
+        const pendingIds = [...this.activeAwarenessIds].filter(
+          docId => !this.joinedAwarenessIds.has(docId)
+        );
+        if (pendingIds.length === 0) {
+          return;
+        }
+
+        for (
+          let index = 0;
+          index < pendingIds.length;
+          index += SPACE_JOIN_BATCH_LIMIT
+        ) {
+          const spaces = pendingIds
+            .slice(index, index + SPACE_JOIN_BATCH_LIMIT)
+            .map(docId => ({
+              spaceType: this.options.type,
+              spaceId: this.options.id,
+              docId,
+            }));
+          const response = await this.socket.emitWithAck('space:join-batch', {
+            spaces,
+            clientVersion: BUILD_CONFIG.appVersion,
+          });
+
+          if ('error' in response) {
+            throw new Error(
+              `Awareness join failed: ${response.error.name}: ${response.error.message}`
+            );
+          }
+          if (!response.data.success) {
+            throw new Error('Awareness join was rejected');
+          }
+        }
+
+        for (const docId of pendingIds) {
+          if (this.activeAwarenessIds.has(docId)) {
+            this.joinedAwarenessIds.add(docId);
+          }
+        }
+      }
+    })();
+
+    const sharedPromise = batchPromise.finally(() => {
+      this.joinPromise = undefined;
+    });
+    this.joinPromise = sharedPromise;
+    return sharedPromise;
+  }
+
+  override async update(record: AwarenessRecord): Promise<void> {
+    const encodedUpdate = await uint8ArrayToBase64(record.bin);
+    this.socket.emit('space:update-awareness', {
+      spaceType: this.options.type,
+      spaceId: this.options.id,
+      docId: record.docId,
+      awarenessUpdate: encodedUpdate,
+    });
+  }
+
+  override subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void {
+    this.activeAwarenessIds.add(id);
+
+    // leave awareness
+    const leave = () => {
+      this.activeAwarenessIds.delete(id);
+      this.joinedAwarenessIds.delete(id);
+      this.socket.off('space:collect-awareness', handleCollectAwareness);
+      this.socket.off(
+        'space:broadcast-awareness-update',
+        handleBroadcastAwarenessUpdate
+      );
+      if (this.connection.status !== 'connected') return;
+      this.socket.emit('space:leave-batch', {
+        spaceType: this.options.type,
+        spaceId: this.options.id,
+        docIds: [id],
+      });
+    };
+
+    // join awareness, and collect awareness from others
+    const joinAndCollect = async () => {
+      this.socket.on('space:collect-awareness', handleCollectAwareness);
+      this.socket.on(
+        'space:broadcast-awareness-update',
+        handleBroadcastAwarenessUpdate
+      );
+      await this.joinActiveAwareness();
+      if (this.connection.status !== 'connected') return;
+      this.socket.emit('space:load-awarenesses', {
+        spaceType: this.options.type,
+        spaceId: this.options.id,
+        docId: id,
+      });
+    };
+
+    const handleCollectAwareness = ({
+      spaceId,
+      spaceType,
+      docId,
+    }: {
+      spaceId: string;
+      spaceType: string;
+      docId: string;
+    }) => {
+      if (
+        spaceId === this.options.id &&
+        spaceType === this.options.type &&
+        docId === id
+      ) {
+        (async () => {
+          const record = await onCollect();
+          if (record) {
+            const encodedUpdate = await uint8ArrayToBase64(record.bin);
+            this.socket.emit('space:update-awareness', {
+              spaceType: this.options.type,
+              spaceId: this.options.id,
+              docId: record.docId,
+              awarenessUpdate: encodedUpdate,
+            });
+          }
+        })().catch(err => console.error('awareness upload failed', err));
+      }
+    };
+
+    const handleBroadcastAwarenessUpdate = ({
+      spaceType,
+      spaceId,
+      docId,
+      awarenessUpdate,
+    }: {
+      spaceType: string;
+      spaceId: string;
+      docId: string;
+      awarenessUpdate: string;
+    }) => {
+      if (
+        spaceId === this.options.id &&
+        spaceType === this.options.type &&
+        docId === id
+      ) {
+        onUpdate({
+          bin: base64ToUint8Array(awarenessUpdate),
+          docId: id,
+        });
+      }
+    };
+
+    if (this.connection.status === 'connected') {
+      joinAndCollect().catch(err =>
+        console.error('awareness join failed', err)
+      );
+    }
+
+    const unsubscribeConnectionStatusChanged = this.connection.onStatusChanged(
+      status => {
+        if (status !== 'connected') {
+          this.joinedAwarenessIds.clear();
+        }
+        if (status === 'connected') {
+          joinAndCollect().catch(err =>
+            console.error('awareness join failed', err)
+          );
+        }
+      }
+    );
+
+    return () => {
+      leave();
+
+      unsubscribeConnectionStatusChanged();
+    };
+  }
+}

packages/common/nbstore/src/impls/cloud/blob-source-registry.ts

@@ -0,0 +1,104 @@
+import type { BlobSource, SourceBlobRecord } from '../../storage';
+
+export interface BlobSourceCandidate {
+  context: 'owned' | 'workspace';
+  source: BlobSource;
+  token: object;
+}
+
+export class BlobSourceRegistry {
+  private readonly sourcesByKey = new Map<
+    string,
+    Map<string, BlobSourceCandidate>
+  >();
+  private readonly ownedEntries = new Map<
+    string,
+    { source: BlobSource; entries: SourceBlobRecord[]; token: object }
+  >();
+  private workspaceEntries: {
+    entry: SourceBlobRecord;
+    token: object;
+  }[] = [];
+
+  sourceId(source: BlobSource) {
+    return source.type === 'currentDoc'
+      ? `currentDoc\0${source.workspaceId}\0${source.docId}`
+      : `history\0${source.workspaceId}\0${source.docId}\0${source.timestampMs}`;
+  }
+
+  candidates(key: string) {
+    return [...(this.sourcesByKey.get(key)?.values() ?? [])];
+  }
+
+  hasCandidate(key: string, candidate: BlobSourceCandidate) {
+    const found =
+      this.sourcesByKey.get(key)?.get(this.candidateId(candidate))?.token ===
+      candidate.token;
+    return found;
+  }
+
+  replaceOwned(source: BlobSource, entries: SourceBlobRecord[], token: object) {
+    const id = this.sourceId(source);
+    this.ownedEntries.set(id, { source, entries, token });
+    this.rebuild();
+  }
+
+  replaceWorkspace(entries: SourceBlobRecord[], token: object) {
+    this.workspaceEntries = entries.map(entry => ({ entry, token }));
+    this.rebuild();
+  }
+
+  removeKeyCandidate(key: string, candidate: BlobSourceCandidate) {
+    const id = this.sourceId(candidate.source);
+    if (candidate.context === 'workspace') {
+      this.workspaceEntries = this.workspaceEntries.filter(
+        workspace =>
+          workspace.token !== candidate.token ||
+          workspace.entry.key !== key ||
+          this.sourceId(workspace.entry.source) !== id
+      );
+    } else {
+      const owned = this.ownedEntries.get(id);
+      if (owned?.token === candidate.token) {
+        owned.entries = owned.entries.filter(entry => entry.key !== key);
+      }
+    }
+    this.rebuild();
+  }
+
+  removeOwned(source: BlobSource) {
+    const id = this.sourceId(source);
+    this.ownedEntries.delete(id);
+    this.rebuild();
+  }
+
+  private rebuild() {
+    this.sourcesByKey.clear();
+    for (const workspace of this.workspaceEntries) {
+      this.merge(workspace.entry, 'workspace', workspace.token);
+    }
+    for (const registration of this.ownedEntries.values()) {
+      for (const entry of registration.entries) {
+        this.merge(entry, 'owned', registration.token);
+      }
+    }
+  }
+
+  private merge(
+    entry: SourceBlobRecord,
+    context: BlobSourceCandidate['context'],
+    token: object
+  ) {
+    let sources = this.sourcesByKey.get(entry.key);
+    if (!sources) {
+      sources = new Map();
+      this.sourcesByKey.set(entry.key, sources);
+    }
+    const candidate = { context, source: entry.source, token };
+    sources.set(this.candidateId(candidate), candidate);
+  }
+
+  private candidateId(candidate: BlobSourceCandidate) {
+    return `${candidate.context}\0${this.sourceId(candidate.source)}`;
+  }
+}

packages/common/nbstore/src/impls/cloud/blob-writer.ts

@@ -0,0 +1,325 @@
+import { UserFriendlyError } from '@affine/error';
+import {
+  abortBlobUploadMutation,
+  BlobUploadMethod,
+  completeBlobUploadMutation,
+  createBlobUploadMutation,
+  deleteBlobMutation,
+  getBlobUploadPartUrlQuery,
+  listBlobsQuery,
+  releaseDeletedBlobsMutation,
+  setBlobMutation,
+  workspaceBlobQuotaQuery,
+} from '@affine/graphql';
+
+import {
+  type BlobRecord,
+  OverCapacityError,
+  OverSizeError,
+} from '../../storage';
+import type { HttpConnection } from './http';
+
+const UPLOAD_REQUEST_TIMEOUT = 0;
+
+function toStrictArrayBuffer(
+  data: ArrayBuffer | ArrayBufferLike | ArrayBufferView
+): ArrayBuffer {
+  if (data instanceof ArrayBuffer) {
+    return data;
+  }
+
+  if (ArrayBuffer.isView(data)) {
+    if (data.buffer instanceof ArrayBuffer) {
+      if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) {
+        return data.buffer;
+      }
+      return data.buffer.slice(
+        data.byteOffset,
+        data.byteOffset + data.byteLength
+      );
+    }
+
+    const bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+    const copy = new Uint8Array(bytes.byteLength);
+    copy.set(bytes);
+    return copy.buffer;
+  }
+
+  const bytes = new Uint8Array(data);
+  const copy = new Uint8Array(bytes.byteLength);
+  copy.set(bytes);
+  return copy.buffer;
+}
+
+export class CloudBlobWriter {
+  private humanReadableBlobSizeLimit: string | null = null;
+  private blobSizeLimit: number | null = null;
+  private blobSizeLimitTime = 0;
+
+  constructor(
+    private readonly connection: HttpConnection,
+    private readonly workspaceId: string,
+    private readonly serverBaseUrl: string
+  ) {}
+
+  async set(blob: BlobRecord, signal?: AbortSignal) {
+    try {
+      signal?.throwIfAborted();
+      const blobSizeLimit = await this.getBlobSizeLimit(signal);
+      if (blob.data.byteLength > blobSizeLimit) {
+        throw new OverSizeError(this.humanReadableBlobSizeLimit);
+      }
+
+      const init = await this.connection.gql({
+        query: createBlobUploadMutation,
+        variables: {
+          workspaceId: this.workspaceId,
+          key: blob.key,
+          size: blob.data.byteLength,
+          mime: blob.mime,
+        },
+        context: { signal },
+      });
+
+      const upload = init.createBlobUpload;
+      if (upload.alreadyUploaded) {
+        return;
+      }
+      try {
+        if (upload.method === BlobUploadMethod.PRESIGNED) {
+          if (!upload.uploadUrl) {
+            throw new Error('Missing upload URL for presigned upload.');
+          }
+          await this.uploadViaPresigned(
+            upload.uploadUrl,
+            upload.headers,
+            blob.data,
+            signal
+          );
+          await this.completeUpload(blob.key, undefined, undefined, signal);
+          return;
+        }
+
+        if (upload.method === BlobUploadMethod.MULTIPART) {
+          if (!upload.uploadId || !upload.partSize) {
+            throw new Error(
+              'Missing upload ID or part size for multipart upload.'
+            );
+          }
+          const parts = await this.uploadViaMultipart(
+            blob.key,
+            upload.uploadId,
+            upload.partSize,
+            blob.data,
+            upload.uploadedParts,
+            signal
+          );
+          await this.completeUpload(blob.key, upload.uploadId, parts, signal);
+          return;
+        }
+      } catch (error) {
+        if (upload.method === BlobUploadMethod.MULTIPART && upload.uploadId) {
+          await this.tryAbortMultipartUpload(blob.key, upload.uploadId);
+        }
+        signal?.throwIfAborted();
+        if (UserFriendlyError.fromAny(error).is('CONTENT_TOO_LARGE')) {
+          throw error;
+        }
+      }
+
+      await this.uploadViaGraphql(blob, signal);
+    } catch (err) {
+      const userFriendlyError = UserFriendlyError.fromAny(err);
+      if (userFriendlyError.is('STORAGE_QUOTA_EXCEEDED')) {
+        throw new OverCapacityError();
+      }
+      if (userFriendlyError.is('BLOB_QUOTA_EXCEEDED')) {
+        throw new OverSizeError(this.humanReadableBlobSizeLimit);
+      }
+      if (userFriendlyError.is('CONTENT_TOO_LARGE')) {
+        throw new OverSizeError(
+          null,
+          'Upload stopped by network proxy: file size exceeds the set limit.'
+        );
+      }
+      throw err;
+    }
+  }
+
+  async delete(key: string, permanently: boolean) {
+    await this.connection.gql({
+      query: deleteBlobMutation,
+      variables: { workspaceId: this.workspaceId, key, permanently },
+    });
+  }
+
+  async release() {
+    await this.connection.gql({
+      query: releaseDeletedBlobsMutation,
+      variables: { workspaceId: this.workspaceId },
+    });
+  }
+
+  async listManageable() {
+    const result = await this.connection.gql({
+      query: listBlobsQuery,
+      variables: { workspaceId: this.workspaceId },
+    });
+    return result.workspace.blobs.map(blob => ({
+      ...blob,
+      createdAt: blob.createdAt ? new Date(blob.createdAt) : undefined,
+    }));
+  }
+
+  private async uploadViaGraphql(blob: BlobRecord, signal?: AbortSignal) {
+    await this.connection.gql({
+      query: setBlobMutation,
+      variables: {
+        workspaceId: this.workspaceId,
+        blob: new File([toStrictArrayBuffer(blob.data)], blob.key, {
+          type: blob.mime,
+        }),
+      },
+      context: { signal },
+      timeout: UPLOAD_REQUEST_TIMEOUT,
+    });
+  }
+
+  private async uploadViaPresigned(
+    uploadUrl: string,
+    headers: Record<string, string> | null | undefined,
+    data: Uint8Array,
+    signal?: AbortSignal
+  ) {
+    const res = await this.fetchUpload(uploadUrl, {
+      method: 'PUT',
+      headers: headers ?? undefined,
+      body: toStrictArrayBuffer(data),
+      signal,
+    });
+    if (!res.ok) {
+      throw new Error(`Presigned upload failed with status ${res.status}`);
+    }
+  }
+
+  private async uploadViaMultipart(
+    key: string,
+    uploadId: string,
+    partSize: number,
+    data: Uint8Array,
+    uploadedParts: { partNumber: number; etag: string }[] | null | undefined,
+    signal?: AbortSignal
+  ) {
+    const partsMap = new Map<number, string>();
+    for (const part of uploadedParts ?? []) {
+      partsMap.set(part.partNumber, part.etag);
+    }
+    const totalParts = Math.ceil(data.byteLength / partSize);
+
+    for (let partNumber = 1; partNumber <= totalParts; partNumber += 1) {
+      if (partsMap.has(partNumber)) {
+        continue;
+      }
+      const start = (partNumber - 1) * partSize;
+      const chunk = data.subarray(
+        start,
+        Math.min(start + partSize, data.byteLength)
+      );
+      const part = await this.connection.gql({
+        query: getBlobUploadPartUrlQuery,
+        variables: {
+          workspaceId: this.workspaceId,
+          key,
+          uploadId,
+          partNumber,
+        },
+        context: { signal },
+      });
+      const res = await this.fetchUpload(
+        part.workspace.blobUploadPartUrl.uploadUrl,
+        {
+          method: 'PUT',
+          headers: part.workspace.blobUploadPartUrl.headers ?? undefined,
+          body: toStrictArrayBuffer(chunk),
+          signal,
+        }
+      );
+      if (!res.ok) {
+        throw new Error(
+          `Multipart upload failed at part ${partNumber} with status ${res.status}`
+        );
+      }
+      const etag = res.headers.get('etag');
+      if (!etag) {
+        throw new Error(`Missing ETag for part ${partNumber}.`);
+      }
+      partsMap.set(partNumber, etag);
+    }
+
+    if (partsMap.size !== totalParts) {
+      throw new Error('Multipart upload has missing parts.');
+    }
+    return [...partsMap.entries()]
+      .sort((left, right) => left[0] - right[0])
+      .map(([partNumber, etag]) => ({ partNumber, etag }));
+  }
+
+  private async completeUpload(
+    key: string,
+    uploadId: string | undefined,
+    parts: { partNumber: number; etag: string }[] | undefined,
+    signal?: AbortSignal
+  ) {
+    await this.connection.gql({
+      query: completeBlobUploadMutation,
+      variables: { workspaceId: this.workspaceId, key, uploadId, parts },
+      context: { signal },
+      timeout: UPLOAD_REQUEST_TIMEOUT,
+    });
+  }
+
+  private async tryAbortMultipartUpload(key: string, uploadId: string) {
+    try {
+      await this.connection.gql({
+        query: abortBlobUploadMutation,
+        variables: { workspaceId: this.workspaceId, key, uploadId },
+      });
+    } catch {}
+  }
+
+  private async fetchUpload(input: string, init: RequestInit) {
+    const res = await globalThis.fetch(
+      new URL(input, this.serverBaseUrl).toString(),
+      init
+    );
+    if (res.status === 413) {
+      throw new UserFriendlyError({
+        status: 413,
+        code: 'CONTENT_TOO_LARGE',
+        type: 'CONTENT_TOO_LARGE',
+        name: 'CONTENT_TOO_LARGE',
+        message: 'Content too large',
+      });
+    }
+    return res;
+  }
+
+  private async getBlobSizeLimit(signal?: AbortSignal) {
+    if (
+      this.blobSizeLimit !== null &&
+      Date.now() - this.blobSizeLimitTime < 120 * 1000
+    ) {
+      return this.blobSizeLimit;
+    }
+    const res = await this.connection.gql({
+      query: workspaceBlobQuotaQuery,
+      variables: { id: this.workspaceId },
+      context: { signal },
+    });
+    this.humanReadableBlobSizeLimit =
+      res.workspace.quota.humanReadable.blobLimit;
+    this.blobSizeLimit = res.workspace.quota.blobLimit;
+    this.blobSizeLimitTime = Date.now();
+    return this.blobSizeLimit;
+  }
+}

packages/common/nbstore/src/impls/cloud/blob.ts

@@ -0,0 +1,358 @@
+import { UserFriendlyError } from '@affine/error';
+
+import {
+  type BlobRecord,
+  type BlobSource,
+  BlobStorageBase,
+  type SourceBlobRecord,
+} from '../../storage';
+import {
+  type BlobSourceCandidate,
+  BlobSourceRegistry,
+} from './blob-source-registry';
+import { CloudBlobWriter } from './blob-writer';
+import { HttpConnection } from './http';
+
+interface CloudBlobStorageOptions {
+  serverBaseUrl: string;
+  id: string;
+}
+
+type SourceRegistration = {
+  controller: AbortController;
+  dirty: boolean;
+  promise?: Promise<void>;
+  source: BlobSource;
+};
+
+const MAX_PENDING_SOURCE_REGISTRATIONS = 32;
+
+export function sourceScopedBlobUrl(
+  workspaceId: string,
+  key: string,
+  source: BlobSource
+) {
+  const query = new URLSearchParams({
+    sourceType: source.type,
+    docId: source.docId,
+  });
+  if (source.type === 'history') {
+    query.set('timestampMs', String(source.timestampMs));
+  }
+  return `/api/workspaces/${encodeURIComponent(workspaceId)}/blobs/v1/${encodeURIComponent(key)}?${query}`;
+}
+
+export class CloudBlobStorage extends BlobStorageBase {
+  static readonly identifier = 'CloudBlobStorage';
+  override readonly isReadonly = false;
+
+  constructor(private readonly options: CloudBlobStorageOptions) {
+    super();
+  }
+
+  readonly connection = new HttpConnection(this.options.serverBaseUrl);
+  private readonly writer = new CloudBlobWriter(
+    this.connection,
+    this.options.id,
+    this.options.serverBaseUrl
+  );
+
+  private readonly sources = new BlobSourceRegistry();
+  private readonly sourceRegistrations = new Map<string, SourceRegistration>();
+  private readonly pendingSourceRegistrations = new Set<SourceRegistration>();
+
+  private sourceId(source: BlobSource) {
+    return this.sources.sourceId(source);
+  }
+
+  private sourceQuery(source: BlobSource) {
+    const query = new URLSearchParams({
+      sourceType: source.type,
+      docId: source.docId,
+    });
+    if (source.type === 'history') {
+      query.set('timestampMs', String(source.timestampMs));
+    }
+    return query;
+  }
+
+  private removeKeyCandidate(key: string, candidate: BlobSourceCandidate) {
+    this.sources.removeKeyCandidate(key, candidate);
+  }
+
+  private removeSource(source: BlobSource) {
+    this.sources.removeOwned(source);
+  }
+
+  private replaceSource(
+    source: BlobSource,
+    entries: SourceBlobRecord[],
+    registration: SourceRegistration
+  ) {
+    this.sources.replaceOwned(source, entries, registration);
+  }
+
+  private isRegistrationCurrent(registration: SourceRegistration) {
+    return (
+      this.sourceRegistrations.get(this.sourceId(registration.source)) ===
+      registration
+    );
+  }
+
+  private isCandidateCurrent(key: string, candidate: BlobSourceCandidate) {
+    return this.sources.hasCandidate(key, candidate);
+  }
+
+  async registerSource(
+    source: BlobSource,
+    signal?: AbortSignal
+  ): Promise<void> {
+    const id = this.sourceId(source);
+    const existing = this.sourceRegistrations.get(id);
+    if (existing) {
+      return this.refreshSource(existing, signal);
+    }
+    if (
+      this.pendingSourceRegistrations.size >= MAX_PENDING_SOURCE_REGISTRATIONS
+    ) {
+      throw new Error('Blob source registration budget exceeded');
+    }
+    const registration = {
+      controller: new AbortController(),
+      dirty: false,
+      source,
+    } satisfies SourceRegistration;
+    this.sourceRegistrations.set(id, registration);
+    return this.refreshSource(registration, signal);
+  }
+
+  private refreshSource(
+    registration: SourceRegistration,
+    signal?: AbortSignal
+  ): Promise<void> {
+    if (registration.promise) {
+      registration.dirty = true;
+      return registration.promise;
+    }
+    if (
+      this.pendingSourceRegistrations.size >= MAX_PENDING_SOURCE_REGISTRATIONS
+    ) {
+      throw new Error('Blob source registration budget exceeded');
+    }
+    const registrationSignal = signal
+      ? AbortSignal.any([signal, registration.controller.signal])
+      : registration.controller.signal;
+    registration.promise = this.loadSource(
+      registration.source,
+      registration,
+      registrationSignal
+    ).finally(() => {
+      this.pendingSourceRegistrations.delete(registration);
+      registration.promise = undefined;
+    });
+    this.pendingSourceRegistrations.add(registration);
+    return registration.promise;
+  }
+
+  async unregisterSource(source: BlobSource) {
+    const id = this.sourceId(source);
+    const registration = this.sourceRegistrations.get(id);
+    if (registration) {
+      this.sourceRegistrations.delete(id);
+      registration.controller.abort();
+    }
+    this.removeSource(source);
+    await registration?.promise?.catch(() => {});
+  }
+
+  private async loadSource(
+    source: BlobSource,
+    registration: SourceRegistration,
+    signal?: AbortSignal
+  ) {
+    do {
+      registration.dirty = false;
+      try {
+        const res = await this.connection.fetch(
+          `/api/workspaces/${encodeURIComponent(this.options.id)}/blob-manifest/v1?${this.sourceQuery(source)}`,
+          { cache: 'no-store', signal }
+        );
+        if (!res.ok) {
+          throw new Error(
+            `Blob source manifest failed with status ${res.status}`
+          );
+        }
+        const manifest = (await res.json()) as {
+          version: 1;
+          entries: SourceBlobRecord[];
+        };
+        if (this.isRegistrationCurrent(registration)) {
+          this.replaceSource(source, manifest.entries, registration);
+        }
+      } catch (error) {
+        if (this.isRegistrationCurrent(registration)) {
+          this.removeSource(source);
+        }
+        throw error;
+      }
+    } while (registration.dirty && this.isRegistrationCurrent(registration));
+  }
+
+  async *readableSources(signal?: AbortSignal) {
+    let cursor: string | undefined;
+    try {
+      do {
+        const query = new URLSearchParams({ limit: '100' });
+        if (cursor) query.set('cursor', cursor);
+        const res = await this.connection.fetch(
+          `/api/workspaces/${encodeURIComponent(this.options.id)}/readable-blob-manifest/v1?${query}`,
+          { cache: 'no-store', signal }
+        );
+        if (!res.ok) {
+          throw new Error(
+            `Readable blob manifest failed with status ${res.status}`
+          );
+        }
+        const manifest = (await res.json()) as {
+          version: 1;
+          entries: SourceBlobRecord[];
+          nextCursor?: string;
+        };
+        for (const entry of manifest.entries) {
+          this.sources.replaceWorkspace([entry], {});
+          yield entry;
+        }
+        cursor = manifest.nextCursor;
+      } while (cursor);
+    } finally {
+      this.sources.replaceWorkspace([], {});
+    }
+  }
+
+  override async get(key: string, signal?: AbortSignal, source?: BlobSource) {
+    signal?.throwIfAborted();
+    let candidates = this.sourceCandidates(key, source);
+    if (candidates.length === 0) {
+      const failed = await this.refreshRegisteredSources(source, signal);
+      signal?.throwIfAborted();
+      candidates = this.sourceCandidates(key, source);
+      if (candidates.length === 0) {
+        if (failed) throw failed.reason;
+        return null;
+      }
+    }
+
+    const attempt = async (candidates: BlobSourceCandidate[]) => {
+      for (const candidate of candidates) {
+        if (!this.isCandidateCurrent(key, candidate)) {
+          continue;
+        }
+        let res: Response;
+        try {
+          res = await this.connection.fetch(
+            sourceScopedBlobUrl(this.options.id, key, candidate.source),
+            { cache: 'no-store', signal }
+          );
+        } catch (error) {
+          if (
+            error instanceof UserFriendlyError &&
+            (error.status === 403 || error.status === 404)
+          ) {
+            if (this.isCandidateCurrent(key, candidate)) {
+              this.removeKeyCandidate(key, candidate);
+            }
+            continue;
+          }
+          throw error;
+        }
+        if (!this.isCandidateCurrent(key, candidate)) {
+          continue;
+        }
+        if (res.status === 403 || res.status === 404) {
+          if (this.isCandidateCurrent(key, candidate)) {
+            this.removeKeyCandidate(key, candidate);
+          }
+          continue;
+        }
+        if (!res.ok) {
+          throw new Error(`Blob download failed with status ${res.status}`);
+        }
+        const blob = await res.blob();
+        const data = new Uint8Array(await blob.arrayBuffer());
+        if (!this.isCandidateCurrent(key, candidate)) {
+          continue;
+        }
+        return {
+          key,
+          data,
+          mime: blob.type,
+          size: blob.size,
+          createdAt: new Date(res.headers.get('last-modified') || Date.now()),
+        };
+      }
+      return null;
+    };
+
+    return attempt(candidates);
+  }
+
+  private sourceCandidates(key: string, source?: BlobSource) {
+    return this.sources
+      .candidates(key)
+      .filter(
+        candidate =>
+          !source || this.sourceId(candidate.source) === this.sourceId(source)
+      );
+  }
+
+  private async refreshRegisteredSources(
+    source?: BlobSource,
+    signal?: AbortSignal
+  ) {
+    const registrations = source
+      ? [this.sourceRegistrations.get(this.sourceId(source))].filter(
+          (registration): registration is SourceRegistration => !!registration
+        )
+      : [...this.sourceRegistrations.values()];
+    if (registrations.length === 0) {
+      throw new Error('Blob source context is required');
+    }
+    let failed: PromiseRejectedResult | undefined;
+    for (
+      let offset = 0;
+      offset < registrations.length;
+      offset += MAX_PENDING_SOURCE_REGISTRATIONS
+    ) {
+      const results = await Promise.allSettled(
+        registrations
+          .slice(offset, offset + MAX_PENDING_SOURCE_REGISTRATIONS)
+          .map(
+            async registration =>
+              registration.promise ?? this.refreshSource(registration, signal)
+          )
+      );
+      failed ??= results.find(result => result.status === 'rejected');
+    }
+    return failed;
+  }
+
+  override set(blob: BlobRecord, signal?: AbortSignal) {
+    return this.writer.set(blob, signal);
+  }
+
+  override delete(key: string, permanently: boolean) {
+    return this.writer.delete(key, permanently);
+  }
+
+  override release() {
+    return this.writer.release();
+  }
+
+  override async list(): Promise<never> {
+    throw new Error('Workspace blob inventory is unavailable');
+  }
+
+  async listManageable() {
+    return this.writer.listManageable();
+  }
+}

packages/common/nbstore/src/impls/cloud/doc-static.ts

@@ -0,0 +1,102 @@
+import {
+  type DocClock,
+  type DocClocks,
+  type DocRecord,
+  DocStorageBase,
+  type DocStorageOptions,
+  type DocUpdate,
+} from '../../storage';
+import { HttpConnection } from './http';
+
+interface CloudDocStorageOptions extends DocStorageOptions {
+  serverBaseUrl: string;
+  publicRootDocId?: string;
+}
+
+const isShareModePrivateSystemDoc = (docId: string) =>
+  docId.startsWith('db$') || docId.startsWith('userdata$');
+
+export class StaticCloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
+  static readonly identifier = 'StaticCloudDocStorage';
+
+  constructor(options: CloudDocStorageOptions) {
+    super({ ...options, readonlyMode: true });
+  }
+
+  override connection = new HttpConnection(this.options.serverBaseUrl);
+  override async pushDocUpdate(
+    update: DocUpdate,
+    _origin?: string
+  ): Promise<DocClock> {
+    // http is readonly
+    return { docId: update.docId, timestamp: new Date() };
+  }
+  override async getDocTimestamp(docId: string): Promise<DocClock | null> {
+    // http doesn't support this, so we just return a new timestamp
+    return {
+      docId,
+      timestamp: new Date(),
+    };
+  }
+  override async getDocTimestamps(): Promise<DocClocks> {
+    // http doesn't support this
+    return {};
+  }
+  override deleteDoc(_docId: string): Promise<void> {
+    // http is readonly
+    return Promise.resolve();
+  }
+  protected override async getDocSnapshot(
+    docId: string
+  ): Promise<DocRecord | null> {
+    try {
+      if (
+        this.options.publicRootDocId &&
+        docId !== this.spaceId &&
+        isShareModePrivateSystemDoc(docId)
+      ) {
+        return null;
+      }
+
+      const path =
+        docId === this.spaceId && this.options.publicRootDocId
+          ? `/api/workspaces/${this.spaceId}/public-docs/${this.options.publicRootDocId}/root-doc`
+          : this.options.publicRootDocId
+            ? `/api/workspaces/${this.spaceId}/public-docs/${docId}`
+            : `/api/workspaces/${this.spaceId}/docs/${docId}`;
+      const arrayBuffer = await this.connection.fetchArrayBuffer(path, {
+        priority: 'high',
+        headers: {
+          Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer
+        },
+      });
+      if (!arrayBuffer) {
+        return null;
+      }
+      return {
+        docId: docId,
+        bin: new Uint8Array(arrayBuffer),
+        timestamp: new Date(),
+      };
+    } catch (error) {
+      console.error(error);
+      return null;
+    }
+  }
+  protected override setDocSnapshot(
+    _snapshot: DocRecord,
+    _prevSnapshot: DocRecord | null
+  ): Promise<boolean> {
+    // http is readonly
+    return Promise.resolve(false);
+  }
+  protected override getDocUpdates(_docId: string): Promise<DocRecord[]> {
+    return Promise.resolve([]);
+  }
+  protected override markUpdatesMerged(
+    _docId: string,
+    _updates: DocRecord[]
+  ): Promise<number> {
+    return Promise.resolve(0);
+  }
+}

packages/common/nbstore/src/impls/cloud/doc.ts

@@ -0,0 +1,388 @@
+import type { Socket } from 'socket.io-client';
+
+import {
+  type DocClock,
+  type DocClocks,
+  DocStorageBase,
+  type DocStorageOptions,
+  type DocUpdate,
+} from '../../storage';
+import { getIdConverter, type IdConverter } from '../../utils/id-converter';
+import type { SpaceType } from '../../utils/universal-id';
+import {
+  base64ToUint8Array,
+  type ServerEventsMap,
+  SocketConnection,
+  SPACE_JOIN_BATCH_LIMIT,
+  uint8ArrayToBase64,
+} from './socket';
+
+interface CloudDocStorageOptions extends DocStorageOptions {
+  serverBaseUrl: string;
+  isSelfHosted: boolean;
+  type: SpaceType;
+}
+
+function createWebsocketError(error: { name: string; message: string }) {
+  const err = new Error(error.message);
+  err.name = error.name;
+  return err;
+}
+
+export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
+  static readonly identifier = 'CloudDocStorage';
+
+  get socket() {
+    return this.connection.inner.socket;
+  }
+  get idConverter() {
+    if (!this.connection.idConverter) {
+      throw new Error('Id converter not initialized');
+    }
+    return this.connection.idConverter;
+  }
+  readonly spaceType = this.options.type;
+
+  onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'] = message => {
+    if (
+      this.spaceType !== message.spaceType ||
+      this.spaceId !== message.spaceId
+    ) {
+      return;
+    }
+
+    const docId = this.idConverter.oldIdToNewId(message.docId);
+    this.serverDocTimestamps.set(docId, message.timestamp);
+    for (const update of message.updates) {
+      this.emit('update', {
+        docId,
+        bin: base64ToUint8Array(update),
+        timestamp: new Date(message.timestamp),
+        editor: message.editor,
+      });
+    }
+  };
+
+  onServerInvalidation: ServerEventsMap['space:broadcast-doc-invalidation'] =
+    message => {
+      if (
+        this.spaceType !== message.spaceType ||
+        this.spaceId !== message.spaceId
+      ) {
+        return;
+      }
+
+      this.invalidationPending = true;
+      this.scheduleInvalidationRepair();
+    };
+
+  private readonly serverDocTimestamps = new Map<string, number>();
+  private invalidationTimer?: ReturnType<typeof setTimeout>;
+  private invalidationRepair?: Promise<void>;
+  private invalidationPending = false;
+
+  private scheduleInvalidationRepair() {
+    if (this.invalidationTimer) {
+      clearTimeout(this.invalidationTimer);
+    }
+    this.invalidationTimer = setTimeout(() => {
+      this.invalidationTimer = undefined;
+      if (this.invalidationRepair) {
+        return;
+      }
+      this.invalidationPending = false;
+      this.invalidationRepair = this.repairInvalidatedDocs()
+        .catch(error => {
+          console.error('failed to repair invalidated docs', error);
+        })
+        .finally(() => {
+          this.invalidationRepair = undefined;
+          if (this.invalidationPending) {
+            this.scheduleInvalidationRepair();
+          }
+        });
+    }, 50);
+  }
+
+  private async repairInvalidatedDocs() {
+    const timestamps = await this.getDocTimestamps();
+    for (const [docId, timestamp] of Object.entries(timestamps)) {
+      const newDocId = this.idConverter.oldIdToNewId(docId);
+      const timestampValue = timestamp.getTime();
+      const previous = this.serverDocTimestamps.get(newDocId);
+      if (previous !== undefined && previous >= timestampValue) {
+        continue;
+      }
+      this.serverDocTimestamps.set(newDocId, timestampValue);
+      this.emit('update', {
+        docId: newDocId,
+        bin: new Uint8Array(),
+        timestamp,
+      });
+    }
+  }
+
+  readonly connection = new CloudDocStorageConnection(
+    this.options,
+    this.onServerUpdates,
+    this.onServerInvalidation
+  );
+
+  override async getDocSnapshot(docId: string) {
+    const response = await this.socket.emitWithAck('space:load-doc', {
+      spaceType: this.spaceType,
+      spaceId: this.spaceId,
+      docId: await this.connection.joinDoc(
+        this.idConverter.newIdToOldId(docId)
+      ),
+    });
+
+    if ('error' in response) {
+      if (response.error.name === 'DOC_NOT_FOUND') {
+        return null;
+      }
+      // TODO: use [UserFriendlyError]
+      throw createWebsocketError(response.error);
+    }
+
+    return {
+      docId,
+      bin: base64ToUint8Array(response.data.missing),
+      timestamp: new Date(response.data.timestamp),
+    };
+  }
+
+  override async getDocDiff(docId: string, state?: Uint8Array) {
+    const response = await this.socket.emitWithAck('space:load-doc', {
+      spaceType: this.spaceType,
+      spaceId: this.spaceId,
+      docId: await this.connection.joinDoc(
+        this.idConverter.newIdToOldId(docId)
+      ),
+      stateVector: state ? await uint8ArrayToBase64(state) : void 0,
+    });
+
+    if ('error' in response) {
+      if (response.error.name === 'DOC_NOT_FOUND') {
+        return null;
+      }
+      // TODO: use [UserFriendlyError]
+      throw createWebsocketError(response.error);
+    }
+
+    return {
+      docId,
+      missing: base64ToUint8Array(response.data.missing),
+      state: base64ToUint8Array(response.data.state),
+      timestamp: new Date(response.data.timestamp),
+    };
+  }
+
+  override async pushDocUpdate(update: DocUpdate) {
+    const response = await this.socket.emitWithAck('space:push-doc-update', {
+      spaceType: this.spaceType,
+      spaceId: this.spaceId,
+      docId: await this.connection.joinDoc(
+        this.idConverter.newIdToOldId(update.docId)
+      ),
+      update: await uint8ArrayToBase64(update.bin),
+    });
+
+    if ('error' in response) {
+      // TODO(@forehalo): use [UserFriendlyError]
+      throw createWebsocketError(response.error);
+    }
+
+    return {
+      docId: update.docId,
+      timestamp: new Date(response.data.timestamp),
+    };
+  }
+
+  /**
+   * Just a rough implementation, cloud doc storage should not need this method.
+   */
+  override async getDocTimestamp(docId: string): Promise<DocClock | null> {
+    const response = await this.socket.emitWithAck('space:load-doc', {
+      spaceType: this.spaceType,
+      spaceId: this.spaceId,
+      docId: await this.connection.joinDoc(
+        this.idConverter.newIdToOldId(docId)
+      ),
+    });
+
+    if ('error' in response) {
+      // TODO: use [UserFriendlyError]
+      throw createWebsocketError(response.error);
+    }
+
+    return {
+      docId,
+      timestamp: new Date(response.data.timestamp),
+    };
+  }
+
+  override async getDocTimestamps(after?: Date) {
+    const response = await this.socket.emitWithAck(
+      'space:load-doc-timestamps',
+      {
+        spaceType: this.spaceType,
+        spaceId: this.spaceId,
+        timestamp: after ? after.getTime() : undefined,
+      }
+    );
+
+    if ('error' in response) {
+      // TODO(@forehalo): use [UserFriendlyError]
+      throw createWebsocketError(response.error);
+    }
+
+    return Object.entries(response.data).reduce((ret, [docId, timestamp]) => {
+      ret[this.idConverter.oldIdToNewId(docId)] = new Date(timestamp);
+      return ret;
+    }, {} as DocClocks);
+  }
+
+  override async deleteDoc(docId: string) {
+    const response = await this.socket.emitWithAck('space:delete-doc', {
+      spaceType: this.spaceType,
+      spaceId: this.spaceId,
+      docId: this.idConverter.newIdToOldId(docId),
+    });
+
+    if ('error' in response) {
+      // TODO(@forehalo): use [UserFriendlyError]
+      throw createWebsocketError(response.error);
+    }
+  }
+
+  async applyDocLifecycle(
+    docId: string,
+    lifecycle: 'trash' | 'restore' | 'delete'
+  ) {
+    const response = await this.socket.emitWithAck('space:doc-lifecycle', {
+      spaceType: this.spaceType,
+      spaceId: this.spaceId,
+      docId: this.idConverter.newIdToOldId(docId),
+      lifecycle,
+    });
+    if ('error' in response) {
+      throw createWebsocketError(response.error);
+    }
+    return {
+      rootUpdate: base64ToUint8Array(response.data.rootUpdate),
+      timestamp: new Date(response.data.timestamp),
+    };
+  }
+
+  protected async setDocSnapshot() {
+    return false;
+  }
+  protected async getDocUpdates() {
+    return [];
+  }
+  protected async markUpdatesMerged() {
+    return 0;
+  }
+}
+
+class CloudDocStorageConnection extends SocketConnection {
+  constructor(
+    private readonly options: CloudDocStorageOptions,
+    private readonly onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'],
+    private readonly onServerInvalidation: ServerEventsMap['space:broadcast-doc-invalidation']
+  ) {
+    super(options.serverBaseUrl, options.isSelfHosted);
+  }
+
+  idConverter: IdConverter | null = null;
+  private readonly joinedDocIds = new Set<string>();
+
+  async joinDoc(docId: string, socket: Socket = this.inner.socket) {
+    const res = await socket.emitWithAck('space:join-batch', {
+      spaces: [
+        { spaceType: this.options.type, spaceId: this.options.id, docId },
+      ],
+      clientVersion: BUILD_CONFIG.appVersion,
+    });
+    if ('error' in res) throw createWebsocketError(res.error);
+    if (!res.data.success) throw new Error('Space join was rejected');
+    this.joinedDocIds.add(docId);
+    return docId;
+  }
+
+  override async doConnect(signal?: AbortSignal) {
+    const { socket, disconnect } = await super.doConnect(signal);
+
+    try {
+      await this.joinDoc(this.options.id, socket);
+
+      if (!this.idConverter) {
+        this.idConverter = await this.getIdConverter(socket);
+      }
+
+      socket.on('space:broadcast-doc-updates', this.onServerUpdates);
+      socket.on('space:broadcast-doc-invalidation', this.onServerInvalidation);
+
+      return { socket, disconnect };
+    } catch (e) {
+      this.doDisconnect({ socket, disconnect });
+      throw e;
+    }
+  }
+
+  override doDisconnect({
+    socket,
+    disconnect,
+  }: {
+    socket: Socket;
+    disconnect: () => void;
+  }) {
+    const docIds = [...this.joinedDocIds];
+    this.joinedDocIds.clear();
+    for (
+      let offset = 0;
+      offset < docIds.length;
+      offset += SPACE_JOIN_BATCH_LIMIT
+    ) {
+      socket.emit('space:leave-batch', {
+        spaceType: this.options.type,
+        spaceId: this.options.id,
+        docIds: docIds.slice(offset, offset + SPACE_JOIN_BATCH_LIMIT),
+      });
+    }
+    socket.emit('space:leave-batch', {
+      spaceType: this.options.type,
+      spaceId: this.options.id,
+      docIds: [],
+    });
+    socket.off('space:broadcast-doc-updates', this.onServerUpdates);
+    socket.off('space:broadcast-doc-invalidation', this.onServerInvalidation);
+    super.doDisconnect({ socket, disconnect });
+  }
+
+  async getIdConverter(socket: Socket) {
+    return getIdConverter(
+      {
+        getDocBuffer: async id => {
+          const response = await socket.emitWithAck('space:load-doc', {
+            spaceType: this.options.type,
+            spaceId: this.options.id,
+            docId: id,
+          });
+
+          if ('error' in response) {
+            if (response.error.name === 'DOC_NOT_FOUND') {
+              return null;
+            }
+            // TODO: use [UserFriendlyError]
+            throw createWebsocketError(response.error);
+          }
+
+          return base64ToUint8Array(response.data.missing);
+        },
+      },
+      this.options.id
+    );
+  }
+}

packages/common/nbstore/src/impls/cloud/http.ts

@@ -0,0 +1,98 @@
+import { UserFriendlyError } from '@affine/error';
+import { gqlFetcherFactory } from '@affine/graphql';
+
+import { DummyConnection } from '../../connection';
+
+export class HttpConnection extends DummyConnection {
+  readonly fetch = async (
+    input: string,
+    init?: RequestInit & { timeout?: number }
+  ) => {
+    const externalSignal = init?.signal;
+    if (externalSignal?.aborted) {
+      throw externalSignal.reason;
+    }
+
+    const abortController = new AbortController();
+    const signal = externalSignal
+      ? AbortSignal.any([externalSignal, abortController.signal])
+      : abortController.signal;
+
+    const timeout = init?.timeout ?? 15000;
+    const timeoutId =
+      timeout > 0
+        ? setTimeout(() => {
+            abortController.abort(new Error('request timeout'));
+          }, timeout)
+        : undefined;
+
+    try {
+      const res = await globalThis
+        .fetch(new URL(input, this.serverBaseUrl), {
+          ...init,
+          signal,
+          headers: {
+            ...this.requestHeaders,
+            ...init?.headers,
+            'x-affine-version': BUILD_CONFIG.appVersion,
+          },
+        })
+        .catch(err => {
+          externalSignal?.throwIfAborted();
+          throw new UserFriendlyError({
+            status: 504,
+            code: 'NETWORK_ERROR',
+            type: 'NETWORK_ERROR',
+            name: 'NETWORK_ERROR',
+            message: `Network error: ${err.message}`,
+            stacktrace: err.stack,
+          });
+        });
+      if (!res.ok && res.status !== 404) {
+        if (res.status === 413) {
+          throw new UserFriendlyError({
+            status: 413,
+            code: 'CONTENT_TOO_LARGE',
+            type: 'CONTENT_TOO_LARGE',
+            name: 'CONTENT_TOO_LARGE',
+            message: 'Content too large',
+          });
+        } else if (
+          res.headers.get('Content-Type')?.startsWith('application/json')
+        ) {
+          throw UserFriendlyError.fromAny(await res.json());
+        } else {
+          throw UserFriendlyError.fromAny(await res.text());
+        }
+      }
+      return res;
+    } finally {
+      if (timeoutId) clearTimeout(timeoutId);
+    }
+  };
+
+  readonly fetchArrayBuffer = async (input: string, init?: RequestInit) => {
+    const res = await this.fetch(input, init);
+    if (res.status === 404) {
+      // 404
+      return null;
+    }
+    try {
+      return await res.arrayBuffer();
+    } catch (err) {
+      throw new Error('fetch download error: ' + err);
+    }
+  };
+
+  readonly gql = gqlFetcherFactory(
+    new URL('/graphql', this.serverBaseUrl).href,
+    this.fetch
+  );
+
+  constructor(
+    private readonly serverBaseUrl: string,
+    private readonly requestHeaders?: Record<string, string>
+  ) {
+    super();
+  }
+}

packages/common/nbstore/src/impls/cloud/index.ts

@@ -0,0 +1,21 @@
+import type { StorageConstructor } from '..';
+import { CloudAwarenessStorage } from './awareness';
+import { CloudBlobStorage } from './blob';
+import { CloudDocStorage } from './doc';
+import { StaticCloudDocStorage } from './doc-static';
+import { CloudIndexerStorage } from './indexer';
+
+export * from './awareness';
+export * from './blob';
+export * from './doc';
+export * from './doc-static';
+export * from './indexer';
+export * from './socket';
+
+export const cloudStorages = [
+  CloudDocStorage,
+  StaticCloudDocStorage,
+  CloudBlobStorage,
+  CloudAwarenessStorage,
+  CloudIndexerStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/cloud/indexer.ts

@@ -0,0 +1,150 @@
+import {
+  type AggregateInput,
+  indexerAggregateQuery,
+  indexerSearchQuery,
+  type SearchInput,
+} from '@affine/graphql';
+import { Observable } from 'rxjs';
+
+import {
+  type AggregateOptions,
+  type AggregateResult,
+  type IndexerDocument,
+  type IndexerSchema,
+  IndexerStorageBase,
+  type Query,
+  type SearchOptions,
+  type SearchResult,
+} from '../../storage/indexer';
+import { HttpConnection } from './http';
+
+interface CloudIndexerStorageOptions {
+  serverBaseUrl: string;
+  id: string;
+}
+
+export class CloudIndexerStorage extends IndexerStorageBase {
+  static readonly identifier = 'CloudIndexerStorage';
+  readonly isReadonly = true;
+  readonly connection = new HttpConnection(this.options.serverBaseUrl);
+
+  constructor(private readonly options: CloudIndexerStorageOptions) {
+    super();
+  }
+
+  override async search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>> {
+    const res = await this.connection.gql({
+      query: indexerSearchQuery,
+      variables: {
+        id: this.options.id,
+        input: {
+          table,
+          query,
+          options,
+        } as SearchInput,
+      },
+    });
+    const result = res.workspace.search as unknown as SearchResult<T, O>;
+    return result;
+  }
+
+  override search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>> {
+    return new Observable(observer => {
+      this.search(table, query, options)
+        .then(data => {
+          observer.next(data);
+          observer.complete();
+        })
+        .catch(error => {
+          observer.error(error);
+        });
+    });
+  }
+
+  override async aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>> {
+    const res = await this.connection.gql({
+      query: indexerAggregateQuery,
+      variables: {
+        id: this.options.id,
+        input: { table, query, field, options } as AggregateInput,
+      },
+    });
+    const result = res.workspace.aggregate as unknown as AggregateResult<T, O>;
+    return result;
+  }
+
+  override aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return new Observable(observer => {
+      this.aggregate(table, query, field, options)
+        .then(data => {
+          observer.next(data);
+          observer.complete();
+        })
+        .catch(error => {
+          observer.error(error);
+        });
+    });
+  }
+
+  override deleteByQuery<T extends keyof IndexerSchema>(
+    _table: T,
+    _query: Query<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override insert<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override delete<T extends keyof IndexerSchema>(
+    _table: T,
+    _id: string
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override update<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override async refreshIfNeed(): Promise<void> {
+    return;
+  }
+
+  override async indexVersion(): Promise<number> {
+    return 1;
+  }
+}

packages/common/nbstore/src/impls/cloud/socket.ts

@@ -0,0 +1,329 @@
+import {
+  type RealtimeEvent,
+  type RealtimeRequestEnvelope,
+  type RealtimeSubscribeEnvelope,
+  type RealtimeUnsubscribeEnvelope,
+} from '@affine/realtime';
+import {
+  Manager as SocketIOManager,
+  type Socket as SocketIO,
+} from 'socket.io-client';
+
+import { AutoReconnectConnection } from '../../connection';
+import type { TelemetryAck, TelemetryBatch } from '../../telemetry/types';
+import { throwIfAborted } from '../../utils/throw-if-aborted';
+
+// TODO(@forehalo): use [UserFriendlyError]
+interface EventError {
+  name: string;
+  message: string;
+}
+
+type WebsocketResponse<T> =
+  | {
+      error: EventError;
+    }
+  | {
+      data: T;
+    };
+
+interface ServerEvents {
+  'space:broadcast-doc-updates': {
+    spaceType: string;
+    spaceId: string;
+    docId: string;
+    updates: string[];
+    timestamp: number;
+    editor?: string;
+    compressed?: boolean;
+  };
+  'space:broadcast-doc-invalidation': {
+    spaceType: string;
+    spaceId: string;
+    timestamp: number;
+  };
+
+  'space:collect-awareness': {
+    spaceType: string;
+    spaceId: string;
+    docId: string;
+  };
+
+  'space:broadcast-awareness-update': {
+    spaceType: string;
+    spaceId: string;
+    docId: string;
+    awarenessUpdate: string;
+  };
+
+  'realtime:event': RealtimeEvent;
+}
+
+interface ClientEvents {
+  'space:join-batch': [
+    {
+      spaces: Array<{
+        spaceType: string;
+        spaceId: string;
+        docId?: string;
+      }>;
+      clientVersion: string;
+    },
+    { clientId: string; success: boolean },
+  ];
+  'space:leave-batch': {
+    spaceType: string;
+    spaceId: string;
+    docIds: string[];
+  };
+  'space:update-awareness': {
+    spaceType: string;
+    spaceId: string;
+    docId: string;
+    awarenessUpdate: string;
+  };
+
+  'space:load-awarenesses': {
+    spaceType: string;
+    spaceId: string;
+    docId: string;
+  };
+
+  'space:push-doc-update': [
+    { spaceType: string; spaceId: string; docId: string; update: string },
+    { timestamp: number },
+  ];
+  'space:load-doc-timestamps': [
+    {
+      spaceType: string;
+      spaceId: string;
+      timestamp?: number;
+    },
+    Record<string, number>,
+  ];
+  'space:load-doc': [
+    {
+      spaceType: string;
+      spaceId: string;
+      docId: string;
+      stateVector?: string;
+    },
+    {
+      missing: string;
+      state: string;
+      timestamp: number;
+    },
+  ];
+  'space:delete-doc': [
+    { spaceType: string; spaceId: string; docId: string },
+    { success?: true },
+  ];
+  'space:doc-lifecycle': [
+    {
+      spaceType: string;
+      spaceId: string;
+      docId: string;
+      lifecycle: 'trash' | 'restore' | 'delete';
+    },
+    { rootUpdate: string; timestamp: number },
+  ];
+
+  'telemetry:batch': [TelemetryBatch, TelemetryAck];
+
+  'realtime:request': [RealtimeRequestEnvelope, unknown];
+  'realtime:subscribe': [RealtimeSubscribeEnvelope, { subscriptionId: string }];
+  'realtime:unsubscribe': [RealtimeUnsubscribeEnvelope, { ok: true }];
+}
+
+export const SPACE_JOIN_BATCH_LIMIT = 100;
+
+export type ServerEventsMap = {
+  [Key in keyof ServerEvents]: (data: ServerEvents[Key]) => void;
+};
+
+export type ClientEventsMap = {
+  [Key in keyof ClientEvents]: ClientEvents[Key] extends Array<any>
+    ? (
+        data: ClientEvents[Key][0],
+        ack: (res: WebsocketResponse<ClientEvents[Key][1]>) => void
+      ) => void
+    : (data: ClientEvents[Key]) => void;
+};
+
+export type Socket = SocketIO<ServerEventsMap, ClientEventsMap>;
+
+type BufferConstructorLike = {
+  from(
+    data: Uint8Array | string,
+    encoding?: string
+  ): Uint8Array & {
+    toString(encoding: string): string;
+  };
+};
+
+const BufferCtor = (globalThis as { Buffer?: BufferConstructorLike }).Buffer;
+const CHUNK_SIZE = 0x8000;
+
+export async function uint8ArrayToBase64(array: Uint8Array): Promise<string> {
+  if (BufferCtor) {
+    return BufferCtor.from(array).toString('base64');
+  }
+
+  let binary = '';
+  for (let i = 0; i < array.length; i += CHUNK_SIZE) {
+    const chunk = array.subarray(i, i + CHUNK_SIZE);
+    binary += String.fromCharCode(...chunk);
+  }
+  return btoa(binary);
+}
+
+export function base64ToUint8Array(base64: string) {
+  if (BufferCtor) {
+    return new Uint8Array(BufferCtor.from(base64, 'base64'));
+  }
+
+  const binaryString = atob(base64);
+  const bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+let authMethod:
+  | ((endpoint: string, cb: (data: object) => void) => void)
+  | undefined;
+
+export function configureSocketAuthMethod(
+  cb: (endpoint: string, cb: (data: object) => void) => void
+) {
+  authMethod = cb;
+}
+
+class SocketManager {
+  private readonly socketIOManager: SocketIOManager;
+  socket: Socket;
+  refCount = 0;
+
+  constructor(endpoint: string, isSelfHosted: boolean) {
+    this.socketIOManager = new SocketIOManager(endpoint, {
+      autoConnect: false,
+      transports: isSelfHosted ? ['polling', 'websocket'] : ['websocket'], // self-hosted server may not support websocket
+      secure: new URL(endpoint).protocol === 'https:',
+      // we will handle reconnection by ourselves
+      reconnection: false,
+    });
+    this.socket = this.socketIOManager.socket('/', {
+      auth(cb) {
+        if (authMethod) {
+          authMethod(endpoint, cb);
+        } else {
+          cb({});
+        }
+      },
+    });
+  }
+
+  connect() {
+    let disconnected = false;
+    this.refCount++;
+    this.socket.connect();
+    return {
+      socket: this.socket,
+      disconnect: () => {
+        if (disconnected) {
+          return;
+        }
+        disconnected = true;
+        this.refCount--;
+        if (this.refCount === 0) {
+          this.socket.disconnect();
+        }
+      },
+    };
+  }
+
+  reset() {
+    this.socket.disconnect();
+  }
+}
+
+const SOCKET_MANAGER_CACHE = new Map<string, SocketManager>();
+function getSocketManagerKey(endpoint: string, isSelfHosted: boolean) {
+  return `${endpoint}:${isSelfHosted ? 'selfhosted' : 'cloud'}`;
+}
+
+function getSocketManager(endpoint: string, isSelfHosted: boolean) {
+  const key = getSocketManagerKey(endpoint, isSelfHosted);
+  let manager = SOCKET_MANAGER_CACHE.get(key);
+  if (!manager) {
+    manager = new SocketManager(endpoint, isSelfHosted);
+    SOCKET_MANAGER_CACHE.set(key, manager);
+  }
+  return manager;
+}
+
+export class SocketConnection extends AutoReconnectConnection<{
+  socket: Socket;
+  disconnect: () => void;
+}> {
+  static resetSharedConnection(endpoint: string, isSelfHosted: boolean) {
+    SOCKET_MANAGER_CACHE.get(
+      getSocketManagerKey(endpoint, isSelfHosted)
+    )?.reset();
+  }
+
+  manager = getSocketManager(this.endpoint, this.isSelfHosted);
+
+  constructor(
+    private readonly endpoint: string,
+    private readonly isSelfHosted: boolean
+  ) {
+    super();
+  }
+
+  override async doConnect(signal?: AbortSignal) {
+    const { socket, disconnect } = this.manager.connect();
+    try {
+      throwIfAborted(signal);
+      await Promise.race([
+        new Promise<void>((resolve, reject) => {
+          if (socket.connected) {
+            resolve();
+            return;
+          }
+          socket.once('connect', () => {
+            resolve();
+          });
+          socket.once('connect_error', err => {
+            reject(err);
+          });
+        }),
+        new Promise<void>((_resolve, reject) => {
+          signal?.addEventListener('abort', () => {
+            reject(signal.reason);
+          });
+        }),
+      ]);
+    } catch (err) {
+      disconnect();
+      throw err;
+    }
+
+    socket.on('disconnect', this.handleDisconnect);
+
+    return {
+      socket,
+      disconnect,
+    };
+  }
+
+  override doDisconnect(conn: { socket: Socket; disconnect: () => void }) {
+    conn.socket.off('disconnect', this.handleDisconnect);
+    conn.disconnect();
+  }
+
+  handleDisconnect = (reason: SocketIO.DisconnectReason) => {
+    this.error = new Error(reason);
+  };
+}

packages/common/nbstore/src/impls/idb/blob-sync.ts

@@ -0,0 +1,36 @@
+import { share } from '../../connection';
+import { BlobSyncStorageBase } from '../../storage';
+import { IDBConnection, type IDBConnectionOptions } from './db';
+
+export class IndexedDBBlobSyncStorage extends BlobSyncStorageBase {
+  static readonly identifier = 'IndexedDBBlobSyncStorage';
+
+  readonly connection = share(new IDBConnection(this.options));
+
+  constructor(private readonly options: IDBConnectionOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection;
+  }
+
+  async setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void> {
+    const trx = this.db.inner.db.transaction('blobSync', 'readwrite');
+    await trx.store.put({
+      peer,
+      key: blobId,
+      uploadedAt,
+    });
+  }
+
+  async getBlobUploadedAt(peer: string, blobId: string): Promise<Date | null> {
+    const trx = this.db.inner.db.transaction('blobSync', 'readonly');
+    const record = await trx.store.get([peer, blobId]);
+    return record?.uploadedAt ?? null;
+  }
+}

packages/common/nbstore/src/impls/idb/blob.ts

@@ -0,0 +1,96 @@
+import { share } from '../../connection';
+import {
+  type BlobRecord,
+  BlobStorageBase,
+  type ListedBlobRecord,
+} from '../../storage';
+import { IDBConnection, type IDBConnectionOptions } from './db';
+
+export class IndexedDBBlobStorage extends BlobStorageBase {
+  static readonly identifier = 'IndexedDBBlobStorage';
+  override readonly isReadonly = false;
+
+  readonly connection = share(new IDBConnection(this.options));
+
+  constructor(private readonly options: IDBConnectionOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection.inner.db;
+  }
+
+  override async get(key: string) {
+    const trx = this.db.transaction(['blobs', 'blobData'], 'readonly');
+    const blob = await trx.objectStore('blobs').get(key);
+    const data = await trx.objectStore('blobData').get(key);
+
+    if (!blob || blob.deletedAt || !data) {
+      return null;
+    }
+
+    return {
+      ...blob,
+      data: data.data,
+    };
+  }
+
+  override async set(blob: BlobRecord) {
+    const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite');
+    await trx.objectStore('blobs').put({
+      key: blob.key,
+      mime: blob.mime,
+      size: blob.data.byteLength,
+      createdAt: new Date(),
+      deletedAt: null,
+    });
+    await trx.objectStore('blobData').put({
+      key: blob.key,
+      data: blob.data,
+    });
+  }
+
+  override async delete(key: string, permanently: boolean) {
+    if (permanently) {
+      const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite');
+      await trx.objectStore('blobs').delete(key);
+      await trx.objectStore('blobData').delete(key);
+    } else {
+      const trx = this.db.transaction('blobs', 'readwrite');
+      const blob = await trx.store.get(key);
+      if (blob) {
+        await trx.store.put({
+          ...blob,
+          deletedAt: new Date(),
+        });
+      }
+    }
+  }
+
+  override async release() {
+    const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite');
+
+    const it = trx.objectStore('blobs').iterate();
+
+    for await (const item of it) {
+      if (item.value.deletedAt) {
+        await item.delete();
+        await trx.objectStore('blobData').delete(item.value.key);
+      }
+    }
+  }
+
+  override async list() {
+    const trx = this.db.transaction('blobs', 'readonly');
+    const it = trx.store.iterate();
+
+    const blobs: ListedBlobRecord[] = [];
+    for await (const item of it) {
+      if (!item.value.deletedAt) {
+        blobs.push(item.value);
+      }
+    }
+
+    return blobs;
+  }
+}

packages/common/nbstore/src/impls/idb/db.ts

@@ -0,0 +1,59 @@
+import { type IDBPDatabase, openDB } from 'idb';
+
+import { AutoReconnectConnection } from '../../connection';
+import type { SpaceType } from '../../utils/universal-id';
+import { type DocStorageSchema, migrator } from './schema';
+
+export interface IDBConnectionOptions {
+  flavour: string;
+  type: SpaceType;
+  id: string;
+}
+
+export class IDBConnection extends AutoReconnectConnection<{
+  db: IDBPDatabase<DocStorageSchema>;
+  channel: BroadcastChannel;
+}> {
+  readonly dbName = `${this.opts.flavour}:${this.opts.type}:${this.opts.id}`;
+
+  override get shareId() {
+    return `idb(${migrator.version}):${this.dbName}`;
+  }
+
+  constructor(private readonly opts: IDBConnectionOptions) {
+    super();
+  }
+
+  override async doConnect() {
+    // indexeddb will responsible for version control, so the db.version always match migrator.version
+    const db = await openDB<DocStorageSchema>(this.dbName, migrator.version, {
+      upgrade: migrator.migrate,
+    });
+    db.addEventListener('versionchange', this.handleVersionChange);
+
+    return {
+      db,
+      channel: new BroadcastChannel('idb:' + this.dbName),
+    };
+  }
+
+  override doDisconnect(db: {
+    db: IDBPDatabase<DocStorageSchema>;
+    channel: BroadcastChannel;
+  }) {
+    db.db.removeEventListener('versionchange', this.handleVersionChange);
+    db.channel.close();
+    db.db.close();
+  }
+
+  handleVersionChange = (e: IDBVersionChangeEvent) => {
+    if (e.newVersion !== migrator.version) {
+      this.error = new Error(
+        'Database version mismatch, expected ' +
+          migrator.version +
+          ' but got ' +
+          e.newVersion
+      );
+    }
+  };
+}

packages/common/nbstore/src/impls/idb/doc-sync.ts

@@ -0,0 +1,149 @@
+import { share } from '../../connection';
+import {
+  type DocClock,
+  type DocClocks,
+  DocSyncStorageBase,
+} from '../../storage';
+import { IDBConnection, type IDBConnectionOptions } from './db';
+
+export class IndexedDBDocSyncStorage extends DocSyncStorageBase {
+  static readonly identifier = 'IndexedDBDocSyncStorage';
+
+  constructor(private readonly options: IDBConnectionOptions) {
+    super();
+  }
+
+  readonly connection = share(new IDBConnection(this.options));
+
+  get db() {
+    return this.connection.inner.db;
+  }
+
+  override async getPeerRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null> {
+    const trx = this.db.transaction('peerClocks', 'readonly');
+
+    const record = await trx.store.get([peer, docId]);
+
+    return record
+      ? {
+          docId: record.docId,
+          timestamp: record.clock,
+        }
+      : null;
+  }
+  override async getPeerRemoteClocks(peer: string) {
+    const trx = this.db.transaction('peerClocks', 'readonly');
+
+    const records = await trx.store.index('peer').getAll(peer);
+
+    return records.reduce((clocks, { docId, clock }) => {
+      clocks[docId] = clock;
+      return clocks;
+    }, {} as DocClocks);
+  }
+
+  override async setPeerRemoteClock(peer: string, clock: DocClock) {
+    const trx = this.db.transaction('peerClocks', 'readwrite');
+    const record = await trx.store.get([peer, clock.docId]);
+
+    if (!record || record.clock < clock.timestamp) {
+      await trx.store.put({
+        peer,
+        docId: clock.docId,
+        clock: clock.timestamp,
+        pulledClock: record?.pulledClock ?? new Date(0),
+        pushedClock: record?.pushedClock ?? new Date(0),
+      });
+    }
+  }
+
+  override async getPeerPulledRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null> {
+    const trx = this.db.transaction('peerClocks', 'readonly');
+
+    const record = await trx.store.get([peer, docId]);
+
+    return record
+      ? {
+          docId: record.docId,
+          timestamp: record.pulledClock,
+        }
+      : null;
+  }
+  override async getPeerPulledRemoteClocks(peer: string) {
+    const trx = this.db.transaction('peerClocks', 'readonly');
+
+    const records = await trx.store.index('peer').getAll(peer);
+
+    return records.reduce((clocks, { docId, pulledClock }) => {
+      clocks[docId] = pulledClock;
+      return clocks;
+    }, {} as DocClocks);
+  }
+  override async setPeerPulledRemoteClock(peer: string, clock: DocClock) {
+    const trx = this.db.transaction('peerClocks', 'readwrite');
+    const record = await trx.store.get([peer, clock.docId]);
+
+    if (!record || record.pulledClock < clock.timestamp) {
+      await trx.store.put({
+        peer,
+        docId: clock.docId,
+        clock: record?.clock ?? new Date(0),
+        pulledClock: clock.timestamp,
+        pushedClock: record?.pushedClock ?? new Date(0),
+      });
+    }
+  }
+
+  override async getPeerPushedClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null> {
+    const trx = this.db.transaction('peerClocks', 'readonly');
+
+    const record = await trx.store.get([peer, docId]);
+
+    return record
+      ? {
+          docId: record.docId,
+          timestamp: record.pushedClock,
+        }
+      : null;
+  }
+  override async getPeerPushedClocks(peer: string) {
+    const trx = this.db.transaction('peerClocks', 'readonly');
+
+    const records = await trx.store.index('peer').getAll(peer);
+
+    return records.reduce((clocks, { docId, pushedClock }) => {
+      clocks[docId] = pushedClock;
+      return clocks;
+    }, {} as DocClocks);
+  }
+
+  override async setPeerPushedClock(peer: string, clock: DocClock) {
+    const trx = this.db.transaction('peerClocks', 'readwrite');
+    const record = await trx.store.get([peer, clock.docId]);
+
+    if (!record || record.pushedClock < clock.timestamp) {
+      await trx.store.put({
+        peer,
+        docId: clock.docId,
+        clock: record?.clock ?? new Date(0),
+        pushedClock: clock.timestamp,
+        pulledClock: record?.pulledClock ?? new Date(0),
+      });
+    }
+  }
+
+  override async clearClocks() {
+    const trx = this.db.transaction('peerClocks', 'readwrite');
+
+    await trx.store.clear();
+  }
+}

packages/common/nbstore/src/impls/idb/doc.ts

@@ -0,0 +1,209 @@
+import { share } from '../../connection';
+import {
+  type DocClock,
+  type DocClocks,
+  type DocRecord,
+  DocStorageBase,
+  type DocUpdate,
+} from '../../storage';
+import { IDBConnection, type IDBConnectionOptions } from './db';
+import { IndexedDBLocker } from './lock';
+
+interface ChannelMessage {
+  type: 'update';
+  update: DocRecord;
+  origin?: string;
+}
+
+export class IndexedDBDocStorage extends DocStorageBase<IDBConnectionOptions> {
+  static readonly identifier = 'IndexedDBDocStorage';
+
+  readonly connection = share(new IDBConnection(this.options));
+
+  get db() {
+    return this.connection.inner.db;
+  }
+
+  get channel() {
+    return this.connection.inner.channel;
+  }
+
+  override locker = new IndexedDBLocker(this.connection);
+
+  override async pushDocUpdate(update: DocUpdate, origin?: string) {
+    let timestamp = new Date();
+
+    let retry = 0;
+
+    while (true) {
+      try {
+        const trx = this.db.transaction(['updates', 'clocks'], 'readwrite');
+
+        await trx.objectStore('updates').add({
+          ...update,
+          createdAt: timestamp,
+        });
+
+        await trx.objectStore('clocks').put({ docId: update.docId, timestamp });
+
+        trx.commit();
+      } catch (e) {
+        if (e instanceof Error && e.name === 'ConstraintError') {
+          retry++;
+          if (retry < 10) {
+            timestamp = new Date(timestamp.getTime() + 1);
+            continue;
+          }
+        }
+        throw e;
+      }
+      break;
+    }
+
+    this.emit(
+      'update',
+      {
+        docId: update.docId,
+        bin: update.bin,
+        timestamp,
+        editor: update.editor,
+      },
+      origin
+    );
+
+    this.channel.postMessage({
+      type: 'update',
+      update: {
+        docId: update.docId,
+        bin: update.bin,
+        timestamp,
+        editor: update.editor,
+      },
+      origin,
+    } satisfies ChannelMessage);
+
+    return { docId: update.docId, timestamp };
+  }
+
+  protected override async getDocSnapshot(docId: string) {
+    const trx = this.db.transaction('snapshots', 'readonly');
+    const record = await trx.store.get(docId);
+
+    if (!record) {
+      return null;
+    }
+
+    return {
+      docId,
+      bin: record.bin,
+      timestamp: record.updatedAt,
+    };
+  }
+
+  override async deleteDoc(docId: string) {
+    const trx = this.db.transaction(
+      ['snapshots', 'updates', 'clocks'],
+      'readwrite'
+    );
+
+    const idx = trx.objectStore('updates').index('docId');
+    const iter = idx.iterate(IDBKeyRange.only(docId));
+
+    for await (const { value } of iter) {
+      await trx.objectStore('updates').delete([value.docId, value.createdAt]);
+    }
+
+    await trx.objectStore('snapshots').delete(docId);
+    await trx.objectStore('clocks').delete(docId);
+  }
+
+  override async getDocTimestamps(after: Date = new Date(0)) {
+    const trx = this.db.transaction('clocks', 'readonly');
+
+    const clocks = await trx.store.getAll();
+
+    return clocks.reduce((ret, cur) => {
+      if (cur.timestamp > after) {
+        ret[cur.docId] = cur.timestamp;
+      }
+      return ret;
+    }, {} as DocClocks);
+  }
+
+  override async getDocTimestamp(docId: string): Promise<DocClock | null> {
+    const trx = this.db.transaction('clocks', 'readonly');
+
+    return (await trx.store.get(docId)) ?? null;
+  }
+
+  protected override async setDocSnapshot(
+    snapshot: DocRecord
+  ): Promise<boolean> {
+    const trx = this.db.transaction('snapshots', 'readwrite');
+    const record = await trx.store.get(snapshot.docId);
+
+    if (!record || record.updatedAt < snapshot.timestamp) {
+      await trx.store.put({
+        docId: snapshot.docId,
+        bin: snapshot.bin,
+        createdAt: record?.createdAt ?? snapshot.timestamp,
+        updatedAt: snapshot.timestamp,
+      });
+    }
+
+    trx.commit();
+    return true;
+  }
+
+  protected override async getDocUpdates(docId: string): Promise<DocRecord[]> {
+    const trx = this.db.transaction('updates', 'readonly');
+    const updates = await trx.store.index('docId').getAll(docId);
+
+    return updates.map(update => ({
+      docId,
+      bin: update.bin,
+      timestamp: update.createdAt,
+    }));
+  }
+
+  protected override async markUpdatesMerged(
+    docId: string,
+    updates: DocRecord[]
+  ): Promise<number> {
+    const trx = this.db.transaction('updates', 'readwrite');
+
+    await Promise.all(
+      updates.map(update => trx.store.delete([docId, update.timestamp]))
+    );
+
+    trx.commit();
+    return updates.length;
+  }
+
+  private docUpdateListener = 0;
+
+  override subscribeDocUpdate(
+    callback: (update: DocRecord, origin?: string) => void
+  ): () => void {
+    if (this.docUpdateListener === 0) {
+      this.channel.addEventListener('message', this.handleChannelMessage);
+    }
+    this.docUpdateListener++;
+
+    const dispose = super.subscribeDocUpdate(callback);
+
+    return () => {
+      dispose();
+      this.docUpdateListener--;
+      if (this.docUpdateListener === 0) {
+        this.channel.removeEventListener('message', this.handleChannelMessage);
+      }
+    };
+  }
+
+  handleChannelMessage = (event: MessageEvent<ChannelMessage>) => {
+    if (event.data.type === 'update') {
+      this.emit('update', event.data.update, event.data.origin);
+    }
+  };
+}

packages/common/nbstore/src/impls/idb/index.ts

@@ -0,0 +1,28 @@
+import type { StorageConstructor } from '..';
+import { IndexedDBBlobStorage } from './blob';
+import { IndexedDBBlobSyncStorage } from './blob-sync';
+import { IndexedDBDocStorage } from './doc';
+import { IndexedDBDocSyncStorage } from './doc-sync';
+import { IndexedDBIndexerStorage } from './indexer';
+import { IndexedDBIndexerSyncStorage } from './indexer-sync';
+
+export * from './blob';
+export * from './blob-sync';
+export * from './doc';
+export * from './doc-sync';
+export * from './indexer';
+export * from './indexer-sync';
+
+export const idbStorages = [
+  IndexedDBDocStorage,
+  IndexedDBBlobStorage,
+  IndexedDBDocSyncStorage,
+  IndexedDBBlobSyncStorage,
+  IndexedDBIndexerStorage,
+  IndexedDBIndexerSyncStorage,
+] satisfies StorageConstructor[];
+
+export const idbStoragesIndexerOnly = [
+  IndexedDBIndexerStorage,
+  IndexedDBIndexerSyncStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/idb/indexer-sync.ts

@@ -0,0 +1,45 @@
+import { share } from '../../connection';
+import {
+  type DocIndexedClock,
+  IndexerSyncStorageBase,
+} from '../../storage/indexer-sync';
+import { IDBConnection, type IDBConnectionOptions } from './db';
+
+export class IndexedDBIndexerSyncStorage extends IndexerSyncStorageBase {
+  static readonly identifier = 'IndexedDBIndexerSyncStorage';
+
+  readonly connection = share(new IDBConnection(this.options));
+
+  constructor(private readonly options: IDBConnectionOptions) {
+    super();
+  }
+
+  async getDocIndexedClock(docId: string): Promise<DocIndexedClock | null> {
+    const tx = this.connection.inner.db.transaction('indexerSync', 'readonly');
+    const store = tx.store;
+    const result = await store.get(docId);
+    return result
+      ? {
+          docId: result.docId,
+          timestamp: result.indexedClock,
+          indexerVersion: result.indexerVersion ?? 0,
+        }
+      : null;
+  }
+
+  async setDocIndexedClock(docClock: DocIndexedClock): Promise<void> {
+    const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
+    const store = tx.store;
+    await store.put({
+      docId: docClock.docId,
+      indexedClock: docClock.timestamp,
+      indexerVersion: docClock.indexerVersion,
+    });
+  }
+
+  async clearDocIndexedClock(docId: string): Promise<void> {
+    const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
+    const store = tx.store;
+    await store.delete(docId);
+  }
+}

packages/common/nbstore/src/impls/idb/indexer/__tests__/bm25.spec.ts

@@ -0,0 +1,10 @@
+import { expect, test } from 'vitest';
+
+import { bm25 } from '../bm25';
+
+test('bm25', () => {
+  expect(bm25(1, 1, 10, 10, 15)).toEqual(3.2792079793859643);
+  expect(bm25(2, 1, 10, 10, 15) > bm25(1, 1, 10, 10, 15)).toBeTruthy();
+  expect(bm25(1, 1, 10, 10, 15) > bm25(2, 1, 10, 100, 15)).toBeTruthy();
+  expect(bm25(1, 1, 10, 10, 15) > bm25(1, 1, 10, 100, 15)).toBeTruthy();
+});

packages/common/nbstore/src/impls/idb/indexer/__tests__/highlighter.spec.ts

@@ -0,0 +1,32 @@
+import { expect, test } from 'vitest';
+
+import { highlighter } from '../highlighter';
+
+test('highlighter', () => {
+  expect(highlighter('0123456789', '<b>', '</b>', [[3, 5]])).toEqual(
+    '012<b>34</b>56789'
+  );
+
+  expect(
+    highlighter(
+      '012345678901234567890123456789012345678901234567890123456789',
+      '<b>',
+      '</b>',
+      [[59, 60]]
+    )
+  ).toEqual('...0123456789012345678901234567890123456789012345678<b>9</b>');
+
+  expect(
+    highlighter(
+      '012345678901234567890123456789012345678901234567890123456789',
+      '<b>',
+      '</b>',
+      [
+        [10, 11],
+        [49, 51],
+      ]
+    )
+  ).toEqual(
+    '0123456789<b>0</b>12345678901234567890123456789012345678<b>9</b>...'
+  );
+});

packages/common/nbstore/src/impls/idb/indexer/__tests__/tokenizer.spec.ts

@@ -0,0 +1,128 @@
+import { expect, test } from 'vitest';
+
+import { GeneralTokenizer } from '../tokenizer';
+
+test('tokenizer', () => {
+  {
+    const tokens = new GeneralTokenizer().tokenize('hello  world,\n AFFiNE');
+
+    expect(tokens).toEqual([
+      { term: 'hello', start: 0, end: 5 },
+      { term: 'world', start: 7, end: 12 },
+      { term: 'affine', start: 15, end: 21 },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('你好世界，阿芬');
+
+    expect(tokens).toEqual([
+      {
+        end: 2,
+        start: 0,
+        term: '你好',
+      },
+      {
+        end: 3,
+        start: 1,
+        term: '好世',
+      },
+      {
+        end: 4,
+        start: 2,
+        term: '世界',
+      },
+      {
+        end: 7,
+        start: 5,
+        term: '阿芬',
+      },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('1阿2芬');
+
+    expect(tokens).toEqual([
+      { term: '1', start: 0, end: 1 },
+      { term: '阿', start: 1, end: 2 },
+      { term: '2', start: 2, end: 3 },
+      { term: '芬', start: 3, end: 4 },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('안녕하세요 세계');
+
+    expect(tokens).toEqual([
+      {
+        end: 2,
+        start: 0,
+        term: '안녕',
+      },
+      {
+        end: 3,
+        start: 1,
+        term: '녕하',
+      },
+      {
+        end: 4,
+        start: 2,
+        term: '하세',
+      },
+      {
+        end: 5,
+        start: 3,
+        term: '세요',
+      },
+      {
+        end: 8,
+        start: 6,
+        term: '세계',
+      },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('ハローワールド');
+
+    expect(tokens).toEqual([
+      { term: 'ハロ', start: 0, end: 2 },
+      { term: 'ロー', start: 1, end: 3 },
+      { term: 'ーワ', start: 2, end: 4 },
+      { term: 'ワー', start: 3, end: 5 },
+      { term: 'ール', start: 4, end: 6 },
+      { term: 'ルド', start: 5, end: 7 },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('はろーわーるど');
+
+    expect(tokens).toEqual([
+      { term: 'はろ', start: 0, end: 2 },
+      { term: 'ろー', start: 1, end: 3 },
+      { term: 'ーわ', start: 2, end: 4 },
+      { term: 'わー', start: 3, end: 5 },
+      { term: 'ーる', start: 4, end: 6 },
+      { term: 'るど', start: 5, end: 7 },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('👋1️⃣🚪👋🏿');
+
+    expect(tokens).toEqual([
+      { term: '👋', start: 0, end: 2 },
+      { term: '1️⃣', start: 2, end: 5 },
+      { term: '🚪', start: 5, end: 7 },
+      { term: '👋🏿', start: 7, end: 11 },
+    ]);
+  }
+
+  {
+    const tokens = new GeneralTokenizer().tokenize('1️');
+
+    expect(tokens).toEqual([{ term: '1️', start: 0, end: 2 }]);
+  }
+});

packages/common/nbstore/src/impls/idb/indexer/bm25.ts

@@ -0,0 +1,62 @@
+/**
+ * Parameters of the BM25+ scoring algorithm. Customizing these is almost never
+ * necessary, and finetuning them requires an understanding of the BM25 scoring
+ * model.
+ *
+ * Some information about BM25 (and BM25+) can be found at these links:
+ *
+ *   - https://en.wikipedia.org/wiki/Okapi_BM25
+ *   - https://opensourceconnections.com/blog/2015/10/16/bm25-the-next-generation-of-lucene-relevation/
+ */
+export type BM25Params = {
+  /** Term frequency saturation point.
+   *
+   * Recommended values are between `1.2` and `2`. Higher values increase the
+   * difference in score between documents with higher and lower term
+   * frequencies. Setting this to `0` or a negative value is invalid. Defaults
+   * to `1.2`
+   */
+  k: number;
+
+  /**
+   * Length normalization impact.
+   *
+   * Recommended values are around `0.75`. Higher values increase the weight
+   * that field length has on scoring. Setting this to `0` (not recommended)
+   * means that the field length has no effect on scoring. Negative values are
+   * invalid. Defaults to `0.7`.
+   */
+  b: number;
+
+  /**
+   * BM25+ frequency normalization lower bound (usually called δ).
+   *
+   * Recommended values are between `0.5` and `1`. Increasing this parameter
+   * increases the minimum relevance of one occurrence of a search term
+   * regardless of its (possibly very long) field length. Negative values are
+   * invalid. Defaults to `0.5`.
+   */
+  d: number;
+};
+
+const defaultBM25params: BM25Params = { k: 1.2, b: 0.7, d: 0.5 };
+
+export const bm25 = (
+  termFreq: number,
+  matchingCount: number,
+  totalCount: number,
+  fieldLength: number,
+  avgFieldLength: number,
+  bm25params: BM25Params = defaultBM25params
+): number => {
+  const { k, b, d } = bm25params;
+  const invDocFreq = Math.log(
+    1 + (totalCount - matchingCount + 0.5) / (matchingCount + 0.5)
+  );
+  return (
+    invDocFreq *
+    (d +
+      (termFreq * (k + 1)) /
+        (termFreq + k * (1 - b + b * (fieldLength / avgFieldLength))))
+  );
+};

packages/common/nbstore/src/impls/idb/indexer/data-struct.ts

@@ -0,0 +1,544 @@
+import { type IDBPDatabase, type IDBPTransaction, type StoreNames } from 'idb';
+
+import {
+  type AggregateOptions,
+  type AggregateResult,
+  type IndexerDocument,
+  type IndexerFieldSchema,
+  IndexerSchema,
+  type Query,
+  type SearchOptions,
+  type SearchResult,
+} from '../../../storage';
+import { shallowEqual } from '../../../utils/shallow-equal';
+import type { DocStorageSchema } from '../schema';
+import { highlighter } from './highlighter';
+import {
+  BooleanInvertedIndex,
+  FullTextInvertedIndex,
+  IntegerInvertedIndex,
+  type InvertedIndex,
+  StringInvertedIndex,
+} from './inverted-index';
+import { Match } from './match';
+
+export type DataStructRWTransaction = IDBPTransaction<
+  DocStorageSchema,
+  ArrayLike<StoreNames<DocStorageSchema>>,
+  'readwrite'
+>;
+
+export type DataStructROTransaction = IDBPTransaction<
+  DocStorageSchema,
+  ArrayLike<StoreNames<DocStorageSchema>>,
+  'readonly' | 'readwrite'
+>;
+
+let debugMarkCount = 0;
+
+export class DataStruct {
+  database: IDBPDatabase<DocStorageSchema> = null as any;
+  invertedIndex = new Map<string, Map<string, InvertedIndex>>();
+
+  constructor() {
+    for (const [tableName, table] of Object.entries(IndexerSchema)) {
+      const tableInvertedIndex = new Map<string, InvertedIndex>();
+
+      for (const [fieldName, type] of Object.entries(table)) {
+        const typeInfo: IndexerFieldSchema =
+          typeof type === 'string' ? { type } : type;
+        if ('index' in typeInfo && typeInfo.index === false) {
+          // If index is false, we don't need to create an inverted index for this field.
+          continue;
+        }
+        if (typeInfo.type === 'String') {
+          tableInvertedIndex.set(
+            fieldName,
+            new StringInvertedIndex(tableName, fieldName)
+          );
+        } else if (typeInfo.type === 'Integer') {
+          tableInvertedIndex.set(
+            fieldName,
+            new IntegerInvertedIndex(tableName, fieldName)
+          );
+        } else if (typeInfo.type === 'FullText') {
+          tableInvertedIndex.set(
+            fieldName,
+            new FullTextInvertedIndex(tableName, fieldName)
+          );
+        } else if (typeInfo.type === 'Boolean') {
+          tableInvertedIndex.set(
+            fieldName,
+            new BooleanInvertedIndex(tableName, fieldName)
+          );
+        } else {
+          throw new Error(`Field type '${typeInfo.type}' not supported`);
+        }
+      }
+
+      this.invertedIndex.set(tableName, tableInvertedIndex);
+    }
+  }
+
+  private async update(
+    trx: DataStructRWTransaction,
+    table: keyof IndexerSchema,
+    document: IndexerDocument
+  ) {
+    using _ = await this.measure(`update`);
+    const existsNid = await trx
+      .objectStore('indexerRecords')
+      .index('id')
+      .getKey([table, document.id]);
+
+    const exists = existsNid
+      ? await trx.objectStore('indexerRecords').get(existsNid)
+      : null;
+
+    if (!existsNid || !exists) {
+      // if not exists, return
+      return;
+    }
+
+    // delete exists one
+    await this.deleteByNid(trx, existsNid);
+
+    const dataMap = new Map([...exists.data, ...document.fields]); // merge exists data with new data
+    const nid = await trx
+      .objectStore('indexerRecords')
+      .put({ table, id: document.id, data: dataMap });
+
+    for (const [key, values] of dataMap) {
+      const type = IndexerSchema[table][
+        key as keyof IndexerSchema[typeof table]
+      ] as IndexerFieldSchema;
+      if (!type) {
+        continue;
+      }
+
+      const typeInfo = typeof type === 'string' ? { type } : type;
+      if (typeInfo.index !== false) {
+        // If index is false, the field will not be indexed
+        const iidx = this.invertedIndex.get(table)?.get(key);
+        if (!iidx) {
+          continue;
+        }
+        using _ = await this.measure(`insert[${typeInfo.type}]`);
+        await iidx.insert(trx, nid, values);
+      }
+    }
+  }
+
+  private async insert(
+    trx: DataStructRWTransaction,
+    table: keyof IndexerSchema,
+    document: IndexerDocument
+  ) {
+    using _ = await this.measure(`insert`);
+    const existsNid = await trx
+      .objectStore('indexerRecords')
+      .index('id')
+      .getKey([table, document.id]);
+
+    if (existsNid) {
+      // delete exists one
+      await this.deleteByNid(trx, existsNid);
+    }
+
+    const dataMap = document.fields;
+
+    const nid = await trx
+      .objectStore('indexerRecords')
+      .put({ table, id: document.id, data: dataMap });
+
+    for (const [key, values] of dataMap) {
+      const type = IndexerSchema[table][
+        key as keyof IndexerSchema[typeof table]
+      ] as IndexerFieldSchema;
+      if (!type) {
+        continue;
+      }
+
+      const typeInfo = typeof type === 'string' ? { type } : type;
+      if (typeInfo.index !== false) {
+        // If index is false, the field will not be indexed
+        const iidx = this.invertedIndex.get(table)?.get(key);
+        if (!iidx) {
+          continue;
+        }
+        using _ = await this.measure(`insert[${typeInfo.type}]`);
+        await iidx.insert(trx, nid, values);
+      }
+    }
+  }
+
+  private async deleteByNid(trx: DataStructRWTransaction, nid: number) {
+    await trx.objectStore('indexerRecords').delete(nid);
+
+    const indexIds = await trx
+      .objectStore('invertedIndex')
+      .index('nid')
+      .getAllKeys(nid);
+
+    await Promise.all(
+      indexIds.map(indexId => trx.objectStore('invertedIndex').delete(indexId))
+    );
+  }
+
+  private async delete(
+    trx: DataStructRWTransaction,
+    table: keyof IndexerSchema,
+    id: string
+  ) {
+    using _ = await this.measure(`delete`);
+    const nid = await trx
+      .objectStore('indexerRecords')
+      .index('id')
+      .getKey([table, id]);
+
+    if (nid) {
+      await this.deleteByNid(trx, nid);
+    } else {
+      return;
+    }
+  }
+
+  private async deleteByQuery(
+    trx: DataStructRWTransaction,
+    table: keyof IndexerSchema,
+    query: Query<any>
+  ) {
+    using _ = await this.measure(`deleteByQuery`);
+    const match = await this.queryRaw(trx, table, query);
+
+    for (const nid of match.scores.keys()) {
+      await this.deleteByNid(trx, nid);
+    }
+  }
+
+  async batchWrite(
+    trx: DataStructRWTransaction,
+    table: keyof IndexerSchema,
+    deleteByQueries: Query<any>[],
+    deletes: string[],
+    inserts: IndexerDocument<any>[],
+    updates: IndexerDocument<any>[]
+  ) {
+    using _ = await this.measure(`batchWrite`);
+    for (const query of deleteByQueries) {
+      await this.deleteByQuery(trx, table, query);
+    }
+    for (const del of deletes) {
+      await this.delete(trx, table, del);
+    }
+    for (const inst of inserts) {
+      await this.insert(trx, table, inst);
+    }
+    for (const update of updates) {
+      await this.update(trx, table, update);
+    }
+  }
+
+  async matchAll(
+    trx: DataStructROTransaction,
+    table: keyof IndexerSchema
+  ): Promise<Match> {
+    const allNids = await trx
+      .objectStore('indexerRecords')
+      .index('table')
+      .getAllKeys(table);
+    const match = new Match();
+
+    for (const nid of allNids) {
+      match.addScore(nid, 1);
+    }
+    return match;
+  }
+
+  async queryRaw(
+    trx: DataStructROTransaction,
+    table: keyof IndexerSchema,
+    query: Query<any>,
+    cache: QueryCache = new QueryCache()
+  ): Promise<Match> {
+    const cached = cache.get(query);
+    if (cached) {
+      return cached;
+    }
+
+    const result = await (async () => {
+      using _ = await this.measure(`query[${query.type}]`);
+      if (query.type === 'match') {
+        const iidx = this.invertedIndex.get(table)?.get(query.field as string);
+        if (!iidx) {
+          return new Match();
+        }
+        return await iidx.match(trx, query.match);
+      } else if (query.type === 'boolean') {
+        const weights = [];
+        for (const q of query.queries) {
+          weights.push(await this.queryRaw(trx, table, q, cache));
+        }
+        if (query.occur === 'must') {
+          return weights.reduce((acc, w) => acc.and(w));
+        } else if (query.occur === 'must_not') {
+          const total = weights.reduce((acc, w) => acc.and(w));
+          return (await this.matchAll(trx, table)).exclude(total);
+        } else if (query.occur === 'should') {
+          return weights.reduce((acc, w) => acc.or(w));
+        }
+      } else if (query.type === 'all') {
+        return await this.matchAll(trx, table);
+      } else if (query.type === 'boost') {
+        return (await this.queryRaw(trx, table, query.query, cache)).boost(
+          query.boost
+        );
+      } else if (query.type === 'exists') {
+        const iidx = this.invertedIndex.get(table)?.get(query.field as string);
+        if (!iidx) {
+          return new Match();
+        }
+        return await iidx.all(trx);
+      }
+      throw new Error(`Query type '${query.type}' not supported`);
+    })();
+
+    cache.set(query, result);
+
+    return result;
+  }
+
+  async clear(trx: DataStructRWTransaction) {
+    await trx.objectStore('indexerRecords').clear();
+    await trx.objectStore('invertedIndex').clear();
+    await trx.objectStore('indexerMetadata').clear();
+  }
+
+  async search(
+    trx: DataStructROTransaction,
+    table: keyof IndexerSchema,
+    query: Query<any>,
+    options: SearchOptions<any> = {}
+  ): Promise<SearchResult<any, any>> {
+    const pagination = {
+      skip: options.pagination?.skip ?? 0,
+      limit: options.pagination?.limit ?? 100,
+    };
+
+    const match = await this.queryRaw(trx, table, query);
+
+    const nids = match
+      .toArray()
+      .slice(pagination.skip, pagination.skip + pagination.limit);
+
+    const nodes = [];
+    for (const nid of nids) {
+      const record = await trx.objectStore('indexerRecords').get(nid);
+      if (!record) {
+        continue;
+      }
+      nodes.push(this.resultNode(record, options, match, nid));
+    }
+
+    return {
+      pagination: {
+        count: match.size(),
+        hasMore: match.size() > pagination.limit + pagination.skip,
+        limit: pagination.limit,
+        skip: pagination.skip,
+      },
+      nodes: nodes,
+    };
+  }
+
+  async aggregate(
+    trx: DataStructROTransaction,
+    table: keyof IndexerSchema,
+    query: Query<any>,
+    field: string,
+    options: AggregateOptions<any> = {}
+  ): Promise<AggregateResult<any, any>> {
+    const pagination = {
+      skip: options.pagination?.skip ?? 0,
+      limit: options.pagination?.limit ?? 100,
+    };
+
+    const hitPagination = options.hits
+      ? {
+          skip: options.hits.pagination?.skip ?? 0,
+          limit: options.hits.pagination?.limit ?? 3,
+        }
+      : { skip: 0, limit: 0 };
+
+    const match = await this.queryRaw(trx, table, query);
+
+    const nids = match.toArray();
+
+    const buckets: {
+      key: string;
+      nids: number[];
+      hits: SearchResult<any, any>['nodes'];
+    }[] = [];
+
+    for (const nid of nids) {
+      const record = await trx.objectStore('indexerRecords').get(nid);
+      if (!record) {
+        continue;
+      }
+      const values = record.data.get(field);
+      for (const value of values ?? []) {
+        let bucket;
+        let bucketIndex = buckets.findIndex(b => b.key === value);
+        if (bucketIndex === -1) {
+          bucket = { key: value, nids: [], hits: [] };
+          buckets.push(bucket);
+          bucketIndex = buckets.length - 1;
+        } else {
+          bucket = buckets[bucketIndex];
+        }
+
+        if (
+          bucketIndex >= pagination.skip &&
+          bucketIndex < pagination.skip + pagination.limit
+        ) {
+          bucket.nids.push(nid);
+          if (
+            bucket.nids.length - 1 >= hitPagination.skip &&
+            bucket.nids.length - 1 < hitPagination.skip + hitPagination.limit
+          ) {
+            bucket.hits.push(
+              this.resultNode(record, options.hits ?? {}, match, nid)
+            );
+          }
+        }
+      }
+    }
+
+    return {
+      buckets: buckets
+        .slice(pagination.skip, pagination.skip + pagination.limit)
+        .map(bucket => {
+          const result = {
+            key: bucket.key,
+            score: match.getScore(bucket.nids[0]),
+            count: bucket.nids.length,
+          } as AggregateResult<any, any>['buckets'][number];
+
+          if (options.hits) {
+            (result as any).hits = {
+              pagination: {
+                count: bucket.nids.length,
+                hasMore:
+                  bucket.nids.length > hitPagination.limit + hitPagination.skip,
+                limit: hitPagination.limit,
+                skip: hitPagination.skip,
+              },
+              nodes: bucket.hits,
+            } as SearchResult<any, any>;
+          }
+
+          return result;
+        }),
+      pagination: {
+        count: buckets.length,
+        hasMore: buckets.length > pagination.limit + pagination.skip,
+        limit: pagination.limit,
+        skip: pagination.skip,
+      },
+    };
+  }
+
+  async readonly(database: IDBPDatabase<DocStorageSchema>) {
+    return database.transaction(
+      ['indexerRecords', 'invertedIndex', 'indexerMetadata'],
+      'readonly',
+      { durability: 'relaxed' }
+    );
+  }
+
+  async readwrite(database: IDBPDatabase<DocStorageSchema>) {
+    return database.transaction(
+      ['indexerRecords', 'invertedIndex', 'indexerMetadata'],
+      'readwrite',
+      { durability: 'relaxed' }
+    );
+  }
+
+  private resultNode(
+    record: { id: string; data: Map<string, string[]> },
+    options: SearchOptions<any>,
+    match?: Match,
+    nid?: number
+  ): SearchResult<any, any>['nodes'][number] {
+    const node = {
+      id: record.id,
+      score: match && nid ? match.getScore(nid) : 1,
+    } as any;
+
+    if (options.fields) {
+      const fields = {} as Record<string, string | string[]>;
+      for (const field of options.fields as string[]) {
+        fields[field] = record.data.get(field) ?? [''];
+        if (fields[field].length === 1) {
+          fields[field] = fields[field][0];
+        }
+      }
+      node.fields = fields;
+    }
+
+    if (match && nid && options.highlights) {
+      const highlights = {} as Record<string, string[]>;
+      for (const { field, before, end } of options.highlights) {
+        const highlightValues = match.getHighlighters(nid, field);
+        if (highlightValues) {
+          const rawValues = record.data.get(field) ?? [];
+          highlights[field] = Array.from(highlightValues)
+            .map(([index, ranges]) => {
+              const raw = rawValues[index];
+
+              if (raw) {
+                return (
+                  highlighter(raw, before, end, ranges, {
+                    maxPrefix: 20,
+                    maxLength: 50,
+                  }) ?? ''
+                );
+              }
+
+              return '';
+            })
+            .filter(Boolean);
+        }
+      }
+      node.highlights = highlights;
+    }
+
+    return node;
+  }
+
+  async measure(name: string) {
+    const count = debugMarkCount++;
+    performance.mark(`${name}Start(${count})`);
+    return {
+      [Symbol.dispose]: () => {
+        performance.mark(`${name}End(${count})`);
+        performance.measure(
+          `${name}`,
+          `${name}Start(${count})`,
+          `${name}End(${count})`
+        );
+      },
+    };
+  }
+}
+class QueryCache {
+  private readonly cache: [Query<any>, Match][] = [];
+
+  get(query: Query<any>) {
+    return this.cache.find(q => shallowEqual(q[0], query))?.[1];
+  }
+
+  set(query: Query<any>, match: Match) {
+    this.cache.push([query, match]);
+  }
+}

packages/common/nbstore/src/impls/idb/indexer/highlighter.ts

@@ -0,0 +1,77 @@
+export function highlighter(
+  originText: string,
+  before: string,
+  after: string,
+  matches: [number, number][],
+  {
+    maxLength = 50,
+    maxPrefix = 20,
+  }: { maxLength?: number; maxPrefix?: number } = {}
+) {
+  const merged = mergeRanges(matches);
+
+  if (merged.length === 0) {
+    return null;
+  }
+
+  const firstMatch = merged[0][0];
+  const start = Math.max(
+    0,
+    Math.min(firstMatch - maxPrefix, originText.length - maxLength)
+  );
+  const end = Math.min(start + maxLength, originText.length);
+  const text = originText.substring(start, end);
+
+  let result = '';
+
+  let pointer = 0;
+  for (const match of merged) {
+    const matchStart = match[0] - start;
+    const matchEnd = match[1] - start;
+    if (matchStart >= text.length) {
+      break;
+    }
+    result += text.substring(pointer, matchStart);
+    pointer = matchStart;
+    const highlighted = text.substring(matchStart, matchEnd);
+
+    if (highlighted.length === 0) {
+      continue;
+    }
+
+    result += `${before}${highlighted}${after}`;
+    pointer = matchEnd;
+  }
+  result += text.substring(pointer);
+
+  if (start > 0) {
+    result = `...${result}`;
+  }
+
+  if (end < originText.length) {
+    result = `${result}...`;
+  }
+
+  return result;
+}
+
+function mergeRanges(intervals: [number, number][]) {
+  if (intervals.length === 0) return [];
+
+  intervals.sort((a, b) => a[0] - b[0]);
+
+  const merged = [intervals[0]];
+
+  for (let i = 1; i < intervals.length; i++) {
+    const last = merged[merged.length - 1];
+    const current = intervals[i];
+
+    if (current[0] <= last[1]) {
+      last[1] = Math.max(last[1], current[1]);
+    } else {
+      merged.push(current);
+    }
+  }
+
+  return merged;
+}

packages/common/nbstore/src/impls/idb/indexer/index.ts

@@ -0,0 +1,227 @@
+import { merge, Observable, of, Subject, throttleTime } from 'rxjs';
+
+import { share } from '../../../connection';
+import type {
+  AggregateOptions,
+  AggregateResult,
+  IndexerDocument,
+  IndexerSchema,
+  Query,
+  SearchOptions,
+  SearchResult,
+} from '../../../storage';
+import { IndexerStorageBase } from '../../../storage';
+import { fromPromise } from '../../../utils/from-promise';
+import { IDBConnection, type IDBConnectionOptions } from '../db';
+import { DataStruct } from './data-struct';
+import { backoffRetry, exhaustMapWithTrailing } from './utils';
+
+export class IndexedDBIndexerStorage extends IndexerStorageBase {
+  static readonly identifier = 'IndexedDBIndexerStorage';
+  override recommendRefreshInterval: number = 0; // force refresh on each indexer operation
+  readonly connection = share(new IDBConnection(this.options));
+  override isReadonly = false;
+  private readonly data = new DataStruct();
+  private readonly tableUpdate$ = new Subject<string>();
+
+  /**
+   * The write operations of IndexedDBIndexerStorage are first cached in pendingUpdates,
+   * and then committed to IndexedDB in a batch through the refresh method.
+   */
+  private readonly pendingUpdates: Record<
+    keyof IndexerSchema,
+    {
+      deleteByQueries: Query<any>[];
+      deletes: string[];
+      inserts: IndexerDocument[];
+      updates: IndexerDocument[];
+    }
+  > = {
+    doc: { deleteByQueries: [], deletes: [], inserts: [], updates: [] },
+    block: { deleteByQueries: [], deletes: [], inserts: [], updates: [] },
+  };
+
+  get channel() {
+    return this.connection.inner.channel;
+  }
+
+  get database() {
+    return this.connection.inner.db;
+  }
+
+  constructor(private readonly options: IDBConnectionOptions) {
+    super();
+  }
+
+  override async search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>> {
+    const trx = await this.data.readonly(this.database);
+    return this.data.search(trx, table, query, options);
+  }
+  override async aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>> {
+    const trx = await this.data.readonly(this.database);
+    return this.data.aggregate(trx, table, query, field as string, options);
+  }
+  override search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>> {
+    return merge(of(1), this.watchTableUpdated(table)).pipe(
+      throttleTime(3000, undefined, { leading: true, trailing: true }),
+      exhaustMapWithTrailing(() => {
+        return fromPromise(async () => {
+          try {
+            const trx = await this.data.readonly(this.database);
+            return await this.data.search(trx, table, query, options);
+          } catch (error) {
+            console.error('search error', error);
+            throw error;
+          }
+        }).pipe(backoffRetry());
+      })
+    );
+  }
+  override aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return merge(of(1), this.watchTableUpdated(table)).pipe(
+      throttleTime(3000, undefined, { leading: true, trailing: true }),
+      exhaustMapWithTrailing(() => {
+        return fromPromise(async () => {
+          try {
+            const trx = await this.data.readonly(this.database);
+            return await this.data.aggregate(
+              trx,
+              table,
+              query,
+              field as string,
+              options
+            );
+          } catch (error) {
+            console.error('aggregate error', error);
+            throw error;
+          }
+        }).pipe(backoffRetry());
+      })
+    );
+  }
+
+  override async deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void> {
+    this.pendingUpdates[table].deleteByQueries.push(query);
+  }
+
+  override insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument
+  ): Promise<void> {
+    this.pendingUpdates[table].inserts.push(document);
+    return Promise.resolve();
+  }
+
+  override delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void> {
+    this.pendingUpdates[table].deletes.push(id);
+    return Promise.resolve();
+  }
+
+  override update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument
+  ): Promise<void> {
+    this.pendingUpdates[table].updates.push(document);
+    return Promise.resolve();
+  }
+
+  override async refresh<T extends keyof IndexerSchema>(
+    table: T
+  ): Promise<void> {
+    const trx = await this.data.readwrite(this.database);
+    const tables = table ? [table] : (['doc', 'block'] as const);
+    for (const table of tables) {
+      await this.data.batchWrite(
+        trx,
+        table,
+        this.pendingUpdates[table].deleteByQueries,
+        this.pendingUpdates[table].deletes,
+        this.pendingUpdates[table].inserts,
+        this.pendingUpdates[table].updates
+      );
+      this.pendingUpdates[table] = {
+        deleteByQueries: [],
+        deletes: [],
+        inserts: [],
+        updates: [],
+      };
+    }
+    this.emitTableUpdated(table);
+  }
+
+  override async refreshIfNeed(): Promise<void> {
+    const needRefreshTable = Object.entries(this.pendingUpdates)
+      .filter(
+        ([, updates]) =>
+          updates.deleteByQueries.length > 0 ||
+          updates.deletes.length > 0 ||
+          updates.inserts.length > 0 ||
+          updates.updates.length > 0
+      )
+      .map(([table]) => table as keyof IndexerSchema);
+    for (const table of needRefreshTable) {
+      await this.refresh(table);
+    }
+  }
+
+  private watchTableUpdated(table: keyof IndexerSchema) {
+    return new Observable(subscriber => {
+      const listener = (ev: MessageEvent) => {
+        if (ev.data.type === 'indexer-updated' && ev.data.table === table) {
+          subscriber.next(1);
+        }
+      };
+
+      const subscription = this.tableUpdate$.subscribe(updatedTable => {
+        if (updatedTable === table) {
+          subscriber.next(1);
+        }
+      });
+
+      this.channel.addEventListener('message', listener);
+      return () => {
+        this.channel.removeEventListener('message', listener);
+        subscription.unsubscribe();
+      };
+    });
+  }
+
+  emitTableUpdated(table: keyof IndexerSchema) {
+    this.tableUpdate$.next(table);
+    this.channel.postMessage({ type: 'indexer-updated', table });
+  }
+
+  // Get the current indexer version
+  // increase this number to re-index all docs
+  async indexVersion(): Promise<number> {
+    return 1;
+  }
+}

packages/common/nbstore/src/impls/idb/indexer/inverted-index.ts

@@ -0,0 +1,537 @@
+import { bm25 } from './bm25';
+import type {
+  DataStructROTransaction,
+  DataStructRWTransaction,
+} from './data-struct';
+import { Match } from './match';
+import { GeneralTokenizer, type Token } from './tokenizer';
+
+export interface InvertedIndex {
+  fieldKey: string;
+
+  match(trx: DataStructROTransaction, term: string): Promise<Match>;
+
+  all(trx: DataStructROTransaction): Promise<Match>;
+
+  insert(
+    trx: DataStructRWTransaction,
+    id: number,
+    terms: string[]
+  ): Promise<void>;
+}
+
+export class StringInvertedIndex implements InvertedIndex {
+  constructor(
+    readonly table: string,
+    readonly fieldKey: string
+  ) {}
+
+  async match(trx: DataStructROTransaction, term: string): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll([
+        this.table,
+        InvertedIndexKey.forString(this.fieldKey, term).buffer(),
+      ]);
+    const match = new Match();
+    for (const obj of objs) {
+      match.addScore(obj.nid, 1);
+    }
+    return match;
+  }
+
+  async all(trx: DataStructROTransaction): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll(
+        IDBKeyRange.bound(
+          [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
+          [
+            this.table,
+            InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
+          ]
+        )
+      );
+
+    const set = new Set<number>();
+    for (const obj of objs) {
+      set.add(obj.nid);
+    }
+
+    const match = new Match();
+    for (const nid of set) {
+      match.addScore(nid, 1);
+    }
+    return match;
+  }
+
+  async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
+    const uniqueTerms = new Set(terms);
+    await Promise.all(
+      Array.from(uniqueTerms).map(term =>
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
+          nid: id,
+        })
+      )
+    );
+  }
+}
+
+export class IntegerInvertedIndex implements InvertedIndex {
+  constructor(
+    readonly table: string,
+    readonly fieldKey: string
+  ) {}
+
+  async match(trx: DataStructROTransaction, term: string): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll([
+        this.table,
+        InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
+      ]);
+    const match = new Match();
+    for (const obj of objs) {
+      match.addScore(obj.nid, 1);
+    }
+    return match;
+  }
+
+  // oxlint-disable-next-line sonarjs/no-identical-functions
+  async all(trx: DataStructROTransaction): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll(
+        IDBKeyRange.bound(
+          [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
+          [
+            this.table,
+            InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
+          ]
+        )
+      );
+
+    const set = new Set<number>();
+    for (const obj of objs) {
+      set.add(obj.nid);
+    }
+
+    const match = new Match();
+    for (const nid of set) {
+      match.addScore(nid, 1);
+    }
+    return match;
+  }
+
+  async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
+    const uniqueTerms = new Set(terms);
+    await Promise.all(
+      Array.from(uniqueTerms).map(term =>
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
+          nid: id,
+        })
+      )
+    );
+  }
+}
+
+export class BooleanInvertedIndex implements InvertedIndex {
+  constructor(
+    readonly table: string,
+    readonly fieldKey: string
+  ) {}
+
+  // oxlint-disable-next-line sonarjs/no-identical-functions
+  async all(trx: DataStructROTransaction): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll(
+        IDBKeyRange.bound(
+          [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
+          [
+            this.table,
+            InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
+          ]
+        )
+      );
+
+    const set = new Set<number>();
+    for (const obj of objs) {
+      set.add(obj.nid);
+    }
+
+    const match = new Match();
+    for (const nid of set) {
+      match.addScore(nid, 1);
+    }
+    return match;
+  }
+
+  async match(trx: DataStructROTransaction, term: string): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll([
+        this.table,
+        InvertedIndexKey.forBoolean(this.fieldKey, term === 'true').buffer(),
+      ]);
+    const match = new Match();
+    for (const obj of objs) {
+      match.addScore(obj.nid, 1);
+    }
+    return match;
+  }
+
+  async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
+    const uniqueTerms = new Set(terms);
+    await Promise.all(
+      Array.from(uniqueTerms).map(term =>
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forBoolean(
+            this.fieldKey,
+            term === 'true'
+          ).buffer(),
+          nid: id,
+        })
+      )
+    );
+  }
+}
+
+export class FullTextInvertedIndex implements InvertedIndex {
+  constructor(
+    readonly table: string,
+    readonly fieldKey: string
+  ) {}
+
+  async match(trx: DataStructROTransaction, term: string): Promise<Match> {
+    const queryTokens = new GeneralTokenizer().tokenize(term);
+    const matched = new Map<
+      number,
+      Map<
+        number, // index
+        {
+          score: number;
+          ranges: [number, number][];
+        }
+      >
+    >();
+    const avgFieldLength =
+      (
+        await trx
+          .objectStore('indexerMetadata')
+          .get(`full-text:avg-field-length:${this.table}:${this.fieldKey}`)
+      )?.value ?? 0;
+    for (const token of queryTokens) {
+      const key = InvertedIndexKey.forString(this.fieldKey, token.term);
+      const objs = [
+        // match exact
+        ...(await trx
+          .objectStore('invertedIndex')
+          .index('key')
+          .getAll([this.table, key.buffer()])),
+        // match prefix
+        ...(await trx
+          .objectStore('invertedIndex')
+          .index('key')
+          .getAll(
+            IDBKeyRange.bound(
+              [this.table, key.buffer()],
+              [this.table, key.add1().buffer()],
+              true,
+              true
+            ),
+            5000 // get maximum 5000 items for prefix match
+          )),
+      ];
+      const submatched: {
+        nid: number;
+        score: number;
+        position: {
+          index: number;
+          ranges: [number, number][];
+        };
+      }[] = [];
+      for (const obj of objs) {
+        if (!obj) {
+          continue;
+        }
+        const key = InvertedIndexKey.fromBuffer(obj.key);
+        const originTokenTerm = key.asString();
+        const matchLength = token.term.length;
+        let positions = obj.pos
+          ? Array.isArray(obj.pos)
+            ? obj.pos
+            : [obj.pos]
+          : [
+              {
+                i: 0,
+                l: 0,
+                rs: [] as [number, number][],
+              },
+            ];
+
+        for (const position of positions) {
+          const termFreq = position.rs.length;
+          const totalCount = objs.length;
+          const fieldLength = position.l;
+          const score =
+            bm25(termFreq, 1, totalCount, fieldLength, avgFieldLength) *
+            (matchLength / originTokenTerm.length);
+          submatched.push({
+            nid: obj.nid,
+            score,
+            position: {
+              index: position.i,
+              ranges: position.rs.map(([start, _end]) => [
+                start,
+                start + matchLength,
+              ]),
+            },
+          });
+        }
+      }
+
+      // normalize score
+      const maxScore = submatched.reduce((acc, s) => Math.max(acc, s.score), 0);
+      const minScore = submatched.reduce((acc, s) => Math.min(acc, s.score), 0);
+      for (const { nid, score, position } of submatched) {
+        const normalizedScore =
+          maxScore === minScore
+            ? score
+            : (score - minScore) / (maxScore - minScore);
+        const match =
+          matched.get(nid) ??
+          new Map<
+            number, // index
+            {
+              score: number;
+              ranges: [number, number][];
+            }
+          >();
+        const item = match.get(position.index) || {
+          score: 0,
+          ranges: [],
+        };
+        item.score += normalizedScore;
+        item.ranges.push(...position.ranges);
+        match.set(position.index, item);
+        matched.set(nid, match);
+      }
+    }
+    const match = new Match();
+    for (const [nid, items] of matched) {
+      if (items.size === 0) {
+        break;
+      }
+      let highestScore = -1;
+      let highestIndex = -1;
+      let highestRanges: [number, number][] = [];
+      for (const [index, { score, ranges }] of items) {
+        if (score > highestScore) {
+          highestScore = score;
+          highestIndex = index;
+          highestRanges = ranges;
+        }
+      }
+      match.addScore(nid, highestScore);
+      match.addHighlighter(nid, this.fieldKey, highestIndex, highestRanges);
+    }
+    return match;
+  }
+
+  // oxlint-disable-next-line sonarjs/no-identical-functions
+  async all(trx: DataStructROTransaction): Promise<Match> {
+    const objs = await trx
+      .objectStore('invertedIndex')
+      .index('key')
+      .getAll(
+        IDBKeyRange.bound(
+          [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
+          [
+            this.table,
+            InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
+          ]
+        )
+      );
+
+    const set = new Set<number>();
+    for (const obj of objs) {
+      set.add(obj.nid);
+    }
+
+    const match = new Match();
+    for (const nid of set) {
+      match.addScore(nid, 1);
+    }
+    return match;
+  }
+
+  async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
+    const promises: Promise<any>[] = [];
+    const totalTermLength = terms.reduce((acc, term) => acc + term.length, 0);
+    const globalTokenMap = new Map<
+      string,
+      { l: number; i: number; rs: [number, number][] }[]
+    >();
+
+    for (let i = 0; i < terms.length; i++) {
+      const tokenMap = new Map<string, Token[]>();
+      const originString = terms[i];
+
+      const tokens = new GeneralTokenizer().tokenize(originString);
+
+      for (const token of tokens) {
+        const tokens = tokenMap.get(token.term) || [];
+        tokens.push(token);
+        tokenMap.set(token.term, tokens);
+      }
+
+      for (const [term, tokens] of tokenMap) {
+        const entry = globalTokenMap.get(term) || [];
+        entry.push({
+          l: originString.length,
+          i: i,
+          rs: tokens.map(token => [token.start, token.end]),
+        });
+        globalTokenMap.set(term, entry);
+      }
+    }
+
+    for (const [term, positions] of globalTokenMap) {
+      promises.push(
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
+          nid: id,
+          pos: positions,
+        })
+      );
+    }
+
+    const indexerMetadataStore = trx.objectStore('indexerMetadata');
+    const countKey = `full-text:field-count:${this.table}:${this.fieldKey}`;
+    const avgKey = `full-text:avg-field-length:${this.table}:${this.fieldKey}`;
+
+    const [countObj, avgObj] = await Promise.all([
+      indexerMetadataStore.get(countKey),
+      indexerMetadataStore.get(avgKey),
+    ]);
+
+    const totalCount = countObj?.value ?? 0;
+    const avgFieldLength = avgObj?.value ?? 0;
+
+    const newTotalCount = totalCount + terms.length;
+    const newAvgFieldLength =
+      (avgFieldLength * totalCount + totalTermLength) / newTotalCount;
+
+    promises.push(
+      indexerMetadataStore.put({
+        key: countKey,
+        value: newTotalCount,
+      })
+    );
+    promises.push(
+      indexerMetadataStore.put({
+        key: avgKey,
+        value: isNaN(newAvgFieldLength) ? 0 : newAvgFieldLength,
+      })
+    );
+
+    await Promise.all(promises);
+  }
+}
+
+export class InvertedIndexKey {
+  constructor(
+    readonly field: Uint8Array,
+    readonly value: Uint8Array,
+    readonly gap: Uint8Array = new Uint8Array([58])
+  ) {}
+
+  asString() {
+    return new TextDecoder().decode(this.value);
+  }
+
+  asInt64() {
+    return new DataView(this.value.buffer).getBigInt64(
+      0,
+      false
+    ); /* big-endian */
+  }
+
+  add1() {
+    if (this.value.byteLength > 0) {
+      const bytes = new Uint8Array(this.value.slice(0));
+      let carry = 1;
+      for (let i = bytes.length - 1; i >= 0 && carry > 0; i--) {
+        const sum = bytes[i] + carry;
+        bytes[i] = sum % 256;
+        carry = sum >> 8;
+      }
+      return new InvertedIndexKey(this.field, bytes);
+    } else {
+      return new InvertedIndexKey(
+        this.field,
+        new Uint8Array(0),
+        new Uint8Array([59])
+      );
+    }
+  }
+
+  static forPrefix(field: string) {
+    return new InvertedIndexKey(
+      new TextEncoder().encode(field),
+      new Uint8Array(0)
+    );
+  }
+
+  static forString(field: string, value: string) {
+    return new InvertedIndexKey(
+      new TextEncoder().encode(field),
+      new TextEncoder().encode(value)
+    );
+  }
+
+  static forBoolean(field: string, value: boolean) {
+    const bytes = new Uint8Array(1);
+    bytes.set([value ? 1 : 0]);
+    return new InvertedIndexKey(new TextEncoder().encode(field), bytes);
+  }
+
+  static forInt64(field: string, value: bigint) {
+    const bytes = new Uint8Array(8);
+    new DataView(bytes.buffer).setBigInt64(0, value, false); /* big-endian */
+    return new InvertedIndexKey(new TextEncoder().encode(field), bytes);
+  }
+
+  buffer() {
+    const tmp = new Uint8Array(
+      this.field.byteLength + (this.value?.byteLength ?? 0) + 1
+    );
+    tmp.set(new Uint8Array(this.field), 0);
+    tmp.set(new Uint8Array(this.gap), this.field.byteLength);
+    if (this.value.byteLength > 0) {
+      tmp.set(new Uint8Array(this.value), this.field.byteLength + 1);
+    }
+    return tmp.buffer;
+  }
+
+  static fromBuffer(buffer: ArrayBuffer) {
+    const array = new Uint8Array(buffer);
+    const fieldLength = array.indexOf(58);
+    const field = array.slice(0, fieldLength);
+    const value = array.slice(fieldLength + 1);
+    return new InvertedIndexKey(field, value);
+  }
+}

packages/common/nbstore/src/impls/idb/indexer/match.ts

@@ -0,0 +1,105 @@
+export class Match {
+  scores = new Map<number, number>();
+  /**
+   * nid -> field -> index(multi value field) -> [start, end][]
+   */
+  highlighters = new Map<
+    number,
+    Map<string, Map<number, [number, number][]>>
+  >();
+
+  constructor() {}
+
+  size() {
+    return this.scores.size;
+  }
+
+  getScore(id: number) {
+    return this.scores.get(id) ?? 0;
+  }
+
+  addScore(id: number, score: number) {
+    const currentScore = this.scores.get(id) || 0;
+    this.scores.set(id, currentScore + score);
+  }
+
+  getHighlighters(id: number, field: string) {
+    return this.highlighters.get(id)?.get(field);
+  }
+
+  addHighlighter(
+    id: number,
+    field: string,
+    index: number,
+    newRanges: [number, number][]
+  ) {
+    const fields =
+      this.highlighters.get(id) ||
+      new Map<string, Map<number, [number, number][]>>();
+    const values = fields.get(field) || new Map<number, [number, number][]>();
+    const ranges = values.get(index) || [];
+    ranges.push(...newRanges);
+    values.set(index, ranges);
+    fields.set(field, values);
+    this.highlighters.set(id, fields);
+  }
+
+  and(other: Match) {
+    const newWeight = new Match();
+    for (const [id, score] of this.scores) {
+      if (other.scores.has(id)) {
+        newWeight.addScore(id, score + (other.scores.get(id) ?? 0));
+        newWeight.copyExtData(this, id);
+        newWeight.copyExtData(other, id);
+      }
+    }
+    return newWeight;
+  }
+
+  or(other: Match) {
+    const newWeight = new Match();
+    for (const [id, score] of this.scores) {
+      newWeight.addScore(id, score);
+      newWeight.copyExtData(this, id);
+    }
+    for (const [id, score] of other.scores) {
+      newWeight.addScore(id, score);
+      newWeight.copyExtData(other, id);
+    }
+    return newWeight;
+  }
+
+  exclude(other: Match) {
+    const newWeight = new Match();
+    for (const [id, score] of this.scores) {
+      if (!other.scores.has(id)) {
+        newWeight.addScore(id, score);
+        newWeight.copyExtData(this, id);
+      }
+    }
+    return newWeight;
+  }
+
+  boost(boost: number) {
+    const newWeight = new Match();
+    for (const [id, score] of this.scores) {
+      newWeight.addScore(id, score * boost);
+      newWeight.copyExtData(this, id);
+    }
+    return newWeight;
+  }
+
+  toArray() {
+    return Array.from(this.scores.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(e => e[0]);
+  }
+
+  private copyExtData(from: Match, id: number) {
+    for (const [field, values] of from.highlighters.get(id) ?? []) {
+      for (const [index, ranges] of values) {
+        this.addHighlighter(id, field, index, ranges);
+      }
+    }
+  }
+}

packages/common/nbstore/src/impls/idb/indexer/tokenizer.ts

@@ -0,0 +1,162 @@
+import Graphemer from 'graphemer';
+
+export interface Tokenizer {
+  tokenize(text: string): Token[];
+}
+
+export interface Token {
+  term: string;
+  start: number;
+  end: number;
+}
+
+export class SimpleTokenizer implements Tokenizer {
+  tokenize(text: string): Token[] {
+    const tokens: Token[] = [];
+    let start = 0;
+    let end = 0;
+    let inWord = false;
+    for (let i = 0; i < text.length; i++) {
+      const c = text[i];
+      if (c.match(/[\n\r\p{Z}\p{P}]/u)) {
+        if (inWord) {
+          end = i;
+          tokens.push({
+            term: text.substring(start, end).toLowerCase(),
+            start,
+            end,
+          });
+          inWord = false;
+        }
+      } else {
+        if (!inWord) {
+          start = i;
+          end = i;
+          inWord = true;
+        }
+      }
+    }
+    if (inWord) {
+      tokens.push({
+        term: text.substring(start).toLowerCase(),
+        start,
+        end: text.length,
+      });
+    }
+    return tokens;
+  }
+}
+
+export class NGramTokenizer implements Tokenizer {
+  constructor(private readonly n: number) {}
+
+  tokenize(text: string): Token[] {
+    const splitted: Token[] = [];
+    for (let i = 0; i < text.length;) {
+      const nextBreak = Graphemer.nextBreak(text, i);
+      const c = text.substring(i, nextBreak);
+
+      splitted.push({
+        term: c,
+        start: i,
+        end: nextBreak,
+      });
+
+      i = nextBreak;
+    }
+    const tokens: Token[] = [];
+    for (let i = 0; i < splitted.length - this.n + 1; i++) {
+      tokens.push(
+        splitted.slice(i, i + this.n).reduce(
+          (acc, t) => ({
+            term: acc.term + t.term,
+            start: Math.min(acc.start, t.start),
+            end: Math.max(acc.end, t.end),
+          }),
+          { term: '', start: Infinity, end: -Infinity }
+        )
+      );
+    }
+    return tokens;
+  }
+}
+
+export class GeneralTokenizer implements Tokenizer {
+  constructor() {}
+
+  tokenizeWord(word: string, lang: string): Token[] {
+    if (lang === 'en') {
+      return [{ term: word.toLowerCase(), start: 0, end: word.length }];
+    } else if (lang === 'cjk') {
+      if (word.length < 3) {
+        return [{ term: word, start: 0, end: word.length }];
+      }
+      return new NGramTokenizer(2).tokenize(word);
+    } else if (lang === 'emoji') {
+      return new NGramTokenizer(1).tokenize(word);
+    } else if (lang === '-') {
+      return [];
+    }
+
+    throw new Error('Not implemented');
+  }
+
+  testLang(c: string): string {
+    if (c.match(/[\p{Emoji}]/u)) {
+      return 'emoji';
+    } else if (c.match(/[\p{sc=Han}\p{scx=Hira}\p{scx=Kana}\p{sc=Hang}]/u)) {
+      return 'cjk';
+    } else if (c.match(/[\n\r\p{Z}\p{P}]/u)) {
+      return '-';
+    } else {
+      return 'en';
+    }
+  }
+
+  tokenize(text: string): Token[] {
+    const tokens: Token[] = [];
+    let start = 0;
+    let end = 0;
+    let lang: string | null = null;
+
+    for (let i = 0; i < text.length;) {
+      const nextBreak = Graphemer.nextBreak(text, i);
+      const c = text.substring(i, nextBreak);
+
+      const l = this.testLang(c);
+      if (lang !== l) {
+        if (lang !== null) {
+          end = i;
+          tokens.push(
+            ...this.tokenizeWord(text.substring(start, end), lang).map(
+              token => ({
+                ...token,
+                start: token.start + start,
+                end: token.end + start,
+              })
+            )
+          );
+        }
+
+        start = i;
+        end = i;
+        lang = l;
+      }
+
+      i = nextBreak;
+    }
+    if (lang !== null) {
+      tokens.push(
+        ...this.tokenizeWord(text.substring(start, text.length), lang).map(
+          token => ({
+            ...token,
+            start: token.start + start,
+            end: token.end + start,
+          })
+        )
+      );
+    }
+
+    return tokens;
+  }
+}

packages/common/nbstore/src/impls/idb/indexer/utils.ts

@@ -0,0 +1,75 @@
+import {
+  asyncScheduler,
+  defer,
+  exhaustMap,
+  finalize,
+  type Observable,
+  type ObservableInput,
+  type OperatorFunction,
+  retry,
+  scheduled,
+  Subject,
+  throttle,
+  throwError,
+  timer,
+} from 'rxjs';
+
+/**
+ * Like exhaustMap, but also includes the trailing value emitted from the source observable while waiting for the preceding inner observable to complete
+ *
+ * Original code adapted from https://github.com/ReactiveX/rxjs/issues/5004
+ * @param {function<T, K>(value: T, ?index: number): ObservableInput<K>} project - A function that, when applied to an item emitted by the
+ * source Observable, returns a projected Observable.
+ */
+export function exhaustMapWithTrailing<T, R>(
+  project: (value: T, index: number) => ObservableInput<R>
+): OperatorFunction<T, R> {
+  return (source$): Observable<R> =>
+    defer(() => {
+      const release$ = new Subject<void>();
+      return source$.pipe(
+        throttle(() => release$, {
+          leading: true,
+          trailing: true,
+        }),
+        exhaustMap((value, index) =>
+          scheduled(project(value, index), asyncScheduler).pipe(
+            finalize(() => {
+              release$.next();
+            })
+          )
+        )
+      );
+    });
+}
+
+/**
+ * An operator that retries the source observable when an error occurs.
+ *
+ * https://en.wikipedia.org/wiki/Exponential_backoff
+ */
+export function backoffRetry<T>({
+  when,
+  count = 3,
+  delay = 200,
+  maxDelay = 15000,
+}: {
+  when?: (err: any) => boolean;
+  count?: number;
+  delay?: number;
+  maxDelay?: number;
+} = {}) {
+  return (obs$: Observable<T>) =>
+    obs$.pipe(
+      retry({
+        count,
+        delay: (err, retryIndex) => {
+          if (when && !when(err)) {
+            return throwError(() => err);
+          }
+          const d = Math.pow(2, retryIndex - 1) * delay;
+          return timer(Math.min(d, maxDelay));
+        },
+      })
+    );
+}

packages/common/nbstore/src/impls/idb/lock.ts

@@ -0,0 +1,83 @@
+import EventEmitter2 from 'eventemitter2';
+
+import { type Locker } from '../../storage/lock';
+import type { IDBConnection } from './db';
+
+interface ChannelMessage {
+  type: 'unlock';
+  key: string;
+}
+
+export class IndexedDBLocker implements Locker {
+  get db() {
+    return this.dbConnection.inner.db;
+  }
+  private readonly eventEmitter = new EventEmitter2();
+
+  get channel() {
+    return this.dbConnection.inner.channel;
+  }
+
+  constructor(private readonly dbConnection: IDBConnection) {}
+
+  async lock(domain: string, resource: string) {
+    const key = `${domain}:${resource}`;
+
+    // oxlint-disable-next-line no-constant-condition
+    while (true) {
+      const trx = this.db.transaction('locks', 'readwrite');
+      const record = await trx.store.get(key);
+      const lockTimestamp = record?.lock.getTime();
+
+      if (
+        lockTimestamp &&
+        lockTimestamp > Date.now() - 30000 /* lock timeout 3s */
+      ) {
+        trx.commit();
+
+        await new Promise<void>(resolve => {
+          const cleanup = () => {
+            this.channel.removeEventListener('message', channelListener);
+            this.eventEmitter.off('unlock', eventListener);
+            clearTimeout(timer);
+          };
+          const channelListener = (event: MessageEvent<ChannelMessage>) => {
+            if (event.data.type === 'unlock' && event.data.key === key) {
+              cleanup();
+              resolve();
+            }
+          };
+          const eventListener = (unlockKey: string) => {
+            if (unlockKey === key) {
+              cleanup();
+              resolve();
+            }
+          };
+          this.channel.addEventListener('message', channelListener); // add listener
+          this.eventEmitter.on('unlock', eventListener);
+
+          const timer = setTimeout(() => {
+            cleanup();
+            resolve();
+          }, 3000);
+          // timeout to avoid dead lock
+        });
+        continue;
+      } else {
+        await trx.store.put({ key, lock: new Date() });
+        trx.commit();
+        break;
+      }
+    }
+
+    return {
+      [Symbol.asyncDispose]: async () => {
+        const trx = this.db.transaction('locks', 'readwrite');
+        await trx.store.delete(key);
+        trx.commit();
+        this.channel.postMessage({ type: 'unlock', key });
+        this.eventEmitter.emit('unlock', key);
+      },
+    };
+  }
+}

packages/common/nbstore/src/impls/idb/schema.ts

@@ -0,0 +1,279 @@
+import { type DBSchema, type OpenDBCallbacks } from 'idb';
+
+/**
+IndexedDB
+  > DB(workspace:${workspaceId})
+     > Table(Snapshots)
+     > Table(Updates)
+     > Table(...)
+
+Table(Snapshots)
+| docId | blob | createdAt | updatedAt |
+|-------|------|-----------|-----------|
+|  str  | bin  |   Date    |   Date    |
+
+Table(Updates)
+| id | docId | blob | createdAt |
+|----|-------|------|-----------|
+|auto|  str  | bin  |   Date    |
+
+Table(Clocks)
+| docId |   clock   |
+|-------|-----------|
+|  str  |   Date    |
+
+Table(Blobs)
+| key | mime | size | createdAt | deletedAt |
+|-----|------|------|-----------|-----------|
+| str |  str | num  |   Date    |   Date    |
+
+Table(BlobData)
+| key | data |
+|-----|------|
+| str | bin  |
+
+Table(PeerClocks)
+| peer | docId |   clock   |  pushed   |
+|------|-------|-----------|-----------|
+| str  |  str  |   Date    |   Date    |
+
+Table(IndexerSync)
+| docId | indexedClock | indexerVersion |
+|-------|--------------|----------------|
+| str   |   Date       |    number      |
+
+Table(BlobSync)
+| peer | key | uploadedAt |
+|------|-----|------------|
+| str  | str |   Date     |
+ */
+export interface DocStorageSchema extends DBSchema {
+  snapshots: {
+    key: string;
+    value: {
+      docId: string;
+      bin: Uint8Array;
+      createdAt: Date;
+      updatedAt: Date;
+    };
+    indexes: {
+      updatedAt: Date;
+    };
+  };
+  updates: {
+    key: [string, Date];
+    value: {
+      docId: string;
+      bin: Uint8Array;
+      createdAt: Date;
+    };
+    indexes: {
+      docId: string;
+    };
+  };
+  clocks: {
+    key: string;
+    value: {
+      docId: string;
+      timestamp: Date;
+    };
+    indexes: {
+      timestamp: Date;
+    };
+  };
+  blobs: {
+    key: string;
+    value: {
+      key: string;
+      mime: string;
+      size: number;
+      createdAt: Date;
+      deletedAt: Date | null;
+    };
+  };
+  blobSync: {
+    key: [string, string];
+    value: {
+      peer: string;
+      key: string;
+      uploadedAt: Date | null;
+    };
+    indexes: {
+      peer: string;
+    };
+  };
+  blobData: {
+    key: string;
+    value: {
+      key: string;
+      data: Uint8Array;
+    };
+  };
+  peerClocks: {
+    key: [string, string];
+    value: {
+      peer: string;
+      docId: string;
+      clock: Date;
+      pulledClock: Date;
+      pushedClock: Date;
+    };
+    indexes: {
+      peer: string;
+    };
+  };
+  locks: {
+    key: string;
+    value: {
+      key: string;
+      lock: Date;
+    };
+  };
+  indexerSync: {
+    key: string;
+    value: {
+      docId: string;
+      indexedClock: Date;
+      indexerVersion?: number;
+    };
+  };
+  indexerMetadata: {
+    key: string;
+    value: {
+      key: string;
+      value: any;
+    };
+  };
+  indexerRecords: {
+    key: number;
+    value: {
+      table: string;
+      id: string;
+      data: Map<string, string[]>;
+    };
+    indexes: { table: string; id: [string, string] };
+  };
+  invertedIndex: {
+    key: number;
+    value: {
+      table: string;
+      nid: number;
+      pos?: {
+        i: number /* index */;
+        l: number /* length */;
+        rs: [number, number][] /* ranges: [start, end] */;
+      }[];
+      key: ArrayBuffer;
+    };
+    indexes: { key: [string, ArrayBuffer]; nid: number };
+  };
+}
+
+const migrate: OpenDBCallbacks<DocStorageSchema>['upgrade'] = (
+  db,
+  oldVersion,
+  _newVersion,
+  trx
+) => {
+  if (!oldVersion) {
+    oldVersion = 0;
+  }
+
+  for (let i = oldVersion; i < migrations.length; i++) {
+    migrations[i](db, trx);
+  }
+};
+
+type MigrateParameters = Parameters<
+  NonNullable<OpenDBCallbacks<DocStorageSchema>['upgrade']>
+>;
+type Migrate = (db: MigrateParameters[0], trx: MigrateParameters[3]) => void;
+
+// START REGION: migrations
+const init: Migrate = db => {
+  const snapshots = db.createObjectStore('snapshots', {
+    keyPath: 'docId',
+    autoIncrement: false,
+  });
+
+  snapshots.createIndex('updatedAt', 'updatedAt', { unique: false });
+
+  const updates = db.createObjectStore('updates', {
+    keyPath: ['docId', 'createdAt'],
+    autoIncrement: false,
+  });
+
+  updates.createIndex('docId', 'docId', { unique: false });
+
+  const clocks = db.createObjectStore('clocks', {
+    keyPath: 'docId',
+    autoIncrement: false,
+  });
+
+  clocks.createIndex('timestamp', 'timestamp', { unique: false });
+
+  const peerClocks = db.createObjectStore('peerClocks', {
+    keyPath: ['peer', 'docId'],
+    autoIncrement: false,
+  });
+
+  peerClocks.createIndex('peer', 'peer', { unique: false });
+
+  db.createObjectStore('blobs', {
+    keyPath: 'key',
+    autoIncrement: false,
+  });
+
+  db.createObjectStore('blobData', {
+    keyPath: 'key',
+    autoIncrement: false,
+  });
+
+  db.createObjectStore('locks', {
+    keyPath: 'key',
+    autoIncrement: false,
+  });
+};
+const initBlobSync: Migrate = db => {
+  const blobSync = db.createObjectStore('blobSync', {
+    keyPath: ['peer', 'key'],
+    autoIncrement: false,
+  });
+
+  blobSync.createIndex('peer', 'peer', { unique: false });
+};
+const initIndexer: Migrate = db => {
+  db.createObjectStore('indexerMetadata', {
+    keyPath: 'key',
+  });
+  const indexRecordsStore = db.createObjectStore('indexerRecords', {
+    autoIncrement: true,
+  });
+  indexRecordsStore.createIndex('table', 'table', {
+    unique: false,
+  });
+  indexRecordsStore.createIndex('id', ['table', 'id'], {
+    unique: true,
+  });
+  const invertedIndexStore = db.createObjectStore('invertedIndex', {
+    autoIncrement: true,
+  });
+  invertedIndexStore.createIndex('key', ['table', 'key'], {
+    unique: false,
+  });
+  invertedIndexStore.createIndex('nid', 'nid', { unique: false });
+  db.createObjectStore('indexerSync', {
+    keyPath: 'docId',
+    autoIncrement: false,
+  });
+};
+// END REGION
+
+// 1. all schema changed should be put in migrations
+// 2. order matters
+const migrations: Migrate[] = [init, initBlobSync, initIndexer];
+
+export const migrator = {
+  version: migrations.length,
+  migrate,
+};

packages/common/nbstore/src/impls/idb/v1/blob.ts

@@ -0,0 +1,78 @@
+import { share } from '../../../connection';
+import { BlobStorageBase, type ListedBlobRecord } from '../../../storage';
+import { BlobIDBConnection, type BlobIDBConnectionOptions } from './db';
+
+/**
+ * @deprecated readonly
+ */
+export class IndexedDBV1BlobStorage extends BlobStorageBase {
+  static readonly identifier = 'IndexedDBV1BlobStorage';
+  override readonly isReadonly = true;
+
+  constructor(private readonly options: BlobIDBConnectionOptions) {
+    super();
+  }
+
+  readonly connection = share(new BlobIDBConnection(this.options));
+
+  get db() {
+    return this.connection.inner;
+  }
+
+  override async get(key: string) {
+    if (!this.db) {
+      return null;
+    }
+    const trx = this.db.transaction('blob', 'readonly');
+    const blob = await trx.store.get(key);
+    if (!blob) {
+      return null;
+    }
+
+    return {
+      key,
+      mime: '',
+      createdAt: new Date(),
+      data: new Uint8Array(blob),
+    };
+  }
+
+  override async delete(key: string, permanently: boolean) {
+    if (!this.db) {
+      return;
+    }
+    if (permanently) {
+      const trx = this.db.transaction('blob', 'readwrite');
+      await trx.store.delete(key);
+    }
+  }
+
+  override async list() {
+    if (!this.db) {
+      return [];
+    }
+    const trx = this.db.transaction('blob', 'readonly');
+    const it = trx.store.iterate();
+
+    const records: ListedBlobRecord[] = [];
+
+    for await (const { key, value } of it) {
+      records.push({
+        key,
+        mime: '',
+        size: value.byteLength,
+        createdAt: new Date(),
+      });
+    }
+
+    return records;
+  }
+
+  override async set() {
+    // no more writes
+  }
+
+  override async release() {
+    // no more writes
+  }
+}

packages/common/nbstore/src/impls/idb/v1/db.ts

@@ -0,0 +1,77 @@
+import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
+
+import { AutoReconnectConnection } from '../../../connection';
+
+export interface DocDBSchema extends DBSchema {
+  workspace: {
+    key: string;
+    value: {
+      id: string;
+      updates: {
+        timestamp: number;
+        update: Uint8Array;
+      }[];
+    };
+  };
+}
+
+export class DocIDBConnection extends AutoReconnectConnection<IDBPDatabase<DocDBSchema> | null> {
+  override get shareId() {
+    return 'idb(old):affine-local';
+  }
+
+  override async doConnect() {
+    const dbs = await indexedDB.databases();
+    if (dbs.some(d => d.name === 'affine-local')) {
+      return openDB<DocDBSchema>('affine-local', 1, {
+        upgrade: db => {
+          db.createObjectStore('workspace', { keyPath: 'id' });
+        },
+      });
+    } else {
+      return null;
+    }
+  }
+
+  override doDisconnect(conn: IDBPDatabase<DocDBSchema> | null) {
+    conn?.close();
+  }
+}
+
+export interface BlobDBSchema extends DBSchema {
+  blob: {
+    key: string;
+    value: ArrayBuffer;
+  };
+}
+
+export interface BlobIDBConnectionOptions {
+  id: string;
+}
+
+export class BlobIDBConnection extends AutoReconnectConnection<IDBPDatabase<BlobDBSchema> | null> {
+  constructor(private readonly options: BlobIDBConnectionOptions) {
+    super();
+  }
+
+  override get shareId() {
+    return `idb(old-blob):${this.options.id}`;
+  }
+
+  override async doConnect() {
+    const dbs = await indexedDB.databases();
+    if (dbs.some(d => d.name === `${this.options.id}_blob`)) {
+      return openDB<BlobDBSchema>(`${this.options.id}_blob`, 1, {
+        upgrade: db => {
+          db.createObjectStore('blob');
+        },
+      });
+    } else {
+      return null;
+    }
+  }
+
+  override doDisconnect(conn: IDBPDatabase<BlobDBSchema> | null) {
+    conn?.close();
+  }
+}

packages/common/nbstore/src/impls/idb/v1/doc.ts

@@ -0,0 +1,201 @@
+import { once } from 'lodash-es';
+import {
+  applyUpdate,
+  type Array as YArray,
+  Doc as YDoc,
+  type Map as YMap,
+} from 'yjs';
+
+import { share } from '../../../connection';
+import {
+  type DocClocks,
+  type DocRecord,
+  DocStorageBase,
+  type DocStorageOptions,
+  type DocUpdate,
+} from '../../../storage';
+import { getIdConverter } from '../../../utils/id-converter';
+import { DocIDBConnection } from './db';
+
+/**
+ * We use a fixed timestamp in v1 because the v1 should never be changed.
+ * This date is chosen because it is large enough to overwrite some previous error data.
+ * In our sync storage, only a larger timestamp can overwrite smaller one.
+ */
+const CONST_TIMESTAMP = new Date(1893456000000);
+
+/**
+ * @deprecated readonly
+ */
+export class IndexedDBV1DocStorage extends DocStorageBase {
+  static readonly identifier = 'IndexedDBV1DocStorage';
+
+  readonly connection = share(new DocIDBConnection());
+
+  constructor(opts: DocStorageOptions) {
+    super({
+      ...opts,
+      readonlyMode: true,
+    });
+  }
+
+  get db() {
+    return this.connection.inner;
+  }
+
+  get name() {
+    return 'idb(old)';
+  }
+
+  override async getDoc(docId: string) {
+    if (!this.db) {
+      return null;
+    }
+    const oldId = (await this.getIdConverter()).newIdToOldId(docId);
+    const rawDoc = await this.rawGetDoc(oldId);
+    return rawDoc
+      ? {
+          docId: rawDoc.docId,
+          bin: rawDoc.bin,
+          timestamp: CONST_TIMESTAMP,
+        }
+      : null;
+  }
+
+  protected override async getDocSnapshot() {
+    return null;
+  }
+
+  override async pushDocUpdate(update: DocUpdate) {
+    // no more writes to old db
+    return { docId: update.docId, timestamp: CONST_TIMESTAMP };
+  }
+
+  override async deleteDoc(docId: string) {
+    if (!this.db) {
+      return;
+    }
+    const oldId = (await this.getIdConverter()).newIdToOldId(docId);
+    const trx = this.db.transaction('workspace', 'readwrite');
+    await trx.store.delete(oldId);
+  }
+
+  override async getDocTimestamps(): Promise<DocClocks> {
+    if (!this.db) {
+      return {};
+    }
+
+    const idConverter = await this.getIdConverter();
+
+    const oldIds: string[] = [this.spaceId];
+
+    const rootDocBuffer = await this.rawGetDoc(this.spaceId);
+    if (rootDocBuffer) {
+      const ydoc = new YDoc({
+        guid: this.spaceId,
+      });
+      applyUpdate(ydoc, rootDocBuffer.bin);
+
+      // get all ids from rootDoc.meta.pages.[*].id, trust this id as normalized id
+      const normalizedDocIds = (
+        (ydoc.getMap('meta') as YMap<any> | undefined)?.get('pages') as
+          | YArray<YMap<any>>
+          | undefined
+      )
+        ?.map(i => i.get('id') as string)
+        .filter(i => !!i);
+
+      const spaces = ydoc.getMap('spaces') as YMap<any> | undefined;
+      for (const pageId of normalizedDocIds ?? []) {
+        const subdoc = spaces?.get(pageId);
+        if (subdoc && subdoc instanceof YDoc) {
+          oldIds.push(subdoc.guid);
+        }
+      }
+    }
+
+    const trx = this.db.transaction('workspace', 'readonly');
+    const allKeys = await trx.store.getAllKeys();
+    oldIds.push(...allKeys.filter(k => k.startsWith(`db$${this.spaceId}$`)));
+    oldIds.push(
+      ...allKeys.filter(k =>
+        k.match(new RegExp(`^userdata\\$[\\w-]+\\$${this.spaceId}$`))
+      )
+    );
+
+    return Object.fromEntries(
+      oldIds.map(id => [idConverter.oldIdToNewId(id), CONST_TIMESTAMP])
+    );
+  }
+
+  override async getDocTimestamp(_docId: string) {
+    return null;
+  }
+
+  protected override async setDocSnapshot(): Promise<boolean> {
+    return false;
+  }
+
+  protected override async getDocUpdates(): Promise<DocRecord[]> {
+    return [];
+  }
+
+  protected override async markUpdatesMerged(): Promise<number> {
+    return 0;
+  }
+
+  private async rawGetDoc(id: string) {
+    if (!this.db) {
+      return null;
+    }
+    const trx = this.db.transaction('workspace', 'readonly');
+    const record = await trx.store.get(id);
+
+    if (!record?.updates.length) {
+      return null;
+    }
+
+    if (record.updates.length === 1) {
+      return {
+        docId: id,
+        bin: record.updates[0].update,
+        timestamp: new Date(record.updates[0].timestamp),
+      };
+    }
+
+    return {
+      docId: id,
+      bin: await this.mergeUpdates(record.updates.map(update => update.update)),
+      timestamp: new Date(record.updates.at(-1)?.timestamp ?? Date.now()),
+    };
+  }
+
+  private readonly getIdConverter = once(async () => {
+    const idConverter = getIdConverter(
+      {
+        getDocBuffer: async id => {
+          if (!this.db) {
+            return null;
+          }
+          const trx = this.db.transaction('workspace', 'readonly');
+          const record = await trx.store.get(id);
+
+          if (!record?.updates.length) {
+            return null;
+          }
+
+          if (record.updates.length === 1) {
+            return record.updates[0].update;
+          }
+
+          return await this.mergeUpdates(
+            record.updates.map(update => update.update)
+          );
+        },
+      },
+      this.spaceId
+    );
+
+    return await idConverter;
+  });
+}

packages/common/nbstore/src/impls/idb/v1/index.ts

@@ -0,0 +1,11 @@
+import type { StorageConstructor } from '../..';
+import { IndexedDBV1BlobStorage } from './blob';
+import { IndexedDBV1DocStorage } from './doc';
+
+export * from './blob';
+export * from './doc';
+
+export const idbV1Storages = [
+  IndexedDBV1DocStorage,
+  IndexedDBV1BlobStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/index.ts

@@ -0,0 +1,27 @@
+import type { Storage } from '../storage';
+import type { broadcastChannelStorages } from './broadcast-channel';
+import type { cloudStorages } from './cloud';
+import type { idbStorages } from './idb';
+import type { idbV1Storages } from './idb/v1';
+import type { sqliteStorages } from './sqlite';
+import type { sqliteV1Storages } from './sqlite/v1';
+
+export type StorageConstructor = {
+  new (...args: any[]): Storage;
+  readonly identifier: string;
+};
+
+type Storages =
+  | typeof cloudStorages
+  | typeof idbV1Storages
+  | typeof idbStorages
+  | typeof sqliteStorages
+  | typeof sqliteV1Storages
+  | typeof broadcastChannelStorages;
+
+// oxlint-disable-next-line no-redeclare
+export type AvailableStorageImplementations = {
+  [key in Storages[number]['identifier']]: Storages[number] & {
+    identifier: key;
+  };
+};

packages/common/nbstore/src/impls/sqlite/blob-sync.ts

@@ -0,0 +1,32 @@
+import { share } from '../../connection';
+import { BlobSyncStorageBase } from '../../storage';
+import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
+
+export class SqliteBlobSyncStorage extends BlobSyncStorageBase {
+  static readonly identifier = 'SqliteBlobSyncStorage';
+
+  override connection = share(new NativeDBConnection(this.options));
+
+  constructor(private readonly options: SqliteNativeDBOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection.apis;
+  }
+
+  override async setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void> {
+    await this.db.setBlobUploadedAt(peer, blobId, uploadedAt);
+  }
+
+  override async getBlobUploadedAt(
+    peer: string,
+    blobId: string
+  ): Promise<Date | null> {
+    return this.db.getBlobUploadedAt(peer, blobId);
+  }
+}

packages/common/nbstore/src/impls/sqlite/blob.ts

@@ -0,0 +1,38 @@
+import { share } from '../../connection';
+import { type BlobRecord, BlobStorageBase } from '../../storage';
+import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
+
+export class SqliteBlobStorage extends BlobStorageBase {
+  static readonly identifier = 'SqliteBlobStorage';
+  override readonly isReadonly = false;
+
+  override connection = share(new NativeDBConnection(this.options));
+
+  constructor(private readonly options: SqliteNativeDBOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection.apis;
+  }
+
+  override async get(key: string) {
+    return this.db.getBlob(key);
+  }
+
+  override async set(blob: BlobRecord) {
+    await this.db.setBlob(blob);
+  }
+
+  override async delete(key: string, permanently: boolean) {
+    await this.db.deleteBlob(key, permanently);
+  }
+
+  override async release() {
+    await this.db.releaseBlobs();
+  }
+
+  override async list() {
+    return this.db.listBlobs();
+  }
+}

packages/common/nbstore/src/impls/sqlite/db.ts

@@ -0,0 +1,244 @@
+import { AutoReconnectConnection } from '../../connection';
+import type {
+  BlobRecord,
+  CrawlResult,
+  DocClock,
+  DocIndexedClock,
+  DocRecord,
+  ListedBlobRecord,
+} from '../../storage';
+import { type SpaceType, universalId } from '../../utils/universal-id';
+
+export interface SqliteNativeDBOptions {
+  readonly flavour: string;
+  readonly type: SpaceType;
+  readonly id: string;
+}
+
+export interface NativeIndexField {
+  field: string;
+  values: string[];
+}
+
+export interface NativeIndexQuery {
+  kind: 'match' | 'exists' | 'all' | 'boolean' | 'boost';
+  field?: string;
+  value?: string;
+  occur?: 'must' | 'should' | 'must_not';
+  clauses?: NativeIndexQuery[];
+  boost?: number;
+}
+
+export interface NativeIndexSearchOptions {
+  limit: number;
+  offset: number;
+  fields: string[];
+  highlights: string[];
+}
+
+export interface NativeIndexHit {
+  id: string;
+  score: number;
+  fields: NativeIndexField[];
+  highlights: {
+    field: string;
+    values: { valueIndex: number; spans: { start: number; end: number }[] }[];
+  }[];
+}
+
+export interface NativeIndexSearchResult {
+  total: number;
+  hits: NativeIndexHit[];
+}
+
+export interface NativeDBApis {
+  connect: (id: string) => Promise<void>;
+  disconnect: (id: string) => Promise<void>;
+  pushUpdate: (id: string, docId: string, update: Uint8Array) => Promise<Date>;
+  getDocSnapshot: (id: string, docId: string) => Promise<DocRecord | null>;
+  setDocSnapshot: (id: string, snapshot: DocRecord) => Promise<boolean>;
+  getDocUpdates: (id: string, docId: string) => Promise<DocRecord[]>;
+  markUpdatesMerged: (
+    id: string,
+    docId: string,
+    updates: Date[]
+  ) => Promise<number>;
+  deleteDoc: (id: string, docId: string) => Promise<void>;
+  getDocClocks: (id: string, after?: Date | null) => Promise<DocClock[]>;
+  getDocClock: (id: string, docId: string) => Promise<DocClock | null>;
+  getDocIndexedClock: (
+    id: string,
+    docId: string
+  ) => Promise<DocIndexedClock | null>;
+  setDocIndexedClock: (
+    id: string,
+    docId: string,
+    indexedClock: Date,
+    indexerVersion: number
+  ) => Promise<void>;
+  setDocIndexedClocks: (id: string, clocks: DocIndexedClock[]) => Promise<void>;
+  clearDocIndexedClock: (id: string, docId: string) => Promise<void>;
+  getBlob: (id: string, key: string) => Promise<BlobRecord | null>;
+  setBlob: (id: string, blob: BlobRecord) => Promise<void>;
+  deleteBlob: (id: string, key: string, permanently: boolean) => Promise<void>;
+  releaseBlobs: (id: string) => Promise<void>;
+  listBlobs: (id: string) => Promise<ListedBlobRecord[]>;
+  getPeerRemoteClocks: (id: string, peer: string) => Promise<DocClock[]>;
+  getPeerRemoteClock: (
+    id: string,
+    peer: string,
+    docId: string
+  ) => Promise<DocClock | null>;
+  setPeerRemoteClock: (
+    id: string,
+    peer: string,
+    docId: string,
+    clock: Date
+  ) => Promise<void>;
+  getPeerPulledRemoteClocks: (id: string, peer: string) => Promise<DocClock[]>;
+  getPeerPulledRemoteClock: (
+    id: string,
+    peer: string,
+    docId: string
+  ) => Promise<DocClock | null>;
+  setPeerPulledRemoteClock: (
+    id: string,
+    peer: string,
+    docId: string,
+    clock: Date
+  ) => Promise<void>;
+  getPeerPushedClocks: (id: string, peer: string) => Promise<DocClock[]>;
+  getPeerPushedClock: (
+    id: string,
+    peer: string,
+    docId: string
+  ) => Promise<DocClock | null>;
+  setPeerPushedClock: (
+    id: string,
+    peer: string,
+    docId: string,
+    clock: Date
+  ) => Promise<void>;
+  clearClocks: (id: string) => Promise<void>;
+  setBlobUploadedAt: (
+    id: string,
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ) => Promise<void>;
+  getBlobUploadedAt: (
+    id: string,
+    peer: string,
+    blobId: string
+  ) => Promise<Date | null>;
+  crawlDocData: (id: string, docId: string) => Promise<CrawlResult>;
+  indexUpsert: (
+    id: string,
+    table: string,
+    document: { id: string; fields: NativeIndexField[] }
+  ) => Promise<void>;
+  indexDelete: (id: string, table: string, docId: string) => Promise<void>;
+  indexSearch: (
+    id: string,
+    table: string,
+    query: NativeIndexQuery,
+    options: NativeIndexSearchOptions
+  ) => Promise<NativeIndexSearchResult>;
+  indexAggregate: (
+    id: string,
+    table: string,
+    query: NativeIndexQuery,
+    field: string,
+    limit: number,
+    offset: number,
+    hits?: NativeIndexSearchOptions
+  ) => Promise<{
+    total: number;
+    buckets: {
+      key: string;
+      count: number;
+      score: number;
+      hits: NativeIndexHit[];
+    }[];
+  }>;
+  indexDeleteByQuery: (
+    id: string,
+    table: string,
+    query: NativeIndexQuery
+  ) => Promise<number>;
+  indexFlush: (id: string) => Promise<void>;
+  indexVersion: () => Promise<number>;
+}
+
+type NativeDBApisWrapper = NativeDBApis extends infer APIs
+  ? {
+      [K in keyof APIs]: APIs[K] extends (...args: any[]) => any
+        ? Parameters<APIs[K]> extends [string, ...infer Rest]
+          ? (...args: Rest) => ReturnType<APIs[K]>
+          : (...args: Parameters<APIs[K]>) => ReturnType<APIs[K]>
+        : never;
+    }
+  : never;
+
+let apis: NativeDBApis | null = null;
+
+export function bindNativeDBApis(a: NativeDBApis) {
+  apis = a;
+}
+
+export class NativeDBConnection extends AutoReconnectConnection<void> {
+  readonly apis: NativeDBApisWrapper;
+
+  readonly flavour = this.options.flavour;
+  readonly type = this.options.type;
+  readonly id = this.options.id;
+
+  constructor(private readonly options: SqliteNativeDBOptions) {
+    super();
+
+    if (!apis) {
+      throw new Error('Not in native context.');
+    }
+
+    this.apis = this.warpApis(apis);
+  }
+
+  override get shareId(): string {
+    return `sqlite:${this.flavour}:${this.type}:${this.id}`;
+  }
+
+  warpApis(originalApis: NativeDBApis): NativeDBApisWrapper {
+    const id = universalId({
+      peer: this.flavour,
+      type: this.type,
+      id: this.id,
+    });
+    return new Proxy(
+      {},
+      {
+        get: (_target, key: keyof NativeDBApisWrapper) => {
+          const v = originalApis[key];
+
+          return async (...args: any[]) => {
+            return v.call(
+              originalApis,
+              id,
+              // @ts-expect-error I don't know why it complains ts(2556)
+              ...args
+            );
+          };
+        },
+      }
+    ) as unknown as NativeDBApisWrapper;
+  }
+
+  override async doConnect() {
+    await this.apis.connect();
+  }
+
+  override doDisconnect() {
+    this.apis.disconnect().catch(err => {
+      console.error('NativeDBConnection close failed', err);
+    });
+  }
+}

packages/common/nbstore/src/impls/sqlite/doc-sync.ts

@@ -0,0 +1,69 @@
+import { share } from '../../connection';
+import { type DocClock, DocSyncStorageBase } from '../../storage';
+import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
+
+export class SqliteDocSyncStorage extends DocSyncStorageBase {
+  static readonly identifier = 'SqliteDocSyncStorage';
+
+  override connection = share(new NativeDBConnection(this.options));
+
+  constructor(private readonly options: SqliteNativeDBOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection.apis;
+  }
+
+  override async getPeerRemoteClocks(peer: string) {
+    return this.db
+      .getPeerRemoteClocks(peer)
+      .then(clocks =>
+        Object.fromEntries(clocks.map(clock => [clock.docId, clock.timestamp]))
+      );
+  }
+
+  override async getPeerRemoteClock(peer: string, docId: string) {
+    return this.db.getPeerRemoteClock(peer, docId);
+  }
+
+  override async setPeerRemoteClock(peer: string, clock: DocClock) {
+    await this.db.setPeerRemoteClock(peer, clock.docId, clock.timestamp);
+  }
+
+  override async getPeerPulledRemoteClocks(peer: string) {
+    return this.db
+      .getPeerPulledRemoteClocks(peer)
+      .then(clocks =>
+        Object.fromEntries(clocks.map(clock => [clock.docId, clock.timestamp]))
+      );
+  }
+
+  override async getPeerPulledRemoteClock(peer: string, docId: string) {
+    return this.db.getPeerPulledRemoteClock(peer, docId);
+  }
+
+  override async setPeerPulledRemoteClock(peer: string, clock: DocClock) {
+    await this.db.setPeerPulledRemoteClock(peer, clock.docId, clock.timestamp);
+  }
+
+  override async getPeerPushedClocks(peer: string) {
+    return this.db
+      .getPeerPushedClocks(peer)
+      .then(clocks =>
+        Object.fromEntries(clocks.map(clock => [clock.docId, clock.timestamp]))
+      );
+  }
+
+  override async getPeerPushedClock(peer: string, docId: string) {
+    return this.db.getPeerPushedClock(peer, docId);
+  }
+
+  override async setPeerPushedClock(peer: string, clock: DocClock) {
+    await this.db.setPeerPushedClock(peer, clock.docId, clock.timestamp);
+  }
+
+  override async clearClocks() {
+    await this.db.clearClocks();
+  }
+}

packages/common/nbstore/src/impls/sqlite/doc.ts

@@ -0,0 +1,207 @@
+import { share } from '../../connection';
+import {
+  type BlockInfo,
+  type CrawlResult,
+  type DocClocks,
+  type DocRecord,
+  DocStorageBase,
+  type DocUpdate,
+} from '../../storage';
+import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
+
+export class SqliteDocStorage extends DocStorageBase<SqliteNativeDBOptions> {
+  static readonly identifier = 'SqliteDocStorage';
+  override connection = share(new NativeDBConnection(this.options));
+
+  get db() {
+    return this.connection.apis;
+  }
+
+  override async pushDocUpdate(update: DocUpdate, origin?: string) {
+    const timestamp = await this.db.pushUpdate(update.docId, update.bin);
+
+    this.emit(
+      'update',
+      {
+        docId: update.docId,
+        bin: update.bin,
+        timestamp,
+        editor: update.editor,
+      },
+      origin
+    );
+
+    return { docId: update.docId, timestamp };
+  }
+
+  override async deleteDoc(docId: string) {
+    await this.db.deleteDoc(docId);
+  }
+
+  override async getDocTimestamps(after?: Date) {
+    const clocks = await this.db.getDocClocks(after);
+
+    return clocks.reduce((ret, cur) => {
+      ret[cur.docId] = cur.timestamp;
+      return ret;
+    }, {} as DocClocks);
+  }
+
+  override async getDocTimestamp(docId: string) {
+    return this.db.getDocClock(docId);
+  }
+
+  protected override async getDocSnapshot(docId: string) {
+    const snapshot = await this.db.getDocSnapshot(docId);
+
+    if (!snapshot) {
+      return null;
+    }
+
+    return snapshot;
+  }
+
+  protected override async setDocSnapshot(
+    snapshot: DocRecord
+  ): Promise<boolean> {
+    return this.db.setDocSnapshot({
+      docId: snapshot.docId,
+      bin: snapshot.bin,
+      timestamp: snapshot.timestamp,
+    });
+  }
+
+  protected override async getDocUpdates(docId: string) {
+    return this.db.getDocUpdates(docId);
+  }
+
+  protected override markUpdatesMerged(docId: string, updates: DocRecord[]) {
+    return this.db.markUpdatesMerged(
+      docId,
+      updates.map(update => update.timestamp)
+    );
+  }
+
+  override async crawlDocData(docId: string): Promise<CrawlResult | null> {
+    const result = await this.db.crawlDocData(docId);
+    return normalizeNativeCrawlResult(result);
+  }
+}
+
+function normalizeNativeCrawlResult(result: unknown): CrawlResult | null {
+  if (!isRecord(result)) {
+    console.warn('[nbstore] crawlDocData returned non-object result');
+    return null;
+  }
+
+  if (
+    typeof result.title !== 'string' ||
+    typeof result.summary !== 'string' ||
+    !Array.isArray(result.blocks)
+  ) {
+    console.warn('[nbstore] crawlDocData result missing basic fields');
+    return null;
+  }
+
+  const { title, summary } = result as { title: string; summary: string };
+  const rawBlocks = result.blocks as unknown[];
+
+  const blocks: BlockInfo[] = [];
+  for (const block of rawBlocks) {
+    const normalized = normalizeBlock(block);
+    if (normalized) {
+      blocks.push(normalized);
+    }
+  }
+
+  if (blocks.length === 0) {
+    console.warn('[nbstore] crawlDocData has no valid blocks');
+    return null;
+  }
+
+  return {
+    blocks,
+    title,
+    summary,
+  };
+}
+
+function normalizeBlock(block: unknown): BlockInfo | null {
+  if (!isRecord(block)) {
+    return null;
+  }
+
+  const blockId = readStringField(block, 'blockId');
+  const flavour = readStringField(block, 'flavour');
+
+  if (!blockId || !flavour) {
+    return null;
+  }
+
+  return {
+    blockId,
+    flavour,
+    content: readStringArrayField(block, 'content'),
+    blob: readStringArrayField(block, 'blob'),
+    refDocId: readStringArrayField(block, 'refDocId'),
+    refInfo: readStringArrayField(block, 'refInfo'),
+    parentFlavour: readStringField(block, 'parentFlavour'),
+    parentBlockId: readStringField(block, 'parentBlockId'),
+    additional: safeAdditionalField(block),
+  };
+}
+
+function readStringField(
+  target: Record<string, unknown>,
+  key: string
+): string | undefined {
+  const value = readField(target, key);
+  return typeof value === 'string' && value ? value : undefined;
+}
+
+function readStringArrayField(
+  target: Record<string, unknown>,
+  key: string
+): string[] | undefined {
+  const value = readField(target, key);
+  if (Array.isArray(value)) {
+    const filtered = value.filter(
+      (item): item is string => typeof item === 'string' && item.length > 0
+    );
+    return filtered.length ? filtered : undefined;
+  }
+  if (typeof value === 'string' && value.length > 0) {
+    return [value];
+  }
+  return undefined;
+}
+
+function safeAdditionalField(
+  target: Record<string, unknown>
+): string | undefined {
+  const value = readField(target, 'additional');
+  if (typeof value !== 'string' || value.length === 0) {
+    return undefined;
+  }
+  try {
+    const parsed = JSON.parse(value);
+    return JSON.stringify(parsed);
+  } catch {
+    console.warn(
+      '[nbstore] ignore invalid additional payload in crawlDocData block'
+    );
+    return undefined;
+  }
+}
+
+function readField(target: Record<string, unknown>, key: string) {
+  return target[key] ?? target[toSnakeCase(key)];
+}
+
+function toSnakeCase(key: string) {
+  return key.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}

packages/common/nbstore/src/impls/sqlite/index.ts

@@ -0,0 +1,32 @@
+import type { StorageConstructor } from '..';
+import { SqliteBlobStorage } from './blob';
+import { SqliteBlobSyncStorage } from './blob-sync';
+import { SqliteDocStorage } from './doc';
+import { SqliteDocSyncStorage } from './doc-sync';
+import { SqliteIndexerStorage } from './indexer';
+import { SqliteIndexerSyncStorage } from './indexer-sync';
+
+export * from './blob';
+export * from './blob-sync';
+export {
+  bindNativeDBApis,
+  type NativeDBApis,
+  type NativeIndexField,
+  type NativeIndexHit,
+  type NativeIndexQuery,
+  type NativeIndexSearchOptions,
+  type NativeIndexSearchResult,
+} from './db';
+export * from './doc';
+export * from './doc-sync';
+export * from './indexer';
+export * from './indexer-sync';
+
+export const sqliteStorages = [
+  SqliteDocStorage,
+  SqliteBlobStorage,
+  SqliteDocSyncStorage,
+  SqliteBlobSyncStorage,
+  SqliteIndexerStorage,
+  SqliteIndexerSyncStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/sqlite/indexer-sync.ts

@@ -0,0 +1,43 @@
+import { share } from '../../connection';
+import {
+  type DocIndexedClock,
+  IndexerSyncStorageBase,
+} from '../../storage/indexer-sync';
+import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
+
+export class SqliteIndexerSyncStorage extends IndexerSyncStorageBase {
+  static readonly identifier = 'SqliteIndexerSyncStorage';
+  override readonly commitsIndexAtomically = true;
+
+  override connection = share(new NativeDBConnection(this.options));
+
+  constructor(private readonly options: SqliteNativeDBOptions) {
+    super();
+  }
+
+  private get db() {
+    return this.connection.apis;
+  }
+
+  override async getDocIndexedClock(
+    docId: string
+  ): Promise<DocIndexedClock | null> {
+    return this.db.getDocIndexedClock(docId);
+  }
+
+  override async setDocIndexedClock(clock: DocIndexedClock): Promise<void> {
+    await this.db.setDocIndexedClock(
+      clock.docId,
+      clock.timestamp,
+      clock.indexerVersion
+    );
+  }
+
+  override async setDocIndexedClocks(clocks: DocIndexedClock[]): Promise<void> {
+    await this.db.setDocIndexedClocks(clocks);
+  }
+
+  override async clearDocIndexedClock(docId: string): Promise<void> {
+    await this.db.clearDocIndexedClock(docId);
+  }
+}

packages/common/nbstore/src/impls/sqlite/indexer/index.ts

@@ -0,0 +1,255 @@
+import type { Observable } from 'rxjs';
+import { merge, of, Subject } from 'rxjs';
+import { filter, throttleTime } from 'rxjs/operators';
+
+import { share } from '../../../connection';
+import type {
+  AggregateOptions,
+  AggregateResult,
+  IndexerDocument,
+  IndexerSchema,
+  Query,
+  SearchOptions,
+  SearchResult,
+} from '../../../storage';
+import { IndexerStorageBase } from '../../../storage';
+import { fromPromise } from '../../../utils/from-promise';
+import { backoffRetry, exhaustMapWithTrailing } from '../../idb/indexer/utils';
+import {
+  NativeDBConnection,
+  type NativeIndexQuery,
+  type NativeIndexSearchOptions,
+  type SqliteNativeDBOptions,
+} from '../db';
+import { createNode } from './node-builder';
+
+const SQLITE_INDEXER_VERSION_OFFSET = 1;
+const NATIVE_INDEXER_MAX_LIMIT = 0xffffffff;
+
+export class SqliteIndexerStorage extends IndexerStorageBase {
+  static readonly identifier = 'SqliteIndexerStorage';
+  override readonly recommendRefreshInterval = 30 * 1000; // 5 seconds
+  readonly connection: NativeDBConnection;
+  readonly isReadonly = false;
+  private readonly tableUpdate$ = new Subject<string>();
+
+  constructor(options: SqliteNativeDBOptions) {
+    super();
+    this.connection = share(new NativeDBConnection(options));
+  }
+
+  private watchTableUpdated(table: string) {
+    return this.tableUpdate$.asObservable().pipe(filter(t => t === table));
+  }
+
+  async search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Promise<SearchResult<T, O>> {
+    const limit = options?.pagination?.limit ?? 10;
+    const skip = options?.pagination?.skip ?? 0;
+    const result = await this.connection.apis.indexSearch(
+      String(table),
+      toNativeQuery(query),
+      toNativeOptions(options, limit, skip)
+    );
+    const nodes = result.hits.map(hit => createNode(hit, options ?? {}));
+
+    return {
+      pagination: {
+        count: result.total,
+        limit,
+        skip,
+        hasMore: result.total > skip + limit,
+      },
+      nodes,
+    };
+  }
+
+  async aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>> {
+    const limit = options?.pagination?.limit ?? 10;
+    const skip = options?.pagination?.skip ?? 0;
+    const hitLimit = options?.hits?.pagination?.limit ?? 3;
+    const hitSkip = options?.hits?.pagination?.skip ?? 0;
+    const result = await this.connection.apis.indexAggregate(
+      String(table),
+      toNativeQuery(query),
+      String(field),
+      toNativeLimit(limit),
+      skip,
+      options?.hits
+        ? toNativeOptions(options.hits, hitLimit, hitSkip)
+        : undefined
+    );
+    const hitsOptions = options?.hits;
+    const buckets = result.buckets.map(bucket => ({
+      key: bucket.key,
+      count: bucket.count,
+      score: bucket.score,
+      ...(hitsOptions
+        ? {
+            hits: {
+              pagination: {
+                count: bucket.count,
+                limit: hitLimit,
+                skip: hitSkip,
+                hasMore: bucket.count > hitSkip + hitLimit,
+              },
+              nodes: bucket.hits.map(hit => createNode(hit, hitsOptions)),
+            },
+          }
+        : {}),
+    }));
+
+    return {
+      pagination: {
+        count: result.total,
+        limit,
+        skip,
+        hasMore: result.total > skip + limit,
+      },
+      buckets,
+    } as AggregateResult<T, O>;
+  }
+
+  search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Observable<SearchResult<T, O>> {
+    return merge(of(1), this.watchTableUpdated(table)).pipe(
+      throttleTime(3000, undefined, { leading: true, trailing: true }),
+      exhaustMapWithTrailing(() => {
+        return fromPromise(async () => {
+          return await this.search(table, query, options);
+        }).pipe(backoffRetry());
+      })
+    );
+  }
+
+  aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return merge(of(1), this.watchTableUpdated(table)).pipe(
+      throttleTime(3000, undefined, { leading: true, trailing: true }),
+      exhaustMapWithTrailing(() => {
+        return fromPromise(async () => {
+          return await this.aggregate(table, query, field, options);
+        }).pipe(backoffRetry());
+      })
+    );
+  }
+
+  async deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void> {
+    await this.connection.apis.indexDeleteByQuery(
+      String(table),
+      toNativeQuery(query)
+    );
+  }
+
+  async insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void> {
+    await this.connection.apis.indexUpsert(String(table), {
+      id: document.id,
+      fields: [...document.fields].map(([field, values]) => ({
+        field: String(field),
+        values,
+      })),
+    });
+    this.tableUpdate$.next(table);
+  }
+
+  async delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void> {
+    await this.connection.apis.indexDelete(String(table), id);
+    this.tableUpdate$.next(table);
+  }
+
+  async update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void> {
+    // Update is essentially insert (overwrite)
+    return this.insert(table, document);
+  }
+
+  async refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
+    // No-op for memory index
+  }
+
+  async refreshIfNeed(): Promise<void> {
+    await this.connection.apis.indexFlush();
+  }
+
+  async indexVersion(): Promise<number> {
+    return (
+      (await this.connection.apis.indexVersion()) +
+      SQLITE_INDEXER_VERSION_OFFSET
+    );
+  }
+}
+
+function toNativeQuery(query: Query<any>): NativeIndexQuery {
+  switch (query.type) {
+    case 'match':
+      return { kind: 'match', field: String(query.field), value: query.match };
+    case 'exists':
+      return { kind: 'exists', field: String(query.field) };
+    case 'all':
+      return { kind: 'all' };
+    case 'boolean':
+      return {
+        kind: 'boolean',
+        occur: query.occur,
+        clauses: query.queries.map(toNativeQuery),
+      };
+    case 'boost':
+      return {
+        kind: 'boost',
+        boost: query.boost,
+        clauses: [toNativeQuery(query.query)],
+      };
+  }
+}
+
+function toNativeOptions(
+  options: SearchOptions<any> | undefined,
+  limit: number,
+  offset: number
+): NativeIndexSearchOptions {
+  const highlights = options?.highlights?.map(item => String(item.field)) ?? [];
+  return {
+    limit: toNativeLimit(limit),
+    offset,
+    fields: [
+      ...new Set([...(options?.fields?.map(String) ?? []), ...highlights]),
+    ],
+    highlights,
+  };
+}
+
+function toNativeLimit(limit: number) {
+  return limit === Infinity ? NATIVE_INDEXER_MAX_LIMIT : limit;
+}

packages/common/nbstore/src/impls/sqlite/indexer/node-builder.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest';
+
+import type { NativeIndexHit } from '../db';
+import { createNode } from './node-builder';
+
+function hit(fields: NativeIndexHit['fields']): NativeIndexHit {
+  return { id: 'doc-id', score: 1, fields, highlights: [] };
+}
+
+describe('sqlite indexer native result mapping', () => {
+  it.each([
+    ['string', ['summary'], 'summary'],
+    ['array', ['one', 'two'], ['one', 'two']],
+    ['missing', [], ''],
+  ])('maps %s stored values', (_, values, expected) => {
+    const node = createNode(hit([{ field: 'title', values }]), {
+      fields: ['title'],
+    });
+    expect(node.fields.title).toEqual(expected);
+  });
+
+  it('formats native highlight spans', () => {
+    const native = hit([{ field: 'title', values: ['hello search'] }]);
+    native.highlights = [
+      {
+        field: 'title',
+        values: [{ valueIndex: 0, spans: [{ start: 6, end: 12 }] }],
+      },
+    ];
+    const node = createNode(native, {
+      highlights: [{ field: 'title', before: '<b>', end: '</b>' }],
+    });
+    expect(node.highlights.title).toEqual(['hello <b>search</b>']);
+  });
+});

packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts

@@ -0,0 +1,44 @@
+import { type SearchOptions } from '../../../storage';
+import { highlighter } from '../../idb/indexer/highlighter';
+import type { NativeIndexHit } from '../db';
+
+export function createNode(hit: NativeIndexHit, options: SearchOptions<any>) {
+  const node: any = { id: hit.id, score: hit.score };
+  const fields = new Map(hit.fields.map(field => [field.field, field.values]));
+
+  if (options.fields) {
+    node.fields = Object.fromEntries(
+      options.fields.map(field => {
+        const values = fields.get(String(field)) ?? [];
+        return [String(field), values.length > 1 ? values : (values[0] ?? '')];
+      })
+    );
+  }
+
+  if (options.highlights) {
+    const highlights = new Map(
+      hit.highlights.map(item => [item.field, item.values])
+    );
+    node.highlights = Object.fromEntries(
+      options.highlights.map(option => {
+        const field = String(option.field);
+        const source = fields.get(field) ?? [];
+        const fragments = (highlights.get(field) ?? []).flatMap(value => {
+          const text = source[value.valueIndex];
+          if (!text) return [];
+          const fragment = highlighter(
+            text,
+            option.before,
+            option.end,
+            value.spans.map(span => [span.start, span.end]),
+            { maxPrefix: 20, maxLength: 50 }
+          );
+          return fragment ? [fragment] : [];
+        });
+        return [field, fragments];
+      })
+    );
+  }
+
+  return node;
+}

packages/common/nbstore/src/impls/sqlite/v1/blob.ts

@@ -0,0 +1,65 @@
+import { DummyConnection } from '../../../connection';
+import { BlobStorageBase } from '../../../storage';
+import type { SpaceType } from '../../../utils/universal-id';
+import { apis } from './db';
+
+/**
+ * @deprecated readonly
+ */
+export class SqliteV1BlobStorage extends BlobStorageBase {
+  static identifier = 'SqliteV1BlobStorage';
+  override connection = new DummyConnection();
+  override readonly isReadonly = true;
+
+  constructor(private readonly options: { type: SpaceType; id: string }) {
+    super();
+  }
+
+  private get db() {
+    if (!apis) {
+      throw new Error('Not in electron context.');
+    }
+
+    return apis;
+  }
+
+  override async get(key: string) {
+    const data: Uint8Array | null = await this.db.getBlob(
+      this.options.type,
+      this.options.id,
+      key
+    );
+
+    if (!data) {
+      return null;
+    }
+
+    return {
+      key,
+      data,
+      mime: '',
+      createdAt: new Date(),
+    };
+  }
+
+  override async list() {
+    const keys = await this.db.getBlobKeys(this.options.type, this.options.id);
+
+    return keys.map(key => ({
+      key,
+      mime: '',
+      size: 0,
+      createdAt: new Date(),
+    }));
+  }
+  override async delete() {
+    // no more deletes
+  }
+
+  override async set() {
+    // no more writes
+  }
+  override async release() {
+    // no more writes
+  }
+}

packages/common/nbstore/src/impls/sqlite/v1/db.ts

@@ -0,0 +1,25 @@
+import type { SpaceType } from '../../../utils/universal-id';
+
+interface NativeDBV1Apis {
+  getBlob: (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string
+  ) => Promise<Buffer | null>;
+  getBlobKeys: (spaceType: SpaceType, workspaceId: string) => Promise<string[]>;
+  getDocAsUpdates: (
+    spaceType: SpaceType,
+    workspaceId: string,
+    subdocId: string
+  ) => Promise<Uint8Array>;
+  getDocTimestamps: (
+    spaceType: SpaceType,
+    workspaceId: string
+  ) => Promise<{ docId?: string; timestamp: Date }[]>;
+}
+
+export let apis: NativeDBV1Apis | null = null;
+
+export function bindNativeDBV1Apis(a: NativeDBV1Apis) {
+  apis = a;
+}

packages/common/nbstore/src/impls/sqlite/v1/doc.ts

@@ -0,0 +1,144 @@
+import { DummyConnection } from '../../../connection';
+import {
+  type DocRecord,
+  DocStorageBase,
+  type DocUpdate,
+} from '../../../storage';
+import { getIdConverter, type IdConverter } from '../../../utils/id-converter';
+import { isEmptyUpdate } from '../../../utils/is-empty-update';
+import type { SpaceType } from '../../../utils/universal-id';
+import { apis } from './db';
+
+/**
+ * We use a fixed timestamp in v1 because the v1 should never be changed.
+ * This date is chosen because it is large enough to overwrite some previous error data.
+ * In our sync storage, only a larger timestamp can overwrite smaller one.
+ */
+const CONST_TIMESTAMP = new Date(1893456000000);
+
+/**
+ * @deprecated readonly
+ */
+export class SqliteV1DocStorage extends DocStorageBase<{
+  type: SpaceType;
+  id: string;
+}> {
+  static identifier = 'SqliteV1DocStorage';
+  cachedIdConverter: Promise<IdConverter> | null = null;
+  override connection = new DummyConnection();
+
+  constructor(options: { type: SpaceType; id: string }) {
+    super({ ...options, readonlyMode: true });
+  }
+
+  private get db() {
+    if (!apis) {
+      throw new Error('Not in electron context.');
+    }
+
+    return apis;
+  }
+
+  override async pushDocUpdate(update: DocUpdate) {
+    // no more writes
+    return { docId: update.docId, timestamp: CONST_TIMESTAMP };
+  }
+
+  override async getDoc(docId: string) {
+    const idConverter = await this.getIdConverter();
+    const bin = await this.db.getDocAsUpdates(
+      this.options.type,
+      this.options.id,
+      idConverter.newIdToOldId(docId)
+    );
+
+    if (isEmptyUpdate(bin)) {
+      return null;
+    }
+
+    return {
+      docId,
+      bin,
+      timestamp: CONST_TIMESTAMP,
+    };
+  }
+
+  override async getDocTimestamps() {
+    const timestamps = await this.db.getDocTimestamps(
+      this.options.type,
+      this.options.id
+    );
+
+    if (!timestamps) {
+      return {};
+    }
+
+    const idConverter = await this.getIdConverter();
+
+    return timestamps.reduce(
+      (ret, { docId }) => {
+        ret[idConverter.oldIdToNewId(docId ?? this.options.id)] =
+          CONST_TIMESTAMP;
+        return ret;
+      },
+      {} as Record<string, Date>
+    );
+  }
+
+  override async deleteDoc() {
+    return;
+  }
+
+  protected override async getDocSnapshot() {
+    return null;
+  }
+
+  override async getDocTimestamp() {
+    return null;
+  }
+
+  protected override async setDocSnapshot(): Promise<boolean> {
+    return false;
+  }
+
+  protected override async getDocUpdates(): Promise<DocRecord[]> {
+    return [];
+  }
+
+  protected override async markUpdatesMerged(): Promise<number> {
+    return 0;
+  }
+
+  private async getIdConverter() {
+    if (this.cachedIdConverter) {
+      return await this.cachedIdConverter;
+    }
+    this.cachedIdConverter = getIdConverter(
+      {
+        getDocBuffer: async id => {
+          if (!this.db) {
+            return null;
+          }
+          const updates = await this.db.getDocAsUpdates(
+            this.options.type,
+            this.options.id,
+            id
+          );
+
+          if (isEmptyUpdate(updates)) {
+            return null;
+          }
+
+          if (!updates) {
+            return null;
+          }
+
+          return updates;
+        },
+      },
+      this.spaceId
+    );
+
+    return await this.cachedIdConverter;
+  }
+}

packages/common/nbstore/src/impls/sqlite/v1/index.ts

@@ -0,0 +1,12 @@
+import type { StorageConstructor } from '../..';
+import { SqliteV1BlobStorage } from './blob';
+import { SqliteV1DocStorage } from './doc';
+
+export * from './blob';
+export { bindNativeDBV1Apis } from './db';
+export * from './doc';
+
+export const sqliteV1Storages = [
+  SqliteV1DocStorage,
+  SqliteV1BlobStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/index.ts

@@ -0,0 +1,5 @@
+export * from './connection';
+export type * from './frontend'; // // Only export types. For implementation, please import from '@affine/nbstore/frontend'
+export * from './storage';
+export type * from './sync'; // Only export types. For implementation, please import from '@affine/nbstore/sync'
+export * from './utils/universal-id';

packages/common/nbstore/src/realtime/__tests__/manager.spec.ts

@@ -0,0 +1,424 @@
+import { getRealtimeInputKey, type RealtimeEvent } from '@affine/realtime';
+import { beforeEach, expect, test, vi } from 'vitest';
+
+import { RealtimeManager } from '../manager';
+
+type Handler = (payload?: unknown) => void;
+
+class FakeSocket {
+  readonly handlers = new Map<string, Handler>();
+  readonly emitted: Array<{ event: string; payload: unknown }> = [];
+  connected = true;
+  disconnected = false;
+  nextRequestAck: unknown = { data: { count: 1 } };
+  subscribeAcks: unknown[] = [];
+  nextSubscriptionId = 0;
+
+  on(event: string, handler: Handler) {
+    this.handlers.set(event, handler);
+  }
+
+  off(event: string) {
+    this.handlers.delete(event);
+  }
+
+  async emitWithAck(event: string, payload: unknown) {
+    this.emitted.push({ event, payload });
+    if (event === 'realtime:subscribe') {
+      const ack = this.subscribeAcks.shift();
+      if (ack) return ack;
+      this.nextSubscriptionId += 1;
+      return { data: { subscriptionId: `sub-${this.nextSubscriptionId}` } };
+    }
+    if (event === 'realtime:request') {
+      return this.nextRequestAck;
+    }
+    return { data: {} };
+  }
+
+  emit(event: string, payload?: unknown) {
+    this.handlers.get(event)?.(payload);
+  }
+}
+
+const { resetSharedConnection, waitForConnected } = vi.hoisted(() => ({
+  resetSharedConnection: vi.fn(),
+  waitForConnected: vi.fn(async () => {}),
+}));
+const socket = new FakeSocket();
+
+vi.mock('../../impls/cloud/socket', () => ({
+  SocketConnection: class {
+    static resetSharedConnection = resetSharedConnection;
+
+    readonly inner = { socket };
+    status = 'connected';
+    readonly maybeConnection = { socket };
+
+    connect() {}
+
+    waitForConnected = waitForConnected;
+
+    disconnect() {
+      socket.disconnected = true;
+    }
+  },
+}));
+
+beforeEach(() => {
+  vi.stubGlobal('BUILD_CONFIG', { appVersion: 'test' });
+  socket.handlers.clear();
+  socket.emitted.length = 0;
+  socket.nextRequestAck = { data: { count: 1 } };
+  socket.subscribeAcks = [];
+  socket.nextSubscriptionId = 0;
+  socket.connected = true;
+  socket.disconnected = false;
+  resetSharedConnection.mockClear();
+  waitForConnected.mockReset();
+  waitForConnected.mockResolvedValue(undefined);
+});
+
+test('getRealtimeInputKey is deterministic for realtime subscription inputs', () => {
+  expect(getRealtimeInputKey({ workspaceId: 'space', docId: 'doc' })).toBe(
+    getRealtimeInputKey({ docId: 'doc', workspaceId: 'space' })
+  );
+});
+
+test('getRealtimeInputKey follows JSON semantics for edge values', () => {
+  expect(getRealtimeInputKey({ a: undefined })).toBe(getRealtimeInputKey({}));
+  expect(getRealtimeInputKey([undefined])).toBe('[null]');
+  expect(getRealtimeInputKey(new Date('2026-01-02T03:04:05.000Z'))).toBe(
+    '"2026-01-02T03:04:05.000Z"'
+  );
+});
+
+test('request sends generic realtime request with client version', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+
+  await expect(manager.request('notification.count.get', {})).resolves.toEqual({
+    count: 1,
+  });
+
+  expect(socket.emitted).toEqual([
+    {
+      event: 'realtime:request',
+      payload: {
+        op: 'notification.count.get',
+        input: {},
+        clientVersion: 'test',
+      },
+    },
+  ]);
+});
+
+test('request rejects server ack error', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  socket.nextRequestAck = {
+    error: { name: 'Forbidden', message: 'No access' },
+  };
+
+  await expect(manager.request('notification.count.get', {})).rejects.toThrow(
+    'No access'
+  );
+});
+
+test('user profile request can bootstrap without authenticated context', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: false,
+  });
+  socket.nextRequestAck = { data: { user: null } };
+
+  await expect(manager.request('user.profile.get', {})).resolves.toEqual({
+    user: null,
+  });
+});
+
+test('non-bootstrap request still requires authenticated context', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: false,
+  });
+
+  await expect(manager.request('notification.count.get', {})).rejects.toThrow(
+    'Realtime is not authenticated'
+  );
+});
+
+test('request rejects when connection times out or is aborted', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const controller = new AbortController();
+  socket.nextRequestAck = new Promise(() => {});
+  const request = manager.request(
+    'notification.count.get',
+    {},
+    { signal: controller.signal }
+  );
+
+  controller.abort();
+
+  await expect(request).rejects.toThrow('Realtime request aborted');
+
+  waitForConnected.mockImplementationOnce(() => new Promise(() => {}));
+  await expect(
+    manager.request('notification.count.get', {}, { timeoutMs: 1 })
+  ).rejects.toMatchObject({
+    name: 'RealtimeRequestTimeout',
+    message: 'Realtime request timed out: notification.count.get',
+  });
+});
+
+test('subscribe routes events by topic and stable input key', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const received: unknown[] = [];
+  const subscription = manager
+    .subscribe('comment.changed', { workspaceId: 'space', docId: 'doc' })
+    .subscribe(event => received.push(event));
+  await vi.waitFor(() => expect(received).toEqual([{ type: 'ready' }]));
+
+  socket.emit('realtime:event', {
+    topic: 'comment.changed',
+    inputKey: getRealtimeInputKey({ workspaceId: 'space', docId: 'other' }),
+    sentAt: 1,
+    event: { changed: true },
+  } satisfies RealtimeEvent);
+  socket.emit('realtime:event', {
+    topic: 'comment.changed',
+    inputKey: getRealtimeInputKey({ workspaceId: 'space', docId: 'doc' }),
+    sentAt: 2,
+    event: { changed: true },
+  } satisfies RealtimeEvent);
+
+  expect(received).toEqual([{ type: 'ready' }, { changed: true }]);
+  subscription.unsubscribe();
+});
+
+test('unsubscribe leaves server room and clears status', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const subscription = manager
+    .subscribe('notification.count.changed', {})
+    .subscribe();
+  await vi.waitFor(() => expect(manager.getStatus().subscriptions).toBe(1));
+
+  subscription.unsubscribe();
+
+  expect(manager.getStatus()).toMatchObject({
+    connected: true,
+    subscriptions: 0,
+  });
+  expect(socket.emitted.at(-1)).toEqual({
+    event: 'realtime:unsubscribe',
+    payload: {
+      subscriptionId: 'sub-1',
+      topic: 'notification.count.changed',
+      input: {},
+      clientVersion: 'test',
+    },
+  });
+});
+
+test('context switch disconnects socket and keeps subscriptions for reauth', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const completed = vi.fn();
+  manager
+    .subscribe('notification.count.changed', {})
+    .subscribe({ complete: completed });
+  await vi.waitFor(() => expect(manager.getStatus().subscriptions).toBe(1));
+
+  manager.setContext({
+    endpoint: 'http://other-server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+
+  expect(socket.disconnected).toBe(true);
+  expect(completed).not.toHaveBeenCalled();
+  expect(manager.getStatus()).toMatchObject({
+    endpoint: 'http://other-server',
+    connected: false,
+    subscriptions: 1,
+  });
+});
+
+test('auth context switch resets shared socket connection', () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: false,
+  });
+
+  expect(resetSharedConnection).toHaveBeenCalledWith('http://server', false);
+});
+
+test('context switch resubscribes existing subscriptions on next connect', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const received: unknown[] = [];
+  const subscription = manager
+    .subscribe('notification.count.changed', {})
+    .subscribe(event => received.push(event));
+  await vi.waitFor(() => expect(received).toEqual([{ type: 'ready' }]));
+
+  manager.setContext({
+    endpoint: 'http://other-server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  await manager.request('notification.count.get', {});
+
+  await vi.waitFor(() =>
+    expect(
+      socket.emitted.filter(item => item.event === 'realtime:subscribe')
+    ).toHaveLength(2)
+  );
+  expect(received).toEqual([{ type: 'ready' }, { type: 'ready' }]);
+  subscription.unsubscribe();
+});
+
+test('unsubscribe uses resubscribed server subscription id', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const subscription = manager
+    .subscribe('notification.count.changed', {})
+    .subscribe();
+  await vi.waitFor(() => expect(manager.getStatus().subscriptions).toBe(1));
+
+  manager.setContext({
+    endpoint: 'http://other-server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  await manager.request('notification.count.get', {});
+  await vi.waitFor(() =>
+    expect(
+      socket.emitted.filter(item => item.event === 'realtime:subscribe')
+    ).toHaveLength(2)
+  );
+
+  subscription.unsubscribe();
+
+  expect(manager.getStatus().subscriptions).toBe(0);
+  expect(socket.emitted.at(-1)).toEqual({
+    event: 'realtime:unsubscribe',
+    payload: {
+      subscriptionId: 'sub-2',
+      topic: 'notification.count.changed',
+      input: {},
+      clientVersion: 'test',
+    },
+  });
+});
+
+test('subscribe registers server room again after reconnect', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const received: unknown[] = [];
+  const subscription = manager
+    .subscribe('notification.count.changed', {})
+    .subscribe(event => received.push(event));
+  await vi.waitFor(() => expect(received).toEqual([{ type: 'ready' }]));
+
+  socket.emit('connect');
+
+  await vi.waitFor(() =>
+    expect(
+      socket.emitted.filter(item => item.event === 'realtime:subscribe')
+    ).toHaveLength(2)
+  );
+  expect(received).toEqual([{ type: 'ready' }, { type: 'ready' }]);
+  subscription.unsubscribe();
+});
+
+test('failed reconnect only errors the affected subscription', async () => {
+  const manager = new RealtimeManager();
+  manager.setContext({
+    endpoint: 'http://server',
+    isSelfHosted: false,
+    authenticated: true,
+  });
+  const first: unknown[] = [];
+  const firstErrors: unknown[] = [];
+  const second: unknown[] = [];
+  const secondErrors: unknown[] = [];
+  const firstSubscription = manager
+    .subscribe('notification.count.changed', {})
+    .subscribe({
+      next: event => first.push(event),
+      error: error => firstErrors.push(error),
+    });
+  const secondSubscription = manager
+    .subscribe('comment.changed', { workspaceId: 'space', docId: 'doc' })
+    .subscribe({
+      next: event => second.push(event),
+      error: error => secondErrors.push(error),
+    });
+  await vi.waitFor(() => expect(manager.getStatus().subscriptions).toBe(2));
+
+  socket.subscribeAcks = [
+    { data: { subscriptionId: 'resub-1' } },
+    { error: { name: 'Forbidden', message: 'No access' } },
+  ];
+  socket.emit('connect');
+
+  await vi.waitFor(() => expect(first).toHaveLength(2));
+  await vi.waitFor(() => expect(secondErrors).toHaveLength(1));
+  expect(firstErrors).toEqual([]);
+  expect(manager.getStatus().subscriptions).toBe(1);
+
+  firstSubscription.unsubscribe();
+  secondSubscription.unsubscribe();
+});

packages/common/nbstore/src/realtime/index.ts

@@ -0,0 +1 @@
+export { RealtimeManager } from './manager';

packages/common/nbstore/src/realtime/manager.ts

@@ -0,0 +1,333 @@
+import type {
+  RealtimeConfigureInput,
+  RealtimeEvent,
+  RealtimeRequestInputOf,
+  RealtimeRequestName,
+  RealtimeRequestOutputOf,
+  RealtimeStatus,
+  RealtimeSubscriptionReady,
+  RealtimeTopicEventOf,
+  RealtimeTopicInputOf,
+  RealtimeTopicName,
+} from '@affine/realtime';
+import { getRealtimeInputKey } from '@affine/realtime';
+import { Observable, Subject } from 'rxjs';
+
+import { SocketConnection } from '../impls/cloud/socket';
+
+const DEFAULT_REQUEST_TIMEOUT = 10_000;
+
+type RealtimeContext = RealtimeConfigureInput;
+
+function normalizeError(error: unknown) {
+  if (error instanceof Error) {
+    return { name: error.name, message: error.message };
+  }
+  return { name: 'RealtimeError', message: String(error) };
+}
+
+function rejectAck(error: { name?: string; message?: string; code?: string }) {
+  const err = new Error(error.message ?? 'Realtime request failed');
+  err.name = error.name ?? 'RealtimeError';
+  return err;
+}
+
+export class RealtimeManager {
+  private context?: RealtimeContext;
+  private socketConnection?: SocketConnection;
+  private socketKey?: string;
+  private lastError?: { name: string; message: string };
+  private subscriptionsNeedResubscribe = false;
+  private readonly subscriptions = new Map<
+    string,
+    {
+      topic: RealtimeTopicName;
+      input: RealtimeTopicInputOf<RealtimeTopicName>;
+      inputKey: string;
+      subject$: Subject<RealtimeEvent | RealtimeSubscriptionReady>;
+      onResubscribed: (subscriptionId: string) => void;
+    }
+  >();
+
+  setContext(context: RealtimeContext) {
+    const nextContext = { ...context };
+    const previousContext = this.context;
+    const changed =
+      !previousContext ||
+      previousContext.endpoint !== nextContext.endpoint ||
+      previousContext.isSelfHosted !== nextContext.isSelfHosted ||
+      previousContext.authenticated !== nextContext.authenticated;
+
+    this.context = nextContext;
+
+    if (changed) {
+      this.resetConnection();
+      if (
+        previousContext &&
+        previousContext.authenticated !== nextContext.authenticated
+      ) {
+        SocketConnection.resetSharedConnection(
+          previousContext.endpoint,
+          previousContext.isSelfHosted
+        );
+      }
+    }
+  }
+
+  async request<Op extends RealtimeRequestName>(
+    op: Op,
+    input: RealtimeRequestInputOf<Op>,
+    options?: { timeoutMs?: number; signal?: AbortSignal }
+  ): Promise<RealtimeRequestOutputOf<Op>> {
+    const timeoutMs = options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
+    const connectAbort = new AbortController();
+    let timeoutId: ReturnType<typeof setTimeout> | undefined;
+    let abortHandler: (() => void) | undefined;
+    const abort = () => {
+      const error = new Error(`Realtime request aborted: ${op}`);
+      error.name = 'AbortError';
+      return error;
+    };
+    if (options?.signal?.aborted) {
+      throw abort();
+    }
+    const timeout = new Promise<never>((_resolve, reject) => {
+      timeoutId = setTimeout(() => {
+        const error = new Error(`Realtime request timed out: ${op}`);
+        error.name = 'RealtimeRequestTimeout';
+        reject(error);
+      }, timeoutMs);
+      timeoutId.unref?.();
+    });
+    const aborted = new Promise<never>((_resolve, reject) => {
+      abortHandler = () => reject(abort());
+      options?.signal?.addEventListener('abort', abortHandler, { once: true });
+    });
+
+    try {
+      const socket = await Promise.race([
+        this.connect(op === 'user.profile.get', connectAbort.signal),
+        timeout,
+        aborted,
+      ]);
+      const ack = await Promise.race([
+        socket.emitWithAck('realtime:request', {
+          op,
+          input,
+          clientVersion: BUILD_CONFIG.appVersion,
+        }),
+        timeout,
+        aborted,
+      ]);
+
+      if ('error' in ack) {
+        throw rejectAck(ack.error);
+      }
+
+      return ack.data as unknown as RealtimeRequestOutputOf<Op>;
+    } finally {
+      connectAbort.abort();
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+      if (abortHandler) {
+        options?.signal?.removeEventListener('abort', abortHandler);
+      }
+    }
+  }
+
+  subscribe<Topic extends RealtimeTopicName>(
+    topic: Topic,
+    input: RealtimeTopicInputOf<Topic>
+  ): Observable<RealtimeTopicEventOf<Topic> | RealtimeSubscriptionReady> {
+    return new Observable(subscriber => {
+      let subscriptionId: string | undefined;
+      let subject$: Subject<RealtimeEvent | RealtimeSubscriptionReady>;
+      let closed = false;
+
+      const setup = async () => {
+        try {
+          const socket = await this.connect();
+          const ack = await socket.emitWithAck('realtime:subscribe', {
+            topic,
+            input,
+            clientVersion: BUILD_CONFIG.appVersion,
+          });
+          if ('error' in ack) {
+            throw rejectAck(ack.error);
+          }
+          const data = ack.data;
+          subscriptionId = data.subscriptionId;
+          if (closed) {
+            await socket.emitWithAck('realtime:unsubscribe', {
+              subscriptionId: data.subscriptionId,
+              topic,
+              input,
+              clientVersion: BUILD_CONFIG.appVersion,
+            });
+            return;
+          }
+
+          subject$ = new Subject();
+          this.subscriptions.set(subscriptionId, {
+            topic,
+            input,
+            inputKey: getRealtimeInputKey(input),
+            subject$,
+            onResubscribed: nextSubscriptionId => {
+              subscriptionId = nextSubscriptionId;
+            },
+          });
+          subscriber.next({
+            type: 'ready',
+          });
+          subject$.subscribe({
+            next: event => {
+              if ('type' in event) {
+                subscriber.next(event);
+              } else {
+                subscriber.next(event.event as RealtimeTopicEventOf<Topic>);
+              }
+            },
+            error: error => subscriber.error(error),
+            complete: () => {},
+          });
+        } catch (error) {
+          this.lastError = normalizeError(error);
+          subscriber.error(error);
+        }
+      };
+
+      setup().catch(error => subscriber.error(error));
+
+      return () => {
+        closed = true;
+        if (!subscriptionId) {
+          return;
+        }
+        const currentSubscriptionId = subscriptionId;
+        this.subscriptions.delete(currentSubscriptionId);
+        subject$?.complete();
+        if (this.socketConnection?.inner?.socket.connected) {
+          this.socketConnection.inner.socket
+            .emitWithAck('realtime:unsubscribe', {
+              subscriptionId: currentSubscriptionId,
+              topic,
+              input,
+              clientVersion: BUILD_CONFIG.appVersion,
+            })
+            .catch(() => {});
+        }
+      };
+    });
+  }
+
+  getStatus(): RealtimeStatus {
+    return {
+      endpoint: this.context?.endpoint,
+      connected: this.socketConnection?.status === 'connected',
+      connecting: this.socketConnection?.status === 'connecting',
+      subscriptions: this.subscriptions.size,
+      lastError: this.lastError,
+    };
+  }
+
+  private async connect(allowUnauthenticated = false, signal?: AbortSignal) {
+    if (
+      !this.context?.endpoint ||
+      (!this.context.authenticated && !allowUnauthenticated)
+    ) {
+      const error = new Error('Realtime is not authenticated');
+      error.name = 'RealtimeUnauthenticated';
+      throw error;
+    }
+
+    const key = `${this.context.endpoint}:${this.context.isSelfHosted}`;
+    if (!this.socketConnection || this.socketKey !== key) {
+      this.resetConnection();
+      this.socketKey = key;
+      this.socketConnection = new SocketConnection(
+        this.context.endpoint,
+        this.context.isSelfHosted
+      );
+      this.socketConnection.connect();
+    }
+
+    await this.socketConnection.waitForConnected(signal);
+    this.socketConnection.inner.socket.off('realtime:event', this.handleEvent);
+    this.socketConnection.inner.socket.on('realtime:event', this.handleEvent);
+    this.socketConnection.inner.socket.off('connect', this.handleReconnect);
+    this.socketConnection.inner.socket.on('connect', this.handleReconnect);
+    if (this.subscriptionsNeedResubscribe && this.context.authenticated) {
+      await this.resubscribeAll();
+    }
+    return this.socketConnection.inner.socket;
+  }
+
+  private readonly handleEvent = (event: RealtimeEvent) => {
+    for (const subscription of this.subscriptions.values()) {
+      if (
+        subscription.topic === event.topic &&
+        subscription.inputKey === event.inputKey
+      ) {
+        subscription.subject$.next(event);
+      }
+    }
+  };
+
+  private readonly handleReconnect = () => {
+    this.resubscribeAll().catch(error => {
+      this.lastError = normalizeError(error);
+    });
+  };
+
+  private async resubscribeAll() {
+    const socket = this.socketConnection?.inner.socket;
+    if (!socket?.connected || this.subscriptions.size === 0) {
+      return;
+    }
+
+    const subscriptions = Array.from(this.subscriptions.entries());
+    for (const [subscriptionId, subscription] of subscriptions) {
+      try {
+        const ack = await socket.emitWithAck('realtime:subscribe', {
+          topic: subscription.topic,
+          input: subscription.input,
+          clientVersion: BUILD_CONFIG.appVersion,
+        });
+        if ('error' in ack) {
+          throw rejectAck(ack.error);
+        }
+
+        this.subscriptions.delete(subscriptionId);
+        this.subscriptions.set(ack.data.subscriptionId, subscription);
+        subscription.onResubscribed(ack.data.subscriptionId);
+        subscription.subject$.next({
+          type: 'ready',
+        });
+      } catch (error) {
+        this.lastError = normalizeError(error);
+        this.subscriptions.delete(subscriptionId);
+        subscription.subject$.error(error);
+      }
+    }
+    this.subscriptionsNeedResubscribe = false;
+  }
+
+  private resetConnection() {
+    if (this.socketConnection) {
+      this.socketConnection.maybeConnection?.socket.off(
+        'realtime:event',
+        this.handleEvent
+      );
+      this.socketConnection.maybeConnection?.socket.off(
+        'connect',
+        this.handleReconnect
+      );
+      this.socketConnection.disconnect(true);
+    }
+    this.socketConnection = undefined;
+    this.socketKey = undefined;
+    this.subscriptionsNeedResubscribe = this.subscriptions.size > 0;
+  }
+}

packages/common/nbstore/src/storage/awareness.ts

@@ -0,0 +1,36 @@
+import type { Connection } from '../connection';
+import { type Storage } from './storage';
+
+export type AwarenessRecord = {
+  docId: string;
+  bin: Uint8Array;
+};
+
+export interface AwarenessStorage extends Storage {
+  readonly storageType: 'awareness';
+
+  /**
+   * Update the awareness record.
+   *
+   * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
+   */
+  update(record: AwarenessRecord, origin?: string): Promise<void>;
+  subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void;
+}
+
+export abstract class AwarenessStorageBase implements AwarenessStorage {
+  readonly storageType = 'awareness';
+  abstract readonly connection: Connection;
+
+  abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
+
+  abstract subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void;
+}

packages/common/nbstore/src/storage/blob-sync.ts

@@ -0,0 +1,30 @@
+import type { Connection } from '../connection';
+import type { Storage } from './storage';
+
+export interface BlobSyncStorage extends Storage {
+  readonly storageType: 'blobSync';
+
+  setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void>;
+
+  getBlobUploadedAt(peer: string, blobId: string): Promise<Date | null>;
+}
+
+export abstract class BlobSyncStorageBase implements BlobSyncStorage {
+  readonly storageType = 'blobSync';
+  abstract readonly connection: Connection;
+
+  abstract setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void>;
+
+  abstract getBlobUploadedAt(
+    peer: string,
+    blobId: string
+  ): Promise<Date | null>;
+}

packages/common/nbstore/src/storage/blob.ts

@@ -0,0 +1,78 @@
+import type { Connection } from '../connection';
+import { type Storage } from './storage';
+
+export interface BlobRecord {
+  key: string;
+  data: Uint8Array;
+  mime: string;
+  createdAt?: Date;
+}
+
+export interface ListedBlobRecord {
+  key: string;
+  mime: string;
+  size: number;
+  createdAt?: Date;
+}
+
+export type BlobSource =
+  | { type: 'currentDoc'; workspaceId: string; docId: string }
+  | {
+      type: 'history';
+      workspaceId: string;
+      docId: string;
+      timestampMs: number;
+    };
+
+export interface SourceBlobRecord extends ListedBlobRecord {
+  source: BlobSource;
+}
+
+export interface SourceScopedBlobStorage {
+  registerSource(source: BlobSource, signal?: AbortSignal): Promise<void>;
+  unregisterSource(source: BlobSource): Promise<void>;
+  readableSources(signal?: AbortSignal): AsyncIterable<SourceBlobRecord>;
+}
+
+export function isSourceScopedBlobStorage(
+  storage: BlobStorage
+): storage is BlobStorage & SourceScopedBlobStorage {
+  return 'registerSource' in storage && 'readableSources' in storage;
+}
+
+export interface BlobStorage extends Storage {
+  readonly storageType: 'blob';
+  readonly isReadonly: boolean;
+  get(
+    key: string,
+    signal?: AbortSignal,
+    source?: BlobSource
+  ): Promise<BlobRecord | null>;
+  set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
+  delete(
+    key: string,
+    permanently: boolean,
+    signal?: AbortSignal
+  ): Promise<void>;
+  release(signal?: AbortSignal): Promise<void>;
+  list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
+}
+
+export abstract class BlobStorageBase implements BlobStorage {
+  readonly storageType = 'blob';
+  abstract readonly connection: Connection;
+  abstract readonly isReadonly: boolean;
+  abstract get(
+    key: string,
+    signal?: AbortSignal,
+    source?: BlobSource
+  ): Promise<BlobRecord | null>;
+  abstract set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
+  abstract delete(
+    key: string,
+    permanently: boolean,
+    signal?: AbortSignal
+  ): Promise<void>;
+  abstract release(signal?: AbortSignal): Promise<void>;
+  abstract list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
+}

packages/common/nbstore/src/storage/doc-sync.ts

@@ -0,0 +1,49 @@
+import type { Connection } from '../connection';
+import type { DocClock, DocClocks } from './doc';
+import { type Storage } from './storage';
+
+export interface DocSyncStorage extends Storage {
+  readonly storageType: 'docSync';
+
+  getPeerRemoteClock(peer: string, docId: string): Promise<DocClock | null>;
+  getPeerRemoteClocks(peer: string): Promise<DocClocks>;
+  setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  getPeerPulledRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
+  setPeerPulledRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  getPeerPushedClock(peer: string, docId: string): Promise<DocClock | null>;
+  getPeerPushedClocks(peer: string): Promise<DocClocks>;
+  setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
+  clearClocks(): Promise<void>;
+}
+
+export abstract class DocSyncStorageBase implements DocSyncStorage {
+  readonly storageType = 'docSync';
+  abstract readonly connection: Connection;
+
+  abstract getPeerRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  abstract getPeerRemoteClocks(peer: string): Promise<DocClocks>;
+  abstract setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  abstract getPeerPulledRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  abstract getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
+  abstract setPeerPulledRemoteClock(
+    peer: string,
+    clock: DocClock
+  ): Promise<void>;
+  abstract getPeerPushedClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  abstract getPeerPushedClocks(peer: string): Promise<DocClocks>;
+  abstract setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
+  abstract clearClocks(): Promise<void>;
+}

packages/common/nbstore/src/storage/doc.ts

@@ -0,0 +1,319 @@
+import EventEmitter2 from 'eventemitter2';
+import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs';
+
+import type { Connection } from '../connection';
+import { isEmptyUpdate } from '../utils/is-empty-update';
+import type { Locker } from './lock';
+import { SingletonLocker } from './lock';
+import { type Storage } from './storage';
+
+export interface BlockInfo {
+  blockId: string;
+  flavour: string;
+  content?: string[];
+  blob?: string[];
+  refDocId?: string[];
+  refInfo?: string[];
+  parentFlavour?: string;
+  parentBlockId?: string;
+  additional?: string;
+}
+
+export interface CrawlResult {
+  blocks: BlockInfo[];
+  title: string;
+  summary: string;
+}
+
+export interface DocClock {
+  docId: string;
+  timestamp: Date;
+}
+
+export type DocClocks = Record<string, Date>;
+export interface DocRecord extends DocClock {
+  bin: Uint8Array;
+  editor?: string;
+}
+
+export interface DocDiff extends DocClock {
+  missing: Uint8Array;
+  state: Uint8Array;
+}
+
+export interface DocUpdate {
+  docId: string;
+  bin: Uint8Array;
+  editor?: string;
+}
+
+export type DocLifecycle = 'trash' | 'restore' | 'delete';
+
+export interface DocLifecycleResult {
+  rootUpdate: Uint8Array;
+  timestamp: Date;
+}
+
+export interface Editor {
+  name: string;
+  avatarUrl: string | null;
+}
+
+export interface DocStorageOptions {
+  mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
+  id: string;
+
+  /**
+   * open as readonly mode.
+   */
+  readonlyMode?: boolean;
+}
+
+export interface DocStorage extends Storage {
+  readonly storageType: 'doc';
+  readonly isReadonly: boolean;
+  readonly spaceId: string;
+  /**
+   * Get a doc record with latest binary.
+   */
+  getDoc(docId: string): Promise<DocRecord | null>;
+  /**
+   * Get a yjs binary diff with the given state vector.
+   */
+  getDocDiff(docId: string, state?: Uint8Array): Promise<DocDiff | null>;
+  /**
+   * Push updates into storage
+   *
+   * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
+   */
+  pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
+
+  /**
+   * Get the timestamp of the latest update of a doc.
+   */
+  getDocTimestamp(docId: string): Promise<DocClock | null>;
+
+  /**
+   * Get all docs timestamps info. especially for useful in sync process.
+   */
+  getDocTimestamps(after?: Date): Promise<DocClocks>;
+
+  /**
+   * Delete a specific doc data with all snapshots and updates
+   */
+  deleteDoc(docId: string): Promise<void>;
+
+  applyDocLifecycle?(
+    docId: string,
+    lifecycle: DocLifecycle
+  ): Promise<DocLifecycleResult>;
+
+  /**
+   * Subscribe on doc updates emitted from storage itself.
+   *
+   * NOTE:
+   *
+   *   There is not always update emitted from storage itself.
+   *
+   *   For example, in Sqlite storage, the update will only come from user's updating on docs,
+   *   in other words, the update will never somehow auto generated in storage internally.
+   *
+   *   But for Cloud storage, there will be updates broadcasted from other clients,
+   *   so the storage will emit updates to notify the client to integrate them.
+   */
+  subscribeDocUpdate(
+    callback: (update: DocRecord, origin?: string) => void
+  ): () => void;
+
+  crawlDocData?(docId: string): Promise<CrawlResult | null>;
+}
+
+export abstract class DocStorageBase<Opts = {}> implements DocStorage {
+  get isReadonly(): boolean {
+    return this.options.readonlyMode ?? false;
+  }
+  private readonly event = new EventEmitter2();
+  readonly storageType = 'doc';
+  abstract readonly connection: Connection;
+  protected readonly locker: Locker = new SingletonLocker();
+  readonly spaceId = this.options.id;
+
+  constructor(protected readonly options: Opts & DocStorageOptions) {}
+
+  async getDoc(docId: string) {
+    await using _lock = this.isReadonly
+      ? undefined
+      : await this.lockDocForUpdate(docId);
+
+    const snapshot = await this.getDocSnapshot(docId);
+    const updates = await this.getDocUpdates(docId);
+
+    if (updates.length) {
+      const { timestamp, bin, editor } = await this.squash(
+        snapshot ? [snapshot, ...updates] : updates
+      );
+
+      const newSnapshot = {
+        spaceId: this.spaceId,
+        docId,
+        bin,
+        timestamp,
+        editor,
+      };
+
+      // if is readonly, we will not set the new snapshot
+      if (!this.isReadonly) {
+        await this.setDocSnapshot(newSnapshot, snapshot);
+
+        // always mark updates as merged unless throws
+        await this.markUpdatesMerged(docId, updates);
+      }
+
+      return newSnapshot;
+    }
+
+    return snapshot;
+  }
+
+  async getDocDiff(docId: string, state?: Uint8Array) {
+    const doc = await this.getDoc(docId);
+
+    if (!doc) {
+      return null;
+    }
+
+    return {
+      docId,
+      missing: state && state.length > 0 ? diffUpdate(doc.bin, state) : doc.bin,
+      state: encodeStateVectorFromUpdate(doc.bin),
+      timestamp: doc.timestamp,
+    };
+  }
+
+  abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
+
+  abstract getDocTimestamp(docId: string): Promise<DocClock | null>;
+
+  abstract getDocTimestamps(after?: Date): Promise<DocClocks>;
+
+  abstract deleteDoc(docId: string): Promise<void>;
+
+  subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
+    this.event.on('update', callback);
+
+    return () => {
+      this.event.off('update', callback);
+    };
+  }
+
+  async crawlDocData(_docId: string): Promise<CrawlResult | null> {
+    return null;
+  }
+
+  // REGION: api for internal usage
+  protected on(
+    event: 'update',
+    callback: (update: DocRecord, origin: string) => void
+  ): () => void;
+  protected on(
+    event: 'snapshot',
+    callback: (snapshot: DocRecord, prevSnapshot: DocRecord | null) => void
+  ): () => void;
+  protected on(event: string, callback: (...args: any[]) => void): () => void {
+    this.event.on(event, callback);
+    return () => {
+      this.event.off(event, callback);
+    };
+  }
+
+  protected emit(event: 'update', update: DocRecord, origin?: string): void;
+  protected emit(
+    event: 'snapshot',
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): void;
+  protected emit(event: string, ...args: any[]): void {
+    this.event.emit(event, ...args);
+  }
+
+  protected off(event: string, callback: (...args: any[]) => void): void {
+    this.event.off(event, callback);
+  }
+
+  /**
+   * Get a doc snapshot from storage
+   */
+  protected abstract getDocSnapshot(docId: string): Promise<DocRecord | null>;
+  /**
+   * Set the doc snapshot into storage
+   *
+   * @safety
+   * be careful when implementing this method.
+   *
+   * It might be called with outdated snapshot when running in multi-thread environment.
+   *
+   * A common solution is update the snapshot record is DB only when the coming one's timestamp is newer.
+   *
+   * @example
+   * ```ts
+   * await using _lock = await this.lockDocForUpdate(docId);
+   * // set snapshot
+   *
+   * ```
+   */
+  protected abstract setDocSnapshot(
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): Promise<boolean>;
+
+  /**
+   * Get all updates of a doc that haven't been merged into snapshot.
+   *
+   * Updates queue design exists for a performace concern:
+   * A huge amount of write time will be saved if we don't merge updates into snapshot immediately.
+   * Updates will be merged into snapshot when the latest doc is requested.
+   */
+  protected abstract getDocUpdates(docId: string): Promise<DocRecord[]>;
+
+  /**
+   * Mark updates as merged into snapshot.
+   */
+  protected abstract markUpdatesMerged(
+    docId: string,
+    updates: DocRecord[]
+  ): Promise<number>;
+
+  /**
+   * Merge doc updates into a single update.
+   */
+  protected async squash(updates: DocRecord[]): Promise<DocRecord> {
+    const lastUpdate = updates.at(-1);
+    if (!lastUpdate) {
+      throw new Error('No updates to be squashed.');
+    }
+
+    // fast return
+    if (updates.length === 1) {
+      return lastUpdate;
+    }
+
+    const finalUpdate = await this.mergeUpdates(updates.map(u => u.bin));
+
+    return {
+      docId: lastUpdate.docId,
+      bin: finalUpdate,
+      timestamp: lastUpdate.timestamp,
+      editor: lastUpdate.editor,
+    };
+  }
+
+  protected mergeUpdates(updates: Uint8Array[]) {
+    const merge = this.options?.mergeUpdates ?? mergeUpdates;
+
+    return merge(updates.filter(bin => !isEmptyUpdate(bin)));
+  }
+
+  protected async lockDocForUpdate(docId: string): Promise<AsyncDisposable> {
+    return this.locker.lock(`workspace:${this.spaceId}:update`, docId);
+  }
+}

packages/common/nbstore/src/storage/dummy/awareness.ts

@@ -0,0 +1,16 @@
+import { DummyConnection } from '../../connection';
+import { type AwarenessRecord, AwarenessStorageBase } from '../awareness';
+
+export class DummyAwarenessStorage extends AwarenessStorageBase {
+  override update(_record: AwarenessRecord, _origin?: string): Promise<void> {
+    return Promise.resolve();
+  }
+  override subscribeUpdate(
+    _id: string,
+    _onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    _onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void {
+    return () => {};
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/blob-sync.ts

@@ -0,0 +1,14 @@
+import { type Connection, DummyConnection } from '../../connection';
+import type { BlobSyncStorage } from '../blob-sync';
+
+export class DummyBlobSyncStorage implements BlobSyncStorage {
+  storageType = 'blobSync' as const;
+  connection: Connection<any> = new DummyConnection();
+
+  setBlobUploadedAt(): Promise<void> {
+    return Promise.resolve();
+  }
+  getBlobUploadedAt(): Promise<Date | null> {
+    return Promise.resolve(new Date());
+  }
+}

packages/common/nbstore/src/storage/dummy/blob.ts

@@ -0,0 +1,33 @@
+import { DummyConnection } from '../../connection';
+import {
+  type BlobRecord,
+  BlobStorageBase,
+  type ListedBlobRecord,
+} from '../blob';
+
+export class DummyBlobStorage extends BlobStorageBase {
+  override readonly isReadonly = true;
+  override get(
+    _key: string,
+    _signal?: AbortSignal
+  ): Promise<BlobRecord | null> {
+    return Promise.resolve(null);
+  }
+  override set(_blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  override delete(
+    _key: string,
+    _permanently: boolean,
+    _signal?: AbortSignal
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override release(_signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  override list(_signal?: AbortSignal): Promise<ListedBlobRecord[]> {
+    return Promise.resolve([]);
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/doc-sync.ts

@@ -0,0 +1,49 @@
+import { DummyConnection } from '../../connection';
+import type { DocClock, DocClocks } from '../doc';
+import { DocSyncStorageBase } from '../doc-sync';
+
+export class DummyDocSyncStorage extends DocSyncStorageBase {
+  override getPeerRemoteClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerRemoteClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerRemoteClock(_peer: string, _clock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override getPeerPulledRemoteClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerPulledRemoteClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerPulledRemoteClock(
+    _peer: string,
+    _clock: DocClock
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override getPeerPushedClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerPushedClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerPushedClock(_peer: string, _clock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override clearClocks(): Promise<void> {
+    return Promise.resolve();
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/doc.ts

@@ -0,0 +1,42 @@
+import { DummyConnection } from '../../connection';
+import {
+  type DocClock,
+  type DocClocks,
+  type DocDiff,
+  type DocRecord,
+  type DocStorage,
+  type DocUpdate,
+} from '../doc';
+
+export class DummyDocStorage implements DocStorage {
+  spaceId = '';
+  readonly storageType = 'doc';
+  readonly isReadonly = true;
+  getDoc(_docId: string): Promise<DocRecord | null> {
+    return Promise.resolve(null);
+  }
+  getDocDiff(_docId: string, _state?: Uint8Array): Promise<DocDiff | null> {
+    return Promise.resolve(null);
+  }
+  pushDocUpdate(update: DocUpdate, _origin?: string): Promise<DocClock> {
+    return Promise.resolve({
+      docId: update.docId,
+      timestamp: new Date(),
+    });
+  }
+  getDocTimestamp(_docId: string): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  getDocTimestamps(_after?: Date): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  deleteDoc(_docId: string): Promise<void> {
+    return Promise.resolve();
+  }
+  subscribeDocUpdate(
+    _callback: (update: DocRecord, origin?: string) => void
+  ): () => void {
+    return () => {};
+  }
+  connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/indexer-sync.ts

@@ -0,0 +1,15 @@
+import { DummyConnection } from '../../connection';
+import { type DocIndexedClock, IndexerSyncStorageBase } from '../indexer-sync';
+
+export class DummyIndexerSyncStorage extends IndexerSyncStorageBase {
+  override connection = new DummyConnection();
+  override getDocIndexedClock(_docId: string): Promise<DocIndexedClock | null> {
+    return Promise.resolve(null);
+  }
+  override setDocIndexedClock(_docClock: DocIndexedClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override clearDocIndexedClock(_docId: string): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/common/nbstore/src/storage/dummy/indexer.ts

@@ -0,0 +1,94 @@
+import { NEVER, type Observable } from 'rxjs';
+
+import { DummyConnection } from '../../connection';
+import {
+  type AggregateOptions,
+  type AggregateResult,
+  type IndexerDocument,
+  type IndexerSchema,
+  IndexerStorageBase,
+  type Query,
+  type SearchOptions,
+  type SearchResult,
+} from '../indexer';
+
+export class DummyIndexerStorage extends IndexerStorageBase {
+  readonly isReadonly = true;
+  readonly connection = new DummyConnection();
+
+  override search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Promise<SearchResult<T, O>> {
+    return Promise.resolve({
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      nodes: [],
+    });
+  }
+  override aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Promise<AggregateResult<T, O>> {
+    return Promise.resolve({
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      buckets: [],
+    });
+  }
+  override search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Observable<SearchResult<T, O>> {
+    return NEVER;
+  }
+  override aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return NEVER;
+  }
+
+  override deleteByQuery<T extends keyof IndexerSchema>(
+    _table: T,
+    _query: Query<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override insert<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override delete<T extends keyof IndexerSchema>(
+    _table: T,
+    _id: string
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override update<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
+    return Promise.resolve();
+  }
+  override async refreshIfNeed(): Promise<void> {
+    return;
+  }
+  override async indexVersion(): Promise<number> {
+    return 0;
+  }
+}

packages/common/nbstore/src/storage/errors/index.ts

@@ -0,0 +1,2 @@
+export * from './over-capacity';
+export * from './over-size';

packages/common/nbstore/src/storage/errors/over-capacity.ts

@@ -0,0 +1,5 @@
+export class OverCapacityError extends Error {
+  constructor(public originError?: any) {
+    super('Storage over capacity.');
+  }
+}

packages/common/nbstore/src/storage/errors/over-size.ts

@@ -0,0 +1,10 @@
+export class OverSizeError extends Error {
+  constructor(limit: string | null, message?: string) {
+    if (message) {
+      super(message);
+    } else {
+      const formattedLimit = limit ? `${limit} ` : '';
+      super(`File size exceeds the ${formattedLimit}limit.`);
+    }
+  }
+}

packages/common/nbstore/src/storage/history.ts

@@ -0,0 +1,122 @@
+import { noop } from 'lodash-es';
+import {
+  applyUpdate,
+  Doc,
+  encodeStateAsUpdate,
+  encodeStateVector,
+  UndoManager,
+} from 'yjs';
+
+import { type DocRecord, DocStorageBase, type DocStorageOptions } from './doc';
+
+export interface HistoryFilter {
+  before?: Date;
+  limit?: Date;
+}
+
+export interface ListedHistory {
+  userId: string | null;
+  timestamp: Date;
+}
+
+export abstract class HistoricalDocStorage<
+  Options extends DocStorageOptions = DocStorageOptions,
+> extends DocStorageBase<Options> {
+  constructor(opts: Options) {
+    super(opts);
+
+    this.on('snapshot', snapshot => {
+      this.createHistory(snapshot.docId, snapshot).catch(noop);
+    });
+  }
+
+  override async setDocSnapshot(
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): Promise<boolean> {
+    const success = await this.upsertDocSnapshot(snapshot, prevSnapshot);
+    if (success) {
+      this.emit('snapshot', snapshot, prevSnapshot);
+    }
+    return success;
+  }
+
+  /**
+   * Update the doc snapshot in storage or create a new one if not exists.
+   *
+   * @safety
+   * be careful when implementing this method.
+   *
+   * It might be called with outdated snapshot when running in multi-thread environment.
+   *
+   * A common solution is update the snapshot record is DB only when the coming one's timestamp is newer.
+   *
+   * @example
+   * ```ts
+   * await using _lock = await this.lockDocForUpdate(docId);
+   * // set snapshot
+   *
+   * ```
+   */
+  abstract upsertDocSnapshot(
+    snapshot: DocRecord,
+    prevSnapshot: DocRecord | null
+  ): Promise<boolean>;
+
+  abstract listHistories(
+    docId: string,
+    filter?: HistoryFilter
+  ): Promise<ListedHistory[]>;
+  abstract getHistory(
+    docId: string,
+    timestamp: Date
+  ): Promise<DocRecord | null>;
+  abstract deleteHistory(docId: string, timestamp: Date): Promise<void>;
+
+  async rollbackDoc(docId: string, timestamp: Date, editor?: string) {
+    const toSnapshot = await this.getHistory(docId, timestamp);
+    if (!toSnapshot) {
+      throw new Error('Can not find the version to rollback to.');
+    }
+
+    const fromSnapshot = await this.getDoc(docId);
+
+    if (!fromSnapshot) {
+      throw new Error('Can not find the current version of the doc.');
+    }
+
+    const change = this.generateRevertUpdate(fromSnapshot.bin, toSnapshot.bin);
+    await this.pushDocUpdate({ docId, bin: change, editor }, 'rollback');
+    // force create a new history record after rollback
+    await this.createHistory(docId, fromSnapshot);
+  }
+
+  // history can only be created upon update pushing.
+  protected abstract createHistory(
+    docId: string,
+    snapshot: DocRecord
+  ): Promise<void>;
+
+  protected generateRevertUpdate(
+    fromNewerBin: Uint8Array,
+    toOlderBin: Uint8Array
+  ): Uint8Array {
+    const newerDoc = new Doc();
+    applyUpdate(newerDoc, fromNewerBin);
+    const olderDoc = new Doc();
+    applyUpdate(olderDoc, toOlderBin);
+
+    const newerState = encodeStateVector(newerDoc);
+    const olderState = encodeStateVector(olderDoc);
+
+    const diff = encodeStateAsUpdate(newerDoc, olderState);
+
+    const undoManager = new UndoManager(Array.from(olderDoc.share.values()));
+
+    applyUpdate(olderDoc, diff);
+
+    undoManager.undo();
+
+    return encodeStateAsUpdate(olderDoc, newerState);
+  }
+}

packages/common/nbstore/src/storage/index.ts

@@ -0,0 +1,96 @@
+import EventEmitter2 from 'eventemitter2';
+
+import type { AwarenessStorage } from './awareness';
+import type { BlobStorage } from './blob';
+import type { BlobSyncStorage } from './blob-sync';
+import type { DocStorage } from './doc';
+import type { DocSyncStorage } from './doc-sync';
+import { DummyAwarenessStorage } from './dummy/awareness';
+import { DummyBlobStorage } from './dummy/blob';
+import { DummyBlobSyncStorage } from './dummy/blob-sync';
+import { DummyDocStorage } from './dummy/doc';
+import { DummyDocSyncStorage } from './dummy/doc-sync';
+import { DummyIndexerStorage } from './dummy/indexer';
+import { DummyIndexerSyncStorage } from './dummy/indexer-sync';
+import type { IndexerStorage } from './indexer';
+import type { IndexerSyncStorage } from './indexer-sync';
+import type { StorageType } from './storage';
+
+type Storages =
+  | DocStorage
+  | BlobStorage
+  | BlobSyncStorage
+  | DocSyncStorage
+  | AwarenessStorage
+  | IndexerStorage
+  | IndexerSyncStorage;
+
+export type SpaceStorageOptions = {
+  [K in StorageType]?: Storages & { storageType: K };
+};
+
+export class SpaceStorage {
+  protected readonly storages: {
+    [K in StorageType]: Storages & { storageType: K };
+  };
+  private readonly event = new EventEmitter2();
+  private readonly disposables: Set<() => void> = new Set();
+
+  constructor(storages: SpaceStorageOptions) {
+    this.storages = {
+      awareness: storages.awareness ?? new DummyAwarenessStorage(),
+      blob: storages.blob ?? new DummyBlobStorage(),
+      blobSync: storages.blobSync ?? new DummyBlobSyncStorage(),
+      doc: storages.doc ?? new DummyDocStorage(),
+      docSync: storages.docSync ?? new DummyDocSyncStorage(),
+      indexer: storages.indexer ?? new DummyIndexerStorage(),
+      indexerSync: storages.indexerSync ?? new DummyIndexerSyncStorage(),
+    };
+  }
+
+  get<T extends StorageType>(type: T): Extract<Storages, { storageType: T }> {
+    const storage = this.storages[type];
+
+    if (!storage) {
+      throw new Error(`Storage ${type} not registered.`);
+    }
+
+    return storage as unknown as Extract<Storages, { storageType: T }>;
+  }
+
+  connect() {
+    Object.values(this.storages).forEach(storage => {
+      storage.connection.connect();
+    });
+  }
+
+  disconnect() {
+    Object.values(this.storages).forEach(storage => {
+      storage.connection.disconnect();
+    });
+  }
+
+  async waitForConnected(signal?: AbortSignal) {
+    await Promise.all(
+      Object.values(this.storages).map(storage =>
+        storage.connection.waitForConnected(signal)
+      )
+    );
+  }
+
+  async destroy() {
+    this.disposables.forEach(disposable => disposable());
+    this.event.removeAllListeners();
+  }
+}
+
+export * from './awareness';
+export * from './blob';
+export * from './blob-sync';
+export * from './doc';
+export * from './doc-sync';
+export * from './errors';
+export * from './history';
+export * from './indexer';
+export * from './indexer-sync';
+export * from './storage';

packages/common/nbstore/src/storage/indexer-sync.ts

@@ -0,0 +1,31 @@
+import type { Connection } from '../connection';
+import type { DocClock } from './doc';
+import type { Storage } from './storage';
+
+export interface DocIndexedClock extends DocClock {
+  indexerVersion: number;
+}
+
+export interface IndexerSyncStorage extends Storage {
+  readonly storageType: 'indexerSync';
+  readonly commitsIndexAtomically: boolean;
+
+  getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
+
+  setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
+  setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise<void>;
+
+  clearDocIndexedClock(docId: string): Promise<void>;
+}
+
+export abstract class IndexerSyncStorageBase implements IndexerSyncStorage {
+  readonly storageType = 'indexerSync';
+  readonly commitsIndexAtomically: boolean = false;
+  abstract connection: Connection<any>;
+  abstract getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
+  abstract setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
+  async setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise<void> {
+    for (const clock of docClocks) await this.setDocIndexedClock(clock);
+  }
+  abstract clearDocIndexedClock(docId: string): Promise<void>;
+}

packages/common/nbstore/src/storage/indexer.ts

@@ -0,0 +1,184 @@
+export * from './indexer/document';
+export * from './indexer/field-type';
+export * from './indexer/query';
+export * from './indexer/schema';
+
+import type { Observable } from 'rxjs';
+
+import type { Connection } from '../connection';
+import type { IndexerDocument } from './indexer/document';
+import type { Query } from './indexer/query';
+import type { IndexerSchema } from './indexer/schema';
+import type { Storage } from './storage';
+
+export interface IndexerStorage extends Storage {
+  readonly storageType: 'indexer';
+  readonly isReadonly: boolean;
+  readonly recommendRefreshInterval: number; // 100ms
+
+  search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Promise<SearchResult<T, O>>;
+
+  aggregate<T extends keyof IndexerSchema, const O extends AggregateOptions<T>>(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>>;
+
+  search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Observable<SearchResult<T, O>>;
+
+  aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>>;
+
+  deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void>;
+
+  insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  delete<T extends keyof IndexerSchema>(table: T, id: string): Promise<void>;
+
+  update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
+  refreshIfNeed(): Promise<void>;
+  indexVersion(): Promise<number>;
+}
+
+type ResultPagination = {
+  count: number;
+  limit: number;
+  skip: number;
+  hasMore: boolean;
+};
+
+type PaginationOption = { limit?: number; skip?: number };
+
+type HighlightAbleField<T extends keyof IndexerSchema> = {
+  [K in keyof IndexerSchema[T]]: IndexerSchema[T][K] extends 'FullText'
+    ? K
+    : never;
+}[keyof IndexerSchema[T]];
+
+export type SearchOptions<T extends keyof IndexerSchema> = {
+  pagination?: PaginationOption;
+  highlights?: { field: HighlightAbleField<T>; before: string; end: string }[];
+  fields?: (keyof IndexerSchema[T])[];
+};
+
+export type SearchResult<
+  T extends keyof IndexerSchema,
+  O extends SearchOptions<T>,
+> = {
+  pagination: ResultPagination;
+  nodes: ({ id: string; score: number } & (O['fields'] extends any[]
+    ? { fields: { [key in O['fields'][number]]: string | string[] } }
+    : unknown) &
+    (O['highlights'] extends any[]
+      ? { highlights: { [key in O['highlights'][number]['field']]: string[] } }
+      : unknown))[];
+};
+
+export interface AggregateOptions<T extends keyof IndexerSchema> {
+  pagination?: PaginationOption;
+  hits?: SearchOptions<T>;
+}
+
+export type AggregateResult<
+  T extends keyof IndexerSchema,
+  O extends AggregateOptions<T>,
+> = {
+  pagination: ResultPagination;
+  buckets: ({
+    key: string;
+    score: number;
+    count: number;
+  } & (O['hits'] extends object
+    ? { hits: SearchResult<T, O['hits']> }
+    : unknown))[];
+};
+
+export abstract class IndexerStorageBase implements IndexerStorage {
+  readonly storageType = 'indexer';
+  readonly recommendRefreshInterval: number = 100; // 100ms
+  abstract readonly connection: Connection;
+  abstract readonly isReadonly: boolean;
+
+  abstract search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>>;
+
+  abstract aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>>;
+
+  abstract search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>>;
+
+  abstract aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>>;
+
+  abstract deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void>;
+
+  abstract insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  abstract delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void>;
+
+  abstract update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  abstract refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
+
+  abstract refreshIfNeed(): Promise<void>;
+
+  abstract indexVersion(): Promise<number>;
+}

packages/common/nbstore/src/storage/indexer/document.ts

@@ -0,0 +1,58 @@
+import type { IndexerSchema } from './schema';
+
+export class IndexerDocument<
+  S extends keyof IndexerSchema = keyof IndexerSchema,
+> {
+  constructor(public readonly id: string) {}
+
+  fields = new Map<keyof IndexerSchema[S], string[]>();
+
+  public insert<F extends keyof IndexerSchema[S]>(
+    field: F,
+    value: string | string[]
+  ) {
+    const values = this.fields.get(field) ?? [];
+    if (Array.isArray(value)) {
+      values.push(...value);
+    } else {
+      values.push(value);
+    }
+    this.fields.set(field, values);
+  }
+
+  get<F extends keyof IndexerSchema[S]>(
+    field: F
+  ): string[] | string | undefined {
+    const values = this.fields.get(field);
+    if (values === undefined) {
+      return undefined;
+    } else if (values.length === 1) {
+      return values[0];
+    } else {
+      return values;
+    }
+  }
+
+  static from<S extends keyof IndexerSchema>(
+    id: string,
+    map:
+      | Partial<Record<keyof IndexerSchema[S], string | string[]>>
+      | Map<keyof IndexerSchema[S], string | string[]>
+  ): IndexerDocument<S> {
+    const doc = new IndexerDocument<S>(id);
+
+    if (map instanceof Map) {
+      for (const [key, value] of map) {
+        doc.insert(key, value);
+      }
+    } else {
+      for (const key in map) {
+        if (map[key] === undefined || map[key] === null) {
+          continue;
+        }
+        doc.insert(key, map[key]);
+      }
+    }
+    return doc;
+  }
+}

packages/common/nbstore/src/storage/indexer/field-type.ts

@@ -0,0 +1 @@
+export type IndexFieldType = 'Integer' | 'FullText' | 'String' | 'Boolean';

packages/common/nbstore/src/storage/indexer/query.ts

@@ -0,0 +1,35 @@
+import type { IndexerSchema } from './schema';
+
+export type MatchQuery<T extends keyof IndexerSchema> = {
+  type: 'match';
+  field: keyof IndexerSchema[T];
+  match: string;
+};
+
+export type BoostQuery = {
+  type: 'boost';
+  query: Query<any>;
+  boost: number;
+};
+
+export type BooleanQuery<T extends keyof IndexerSchema> = {
+  type: 'boolean';
+  occur: 'should' | 'must' | 'must_not';
+  queries: Query<T>[];
+};
+
+export type ExistsQuery<T extends keyof IndexerSchema> = {
+  type: 'exists';
+  field: keyof IndexerSchema[T];
+};
+
+export type AllQuery = {
+  type: 'all';
+};
+
+export type Query<T extends keyof IndexerSchema> =
+  | BooleanQuery<T>
+  | MatchQuery<T>
+  | AllQuery
+  | ExistsQuery<T>
+  | BoostQuery;

packages/common/nbstore/src/storage/indexer/schema.ts

@@ -0,0 +1,51 @@
+import type { IndexFieldType } from './field-type';
+
+export const IndexerSchema = {
+  doc: {
+    docId: 'String',
+    title: 'FullText',
+    // summary of the doc, used for preview
+    summary: { type: 'String', index: false },
+  },
+  block: {
+    docId: 'String',
+    blockId: 'String',
+    content: 'FullText',
+    flavour: 'String',
+    blob: 'String',
+    // reference doc id
+    // ['xxx','yyy']
+    refDocId: 'String',
+    // reference info, used for backlink to specific block
+    // [{"docId":"xxx","mode":"page","blockIds":["gt5Yfq1maYvgNgpi13rIq"]},{"docId":"yyy","mode":"edgeless","blockIds":["k5prpOlDF-9CzfatmO0W7"]}]
+    ref: { type: 'String', index: false },
+    // parent block flavour
+    parentFlavour: 'String',
+    // parent block id
+    parentBlockId: 'String',
+    // additional info
+    // { "databaseName": "xxx", "displayMode": "page/edgeless", "noteBlockId": "xxx" }
+    additional: { type: 'String', index: false },
+    markdownPreview: { type: 'String', index: false },
+  },
+} satisfies Record<string, Record<string, IndexerFieldSchema>>;
+
+export type IndexerFieldSchema =
+  | IndexFieldType
+  | {
+      type: IndexFieldType;
+      /**
+       * If false, the field will not be indexed, and thus not searchable.
+       *
+       * default: true
+       */
+      index?: boolean;
+      /**
+       * If false, the field will not be stored, and not included in the search result.
+       *
+       * default: true
+       */
+      store?: boolean;
+    };
+
+export type IndexerSchema = typeof IndexerSchema;

packages/common/nbstore/src/storage/lock.ts

@@ -0,0 +1,44 @@
+export interface Locker {
+  lock(domain: string, resource: string): Promise<AsyncDisposable>;
+}
+
+export class SingletonLocker implements Locker {
+  lockedResource = new Map<string, Lock>();
+  constructor() {}
+
+  async lock(domain: string, resource: string) {
+    const key = `${domain}:${resource}`;
+    let lock = this.lockedResource.get(key);
+
+    if (!lock) {
+      lock = new Lock();
+      this.lockedResource.set(key, lock);
+    }
+
+    await lock.acquire();
+
+    return lock;
+  }
+}
+
+export class Lock {
+  private inner: Promise<void> = Promise.resolve();
+  private release: () => void = () => {};
+
+  async acquire() {
+    // oxlint-disable-next-line typescript/no-non-null-assertion
+    let release: () => void = null!;
+    const nextLock = new Promise<void>(resolve => {
+      release = resolve;
+    });
+
+    await this.inner;
+    this.inner = nextLock;
+    this.release = release;
+  }
+
+  [Symbol.asyncDispose]() {
+    this.release();
+    return Promise.resolve();
+  }
+}

packages/common/nbstore/src/storage/storage.ts

@@ -0,0 +1,15 @@
+import type { Connection } from '../connection';
+
+export type StorageType =
+  | 'blob'
+  | 'blobSync'
+  | 'doc'
+  | 'docSync'
+  | 'awareness'
+  | 'indexer'
+  | 'indexerSync';
+
+export interface Storage {
+  readonly storageType: StorageType;
+  readonly connection: Connection;
+}