snapshot of https://github.com/toeverything/AFFiNE.git (canary, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Build & Test / Analyze (javascript, affine) (push) Failing after 4s
Build & Test / Analyze (typescript, affine) (push) Failing after 3s
Build & Test / Rust test filter (push) Failing after 4s
Build & Test / Check yarn binary (push) Failing after 3s
Build & Test / E2E BlockSuite Test (3) (push) Failing after 3s
Build & Test / Analyze (javascript, blocksuite) (push) Failing after 3s
Build & Test / Analyze (typescript, blocksuite) (push) Failing after 3s
Build & Test / Mobile native build filter (push) Failing after 3s
Build & Test / E2E BlockSuite Test (1) (push) Failing after 4s
Build & Test / E2E BlockSuite Test (2) (push) Failing after 4s
Build & Test / E2E Mobile Test (1) (push) Failing after 3s
Build & Test / Unit Test (1) (push) Has been skipped
Build & Test / Unit Test (2) (push) Has been skipped
Build & Test / Unit Test (3) (push) Has been skipped
Build & Test / Unit Test (4) (push) Has been skipped
Build & Test / Unit Test (5) (push) Has been skipped
Build & Test / Native Unit Test (push) Has been skipped
Build & Test / E2E BlockSuite Test (4) (push) Failing after 3s
Build & Test / E2E BlockSuite Test (5) (push) Failing after 3s
Build & Test / E2E BlockSuite Cross Browser Test (push) Failing after 3s
Build & Test / E2E Mobile Test (2) (push) Failing after 3s
Build & Test / Build AFFiNE native (x86_64-unknown-linux-gnu) (push) Failing after 3s
Build & Test / Build Server native (push) Failing after 3s
Build & Test / Check Git Status (push) Has been skipped
Build & Test / Server Test (0, 4) (push) Has been skipped
Build & Test / Server Test (1, 4) (push) Has been skipped
Build & Test / Server Test (2, 4) (push) Has been skipped
Build & Test / Server Test (3, 4) (push) Has been skipped
Build & Test / Server Test with Elasticsearch (push) Has been skipped
Build & Test / Build @affine/electron renderer (push) Failing after 3s
Build & Test / Copilot test filter (push) Failing after 4s
Build & Test / Lint Rust (push) Has been skipped
Build & Test / Server Test with Embedded Search (push) Has been skipped
Build & Test / Server Test with Manticore Search (push) Has been skipped
Build & Test / Server E2E Test (push) Has been skipped
Build & Test / Cloud Desktop E2E Test (push) Has been skipped
Build & Test / Cloud E2E Test 1/2 (push) Has been skipped
Build & Test / Cloud E2E Test 2/2 (push) Has been skipped
Build & Test / Build Android app (push) Has been skipped
Build & Test / Test Android app (push) Has been skipped
Build & Test / Run native tests (push) Has been skipped
Build & Test / Server Copilot Api Test (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (1, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (2, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (3, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (4, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (5, 5) (push) Has been skipped
Release / Canary Gate (push) Has been skipped
Build & Test / Build AFFiNE native (aarch64-apple-darwin) (push) Has been cancelled
Build & Test / Lint (push) Has been cancelled
Build & Test / Typecheck (push) Has been cancelled
Build & Test / Build AFFiNE native (x86_64-apple-darwin) (push) Has been cancelled
Build & Test / Desktop Test (ubuntu-latest, linux, x64, x86_64-unknown-linux-gnu, true) (push) Has been cancelled
Build & Test / Desktop Test (windows-latest, windows, x64, x86_64-pc-windows-msvc, true) (push) Has been cancelled
Build & Test / E2E Test (1) (push) Has been cancelled
Build & Test / E2E Test (2) (push) Has been cancelled
Build & Test / E2E Test (3) (push) Has been cancelled
Build & Test / E2E Test (4) (push) Has been cancelled
Build & Test / E2E Test (5) (push) Has been cancelled
Build & Test / Build AFFiNE native (aarch64-pc-windows-msvc) (push) Has been cancelled
Build & Test / Build AFFiNE native (x86_64-pc-windows-msvc) (push) Has been cancelled
Build & Test / Desktop Test (macos-latest, macos, arm64, aarch64-apple-darwin, true) (push) Has been cancelled
Build & Test / Desktop Test (macos-latest, macos, x64, x86_64-apple-darwin, false) (push) Has been cancelled
Build & Test / 3, 2, 1 Launch (push) Has been cancelled
Build & Test / Build iOS app (push) Has been cancelled
Copilot Test Automatically / Setup Test (push) Failing after 1s
Release / Prepare (push) Failing after 2s
Release / Release Cloud (push) Has been skipped
Release / Release Desktop (push) Has been skipped
Release / Release Docker Image (push) Has been skipped
Release / Release Mobile (push) Has been skipped
Some checks failed
Build & Test / Analyze (javascript, affine) (push) Failing after 4s
Build & Test / Analyze (typescript, affine) (push) Failing after 3s
Build & Test / Rust test filter (push) Failing after 4s
Build & Test / Check yarn binary (push) Failing after 3s
Build & Test / E2E BlockSuite Test (3) (push) Failing after 3s
Build & Test / Analyze (javascript, blocksuite) (push) Failing after 3s
Build & Test / Analyze (typescript, blocksuite) (push) Failing after 3s
Build & Test / Mobile native build filter (push) Failing after 3s
Build & Test / E2E BlockSuite Test (1) (push) Failing after 4s
Build & Test / E2E BlockSuite Test (2) (push) Failing after 4s
Build & Test / E2E Mobile Test (1) (push) Failing after 3s
Build & Test / Unit Test (1) (push) Has been skipped
Build & Test / Unit Test (2) (push) Has been skipped
Build & Test / Unit Test (3) (push) Has been skipped
Build & Test / Unit Test (4) (push) Has been skipped
Build & Test / Unit Test (5) (push) Has been skipped
Build & Test / Native Unit Test (push) Has been skipped
Build & Test / E2E BlockSuite Test (4) (push) Failing after 3s
Build & Test / E2E BlockSuite Test (5) (push) Failing after 3s
Build & Test / E2E BlockSuite Cross Browser Test (push) Failing after 3s
Build & Test / E2E Mobile Test (2) (push) Failing after 3s
Build & Test / Build AFFiNE native (x86_64-unknown-linux-gnu) (push) Failing after 3s
Build & Test / Build Server native (push) Failing after 3s
Build & Test / Check Git Status (push) Has been skipped
Build & Test / Server Test (0, 4) (push) Has been skipped
Build & Test / Server Test (1, 4) (push) Has been skipped
Build & Test / Server Test (2, 4) (push) Has been skipped
Build & Test / Server Test (3, 4) (push) Has been skipped
Build & Test / Server Test with Elasticsearch (push) Has been skipped
Build & Test / Build @affine/electron renderer (push) Failing after 3s
Build & Test / Copilot test filter (push) Failing after 4s
Build & Test / Lint Rust (push) Has been skipped
Build & Test / Server Test with Embedded Search (push) Has been skipped
Build & Test / Server Test with Manticore Search (push) Has been skipped
Build & Test / Server E2E Test (push) Has been skipped
Build & Test / Cloud Desktop E2E Test (push) Has been skipped
Build & Test / Cloud E2E Test 1/2 (push) Has been skipped
Build & Test / Cloud E2E Test 2/2 (push) Has been skipped
Build & Test / Build Android app (push) Has been skipped
Build & Test / Test Android app (push) Has been skipped
Build & Test / Run native tests (push) Has been skipped
Build & Test / Server Copilot Api Test (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (1, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (2, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (3, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (4, 5) (push) Has been skipped
Build & Test / Frontend Copilot E2E Test (5, 5) (push) Has been skipped
Release / Canary Gate (push) Has been skipped
Build & Test / Build AFFiNE native (aarch64-apple-darwin) (push) Has been cancelled
Build & Test / Lint (push) Has been cancelled
Build & Test / Typecheck (push) Has been cancelled
Build & Test / Build AFFiNE native (x86_64-apple-darwin) (push) Has been cancelled
Build & Test / Desktop Test (ubuntu-latest, linux, x64, x86_64-unknown-linux-gnu, true) (push) Has been cancelled
Build & Test / Desktop Test (windows-latest, windows, x64, x86_64-pc-windows-msvc, true) (push) Has been cancelled
Build & Test / E2E Test (1) (push) Has been cancelled
Build & Test / E2E Test (2) (push) Has been cancelled
Build & Test / E2E Test (3) (push) Has been cancelled
Build & Test / E2E Test (4) (push) Has been cancelled
Build & Test / E2E Test (5) (push) Has been cancelled
Build & Test / Build AFFiNE native (aarch64-pc-windows-msvc) (push) Has been cancelled
Build & Test / Build AFFiNE native (x86_64-pc-windows-msvc) (push) Has been cancelled
Build & Test / Desktop Test (macos-latest, macos, arm64, aarch64-apple-darwin, true) (push) Has been cancelled
Build & Test / Desktop Test (macos-latest, macos, x64, x86_64-apple-darwin, false) (push) Has been cancelled
Build & Test / 3, 2, 1 Launch (push) Has been cancelled
Build & Test / Build iOS app (push) Has been cancelled
Copilot Test Automatically / Setup Test (push) Failing after 1s
Release / Prepare (push) Failing after 2s
Release / Release Cloud (push) Has been skipped
Release / Release Desktop (push) Has been skipped
Release / Release Docker Image (push) Has been skipped
Release / Release Mobile (push) Has been skipped
This commit is contained in:
3
packages/frontend/native/.gitignore
vendored
Normal file
3
packages/frontend/native/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
*.fixture
|
||||
lib
|
||||
*.bin
|
||||
63
packages/frontend/native/Cargo.toml
Normal file
63
packages/frontend/native/Cargo.toml
Normal file
@@ -0,0 +1,63 @@
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "affine_native"
|
||||
publish = false
|
||||
version = "0.0.0"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[dependencies]
|
||||
affine_common = { workspace = true, features = ["hashcash"] }
|
||||
affine_doc_loader = { workspace = true }
|
||||
affine_importer = { workspace = true }
|
||||
affine_media_capture = { path = "./media_capture" }
|
||||
affine_nbstore = { workspace = true, features = ["napi"] }
|
||||
affine_sqlite_v1 = { path = "./sqlite_v1" }
|
||||
hex = { workspace = true }
|
||||
infer = { workspace = true }
|
||||
napi = { workspace = true }
|
||||
napi-derive = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
onenote_parser = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
sqlx = { workspace = true, default-features = false, features = [
|
||||
"chrono",
|
||||
"macros",
|
||||
"migrate",
|
||||
"runtime-tokio",
|
||||
"sqlite",
|
||||
] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
typed-path = { workspace = true }
|
||||
|
||||
[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
|
||||
affine_preview = { workspace = true, features = ["mermaid", "typst"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
bytes = { workspace = true }
|
||||
|
||||
[target.'cfg(not(target_os = "linux"))'.dependencies]
|
||||
mimalloc = { workspace = true }
|
||||
|
||||
[target.'cfg(all(target_os = "linux", not(target_arch = "arm")))'.dependencies]
|
||||
mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
|
||||
|
||||
[dev-dependencies]
|
||||
chrono = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
napi-build = { workspace = true }
|
||||
sqlx = { workspace = true, default-features = false, features = [
|
||||
"chrono",
|
||||
"json",
|
||||
"macros",
|
||||
"migrate",
|
||||
"runtime-tokio",
|
||||
"sqlite",
|
||||
] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
13
packages/frontend/native/__tests__/db.spec.mts
Normal file
13
packages/frontend/native/__tests__/db.spec.mts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import test from 'ava';
|
||||
|
||||
import { SqliteConnection, ValidationResult } from '../index.js';
|
||||
|
||||
test('db validate', async t => {
|
||||
const path = fileURLToPath(
|
||||
new URL('./fixtures/test01.affine', import.meta.url)
|
||||
);
|
||||
const result = await SqliteConnection.validate(path);
|
||||
t.is(result, ValidationResult.MissingVersionColumn);
|
||||
});
|
||||
32
packages/frontend/native/__tests__/pool.spec.mts
Normal file
32
packages/frontend/native/__tests__/pool.spec.mts
Normal file
@@ -0,0 +1,32 @@
|
||||
import test from 'ava';
|
||||
|
||||
import { DocStoragePool } from '../index.js';
|
||||
|
||||
test('can batch read/write pool', async t => {
|
||||
const pool = new DocStoragePool();
|
||||
await pool.connect('test', ':memory:');
|
||||
|
||||
const batch = 512;
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: batch }).map(async (_, i) => {
|
||||
return pool.setBlob('test', {
|
||||
key: `test-blob-${i}`,
|
||||
data: new Uint8Array([i % 255]),
|
||||
mime: 'text/plain',
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
const blobs = await Promise.all(
|
||||
Array.from({ length: batch }).map(async (_, i) => {
|
||||
return pool.getBlob('test', `test-blob-${i}`);
|
||||
})
|
||||
);
|
||||
|
||||
t.is(blobs.length, batch);
|
||||
t.is(
|
||||
blobs.every((blob, i) => blob!.data.at(0) === i % 255),
|
||||
true
|
||||
);
|
||||
});
|
||||
5
packages/frontend/native/build.rs
Normal file
5
packages/frontend/native/build.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), std::io::Error> {
|
||||
napi_build::setup();
|
||||
Ok(())
|
||||
}
|
||||
379
packages/frontend/native/index.d.ts
vendored
Normal file
379
packages/frontend/native/index.d.ts
vendored
Normal file
@@ -0,0 +1,379 @@
|
||||
/* auto-generated by NAPI-RS */
|
||||
/* oxlint-disable */
|
||||
export declare class ApplicationInfo {
|
||||
processId: number
|
||||
name: string
|
||||
objectId: number
|
||||
constructor(processId: number, name: string, objectId: number)
|
||||
get processGroupId(): number
|
||||
get bundleIdentifier(): string
|
||||
get icon(): Buffer
|
||||
}
|
||||
|
||||
export declare class ApplicationListChangedSubscriber {
|
||||
unsubscribe(): void
|
||||
}
|
||||
|
||||
export declare class ApplicationStateChangedSubscriber {
|
||||
unsubscribe(): void
|
||||
}
|
||||
|
||||
export declare class ShareableContent {
|
||||
static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber
|
||||
static onAppStateChanged(app: ApplicationInfo, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber
|
||||
constructor()
|
||||
static applications(): Array<ApplicationInfo>
|
||||
static applicationWithProcessId(processId: number): ApplicationInfo | null
|
||||
static isUsingMicrophone(processId: number): boolean
|
||||
}
|
||||
|
||||
export declare function abortRecording(id: string): Promise<void>
|
||||
|
||||
export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise<Float32Array>
|
||||
|
||||
/** Decode audio file into a Float32Array */
|
||||
export declare function decodeAudioSync(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null): Float32Array
|
||||
|
||||
export interface RecordingArtifact {
|
||||
id: string
|
||||
filepath: string
|
||||
sampleRate: number
|
||||
channels: number
|
||||
durationMs: number
|
||||
size: number
|
||||
degraded: boolean
|
||||
overflowCount: number
|
||||
}
|
||||
|
||||
export interface RecordingSessionMeta {
|
||||
id: string
|
||||
filepath: string
|
||||
sampleRate: number
|
||||
channels: number
|
||||
startedAt: number
|
||||
}
|
||||
|
||||
export interface RecordingStartOptions {
|
||||
appProcessId?: number
|
||||
excludeProcessIds?: Array<number>
|
||||
outputDir: string
|
||||
format?: string
|
||||
sampleRate?: number
|
||||
channels?: number
|
||||
id?: string
|
||||
}
|
||||
|
||||
export declare function startRecording(opts: RecordingStartOptions): Promise<RecordingSessionMeta>
|
||||
|
||||
export declare function stopRecording(id: string): Promise<RecordingArtifact>
|
||||
export declare function cancelImportSession(sessionId: string): void
|
||||
|
||||
export interface CreateImportBatchLimits {
|
||||
maxDocs?: number
|
||||
maxBlobs?: number
|
||||
maxBlobBytes?: number
|
||||
}
|
||||
|
||||
export declare function createImportSession(options: CreateImportSessionOptions): string
|
||||
|
||||
export interface CreateImportSessionOptions {
|
||||
format: string
|
||||
source: CreateImportSessionSource
|
||||
batchLimits?: CreateImportBatchLimits
|
||||
}
|
||||
|
||||
export interface CreateImportSessionSource {
|
||||
kind: string
|
||||
path: string
|
||||
}
|
||||
|
||||
export declare function disposeImportSession(sessionId: string): void
|
||||
|
||||
export interface MermaidRenderOptions {
|
||||
theme?: string
|
||||
fontFamily?: string
|
||||
fontSize?: number
|
||||
}
|
||||
|
||||
export interface MermaidRenderRequest {
|
||||
code: string
|
||||
options?: MermaidRenderOptions
|
||||
}
|
||||
|
||||
export interface MermaidRenderResult {
|
||||
svg: string
|
||||
}
|
||||
|
||||
export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
|
||||
|
||||
export declare function nextImportBatch(sessionId: string): string | null
|
||||
|
||||
export declare function renderMermaidSvg(request: MermaidRenderRequest): MermaidRenderResult
|
||||
|
||||
export declare function renderTypstSvg(request: TypstRenderRequest): TypstRenderResult
|
||||
|
||||
export interface TypstRenderOptions {
|
||||
fontUrls?: Array<string>
|
||||
fontDirs?: Array<string>
|
||||
}
|
||||
|
||||
export interface TypstRenderRequest {
|
||||
code: string
|
||||
options?: TypstRenderOptions
|
||||
}
|
||||
|
||||
export interface TypstRenderResult {
|
||||
svg: string
|
||||
}
|
||||
|
||||
export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise<boolean>
|
||||
export declare class DocStorage {
|
||||
constructor(path: string)
|
||||
validate(): Promise<boolean>
|
||||
validateImportSchema(): Promise<boolean>
|
||||
vacuumInto(path: string): Promise<void>
|
||||
setSpaceId(spaceId: string): Promise<void>
|
||||
}
|
||||
|
||||
export declare class DocStoragePool {
|
||||
constructor()
|
||||
/** Initialize the database and run migrations. */
|
||||
connect(universalId: string, path: string): Promise<void>
|
||||
disconnect(universalId: string): Promise<void>
|
||||
checkpoint(universalId: string): Promise<void>
|
||||
vacuumInto(universalId: string, path: string): Promise<void>
|
||||
crawlDocData(universalId: string, docId: string): Promise<NativeCrawlResult>
|
||||
setSpaceId(universalId: string, spaceId: string): Promise<void>
|
||||
pushUpdate(universalId: string, docId: string, update: Uint8Array): Promise<Date>
|
||||
getDocSnapshot(universalId: string, docId: string): Promise<DocRecord | null>
|
||||
setDocSnapshot(universalId: string, snapshot: DocRecord): Promise<boolean>
|
||||
getDocUpdates(universalId: string, docId: string): Promise<Array<DocUpdate>>
|
||||
markUpdatesMerged(universalId: string, docId: string, updates: Array<Date>): Promise<number>
|
||||
deleteDoc(universalId: string, docId: string): Promise<void>
|
||||
getDocClocks(universalId: string, after?: Date | undefined | null): Promise<Array<DocClock>>
|
||||
getDocClock(universalId: string, docId: string): Promise<DocClock | null>
|
||||
getDocIndexedClock(universalId: string, docId: string): Promise<DocIndexedClock | null>
|
||||
setDocIndexedClock(universalId: string, docId: string, indexedClock: Date, indexerVersion: number): Promise<void>
|
||||
setDocIndexedClocks(universalId: string, clocks: Array<DocIndexedClock>): Promise<void>
|
||||
clearDocIndexedClock(universalId: string, docId: string): Promise<void>
|
||||
getBlob(universalId: string, key: string): Promise<Blob | null>
|
||||
setBlob(universalId: string, blob: SetBlob): Promise<void>
|
||||
deleteBlob(universalId: string, key: string, permanently: boolean): Promise<void>
|
||||
releaseBlobs(universalId: string): Promise<void>
|
||||
listBlobs(universalId: string): Promise<Array<ListedBlob>>
|
||||
getPeerRemoteClocks(universalId: string, peer: string): Promise<Array<DocClock>>
|
||||
getPeerRemoteClock(universalId: string, peer: string, docId: string): Promise<DocClock | null>
|
||||
setPeerRemoteClock(universalId: string, peer: string, docId: string, clock: Date): Promise<void>
|
||||
getPeerPulledRemoteClocks(universalId: string, peer: string): Promise<Array<DocClock>>
|
||||
getPeerPulledRemoteClock(universalId: string, peer: string, docId: string): Promise<DocClock | null>
|
||||
setPeerPulledRemoteClock(universalId: string, peer: string, docId: string, clock: Date): Promise<void>
|
||||
getPeerPushedClocks(universalId: string, peer: string): Promise<Array<DocClock>>
|
||||
getPeerPushedClock(universalId: string, peer: string, docId: string): Promise<DocClock | null>
|
||||
setPeerPushedClock(universalId: string, peer: string, docId: string, clock: Date): Promise<void>
|
||||
clearClocks(universalId: string): Promise<void>
|
||||
setBlobUploadedAt(universalId: string, peer: string, blobId: string, uploadedAt?: Date | undefined | null): Promise<void>
|
||||
getBlobUploadedAt(universalId: string, peer: string, blobId: string): Promise<Date | null>
|
||||
indexUpsert(id: string, table: string, document: NativeIndexDocument): Promise<void>
|
||||
indexFlush(id: string): Promise<void>
|
||||
indexVersion(): Promise<number>
|
||||
indexDelete(id: string, table: string, docId: string): Promise<void>
|
||||
indexSearch(id: string, table: string, query: NativeIndexQuery, options: NativeIndexSearchOptions): Promise<NativeIndexSearchResult>
|
||||
indexAggregate(id: string, table: string, query: NativeIndexQuery, field: string, limit: number, offset: number, hits?: NativeIndexSearchOptions | undefined | null): Promise<NativeIndexAggregateResult>
|
||||
indexDeleteByQuery(id: string, table: string, query: NativeIndexQuery): Promise<number>
|
||||
}
|
||||
|
||||
export interface Blob {
|
||||
key: string
|
||||
data: Uint8Array
|
||||
mime: string
|
||||
size: number
|
||||
createdAt: Date
|
||||
}
|
||||
|
||||
export interface DocClock {
|
||||
docId: string
|
||||
timestamp: Date
|
||||
}
|
||||
|
||||
export interface DocIndexedClock {
|
||||
docId: string
|
||||
timestamp: Date
|
||||
indexerVersion: number
|
||||
}
|
||||
|
||||
export interface DocRecord {
|
||||
docId: string
|
||||
bin: Uint8Array
|
||||
timestamp: Date
|
||||
}
|
||||
|
||||
export interface DocUpdate {
|
||||
docId: string
|
||||
timestamp: Date
|
||||
bin: Uint8Array
|
||||
}
|
||||
|
||||
export interface ListedBlob {
|
||||
key: string
|
||||
size: number
|
||||
mime: string
|
||||
createdAt: Date
|
||||
}
|
||||
|
||||
export interface NativeBlockInfo {
|
||||
blockId: string
|
||||
flavour: string
|
||||
content?: Array<string>
|
||||
blob?: Array<string>
|
||||
refDocId?: Array<string>
|
||||
refInfo?: Array<string>
|
||||
parentFlavour?: string
|
||||
parentBlockId?: string
|
||||
additional?: string
|
||||
}
|
||||
|
||||
export interface NativeCrawlResult {
|
||||
blocks: Array<NativeBlockInfo>
|
||||
title: string
|
||||
summary: string
|
||||
}
|
||||
|
||||
export interface NativeIndexAggregateResult {
|
||||
total: number
|
||||
buckets: Array<NativeIndexBucket>
|
||||
}
|
||||
|
||||
export interface NativeIndexBucket {
|
||||
key: string
|
||||
count: number
|
||||
score: number
|
||||
hits: Array<NativeIndexHit>
|
||||
}
|
||||
|
||||
export interface NativeIndexDocument {
|
||||
id: string
|
||||
fields: Array<NativeIndexField>
|
||||
}
|
||||
|
||||
export interface NativeIndexField {
|
||||
field: string
|
||||
values: Array<string>
|
||||
}
|
||||
|
||||
export interface NativeIndexHighlight {
|
||||
field: string
|
||||
values: Array<NativeIndexHighlightValue>
|
||||
}
|
||||
|
||||
export interface NativeIndexHighlightValue {
|
||||
valueIndex: number
|
||||
spans: Array<NativeIndexSpan>
|
||||
}
|
||||
|
||||
export interface NativeIndexHit {
|
||||
id: string
|
||||
score: number
|
||||
fields: Array<NativeIndexField>
|
||||
highlights: Array<NativeIndexHighlight>
|
||||
}
|
||||
|
||||
export interface NativeIndexQuery {
|
||||
kind: string
|
||||
field?: string
|
||||
value?: string
|
||||
occur?: string
|
||||
clauses?: Array<NativeIndexQuery>
|
||||
boost?: number
|
||||
}
|
||||
|
||||
export interface NativeIndexSearchOptions {
|
||||
limit: number
|
||||
offset: number
|
||||
fields: Array<string>
|
||||
highlights: Array<string>
|
||||
}
|
||||
|
||||
export interface NativeIndexSearchResult {
|
||||
total: number
|
||||
hits: Array<NativeIndexHit>
|
||||
}
|
||||
|
||||
export interface NativeIndexSpan {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
export interface SetBlob {
|
||||
key: string
|
||||
data: Uint8Array
|
||||
mime: string
|
||||
}
|
||||
export declare class SqliteConnection {
|
||||
constructor(path: string)
|
||||
connect(): Promise<void>
|
||||
addBlob(key: string, blob: Uint8Array): Promise<void>
|
||||
getBlob(key: string): Promise<BlobRow | null>
|
||||
deleteBlob(key: string): Promise<void>
|
||||
getBlobKeys(): Promise<Array<string>>
|
||||
getUpdates(docId?: string | undefined | null): Promise<Array<UpdateRow>>
|
||||
getDocTimestamps(): Promise<Array<DocTimestampRow>>
|
||||
deleteUpdates(docId?: string | undefined | null): Promise<void>
|
||||
getUpdatesCount(docId?: string | undefined | null): Promise<number>
|
||||
getAllUpdates(): Promise<Array<UpdateRow>>
|
||||
insertUpdates(updates: Array<InsertRow>): Promise<void>
|
||||
replaceUpdates(docId: string | undefined | null, updates: Array<InsertRow>): Promise<void>
|
||||
getServerClock(key: string): Promise<BlobRow | null>
|
||||
setServerClock(key: string, data: Uint8Array): Promise<void>
|
||||
getServerClockKeys(): Promise<Array<string>>
|
||||
clearServerClock(): Promise<void>
|
||||
delServerClock(key: string): Promise<void>
|
||||
getSyncMetadata(key: string): Promise<BlobRow | null>
|
||||
setSyncMetadata(key: string, data: Uint8Array): Promise<void>
|
||||
getSyncMetadataKeys(): Promise<Array<string>>
|
||||
clearSyncMetadata(): Promise<void>
|
||||
delSyncMetadata(key: string): Promise<void>
|
||||
initVersion(): Promise<void>
|
||||
setVersion(version: number): Promise<void>
|
||||
getMaxVersion(): Promise<number>
|
||||
close(): Promise<void>
|
||||
get isClose(): boolean
|
||||
static validate(path: string): Promise<ValidationResult>
|
||||
validateImportSchema(): Promise<boolean>
|
||||
migrateAddDocId(): Promise<void>
|
||||
/** * Flush the WAL file to the database file.
|
||||
* See https://www.sqlite.org/pragma.html#pragma_wal_checkpoint:~:text=PRAGMA%20schema.wal_checkpoint%3B
|
||||
*/
|
||||
checkpoint(): Promise<void>
|
||||
vacuumInto(path: string): Promise<void>
|
||||
}
|
||||
|
||||
export interface BlobRow {
|
||||
key: string
|
||||
data: Buffer
|
||||
timestamp: Date
|
||||
}
|
||||
|
||||
export interface DocTimestampRow {
|
||||
docId?: string
|
||||
timestamp: Date
|
||||
}
|
||||
|
||||
export interface InsertRow {
|
||||
docId?: string
|
||||
data: Uint8Array
|
||||
}
|
||||
|
||||
export interface UpdateRow {
|
||||
id: number
|
||||
timestamp: Date
|
||||
data: Buffer
|
||||
docId?: string
|
||||
}
|
||||
|
||||
export declare enum ValidationResult {
|
||||
MissingTables = 0,
|
||||
MissingDocIdColumn = 1,
|
||||
MissingVersionColumn = 2,
|
||||
GeneralError = 3,
|
||||
Valid = 4
|
||||
}
|
||||
595
packages/frontend/native/index.js
Normal file
595
packages/frontend/native/index.js
Normal file
@@ -0,0 +1,595 @@
|
||||
// prettier-ignore
|
||||
/* oxlint-disable */
|
||||
// @ts-nocheck
|
||||
/* auto-generated by NAPI-RS */
|
||||
|
||||
const { readFileSync } = require('node:fs')
|
||||
let nativeBinding = null
|
||||
const loadErrors = []
|
||||
|
||||
const isMusl = () => {
|
||||
let musl = false
|
||||
if (process.platform === 'linux') {
|
||||
musl = isMuslFromFilesystem()
|
||||
if (musl === null) {
|
||||
musl = isMuslFromReport()
|
||||
}
|
||||
if (musl === null) {
|
||||
musl = isMuslFromChildProcess()
|
||||
}
|
||||
}
|
||||
return musl
|
||||
}
|
||||
|
||||
const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-')
|
||||
|
||||
const isMuslFromFilesystem = () => {
|
||||
try {
|
||||
return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl')
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const isMuslFromReport = () => {
|
||||
let report = null
|
||||
if (typeof process.report?.getReport === 'function') {
|
||||
process.report.excludeNetwork = true
|
||||
report = process.report.getReport()
|
||||
}
|
||||
if (!report) {
|
||||
return null
|
||||
}
|
||||
if (report.header && report.header.glibcVersionRuntime) {
|
||||
return false
|
||||
}
|
||||
if (Array.isArray(report.sharedObjects)) {
|
||||
if (report.sharedObjects.some(isFileMusl)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const isMuslFromChildProcess = () => {
|
||||
try {
|
||||
return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl')
|
||||
} catch (e) {
|
||||
// If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function requireNative() {
|
||||
if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
|
||||
try {
|
||||
return require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
|
||||
} catch (err) {
|
||||
loadErrors.push(err)
|
||||
}
|
||||
} else if (process.platform === 'android') {
|
||||
if (process.arch === 'arm64') {
|
||||
try {
|
||||
return require('./affine.android-arm64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-android-arm64')
|
||||
const bindingPackageVersion = require('@affine/native-android-arm64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 'arm') {
|
||||
try {
|
||||
return require('./affine.android-arm-eabi.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-android-arm-eabi')
|
||||
const bindingPackageVersion = require('@affine/native-android-arm-eabi/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`))
|
||||
}
|
||||
} else if (process.platform === 'win32') {
|
||||
if (process.arch === 'x64') {
|
||||
if (process.config?.variables?.shlib_suffix === 'dll.a' || process.config?.variables?.node_target_type === 'shared_library') {
|
||||
try {
|
||||
return require('./affine.win32-x64-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-win32-x64-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-win32-x64-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
return require('./affine.win32-x64-msvc.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-win32-x64-msvc')
|
||||
const bindingPackageVersion = require('@affine/native-win32-x64-msvc/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
}
|
||||
} else if (process.arch === 'ia32') {
|
||||
try {
|
||||
return require('./affine.win32-ia32-msvc.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-win32-ia32-msvc')
|
||||
const bindingPackageVersion = require('@affine/native-win32-ia32-msvc/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 'arm64') {
|
||||
try {
|
||||
return require('./affine.win32-arm64-msvc.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-win32-arm64-msvc')
|
||||
const bindingPackageVersion = require('@affine/native-win32-arm64-msvc/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`))
|
||||
}
|
||||
} else if (process.platform === 'darwin') {
|
||||
try {
|
||||
return require('./affine.darwin-universal.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-darwin-universal')
|
||||
const bindingPackageVersion = require('@affine/native-darwin-universal/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
if (process.arch === 'x64') {
|
||||
try {
|
||||
return require('./affine.darwin-x64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-darwin-x64')
|
||||
const bindingPackageVersion = require('@affine/native-darwin-x64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 'arm64') {
|
||||
try {
|
||||
return require('./affine.darwin-arm64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-darwin-arm64')
|
||||
const bindingPackageVersion = require('@affine/native-darwin-arm64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`))
|
||||
}
|
||||
} else if (process.platform === 'freebsd') {
|
||||
if (process.arch === 'x64') {
|
||||
try {
|
||||
return require('./affine.freebsd-x64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-freebsd-x64')
|
||||
const bindingPackageVersion = require('@affine/native-freebsd-x64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 'arm64') {
|
||||
try {
|
||||
return require('./affine.freebsd-arm64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-freebsd-arm64')
|
||||
const bindingPackageVersion = require('@affine/native-freebsd-arm64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`))
|
||||
}
|
||||
} else if (process.platform === 'linux') {
|
||||
if (process.arch === 'x64') {
|
||||
if (isMusl()) {
|
||||
try {
|
||||
return require('./affine.linux-x64-musl.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-x64-musl')
|
||||
const bindingPackageVersion = require('@affine/native-linux-x64-musl/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
return require('./affine.linux-x64-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-x64-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-linux-x64-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
}
|
||||
} else if (process.arch === 'arm64') {
|
||||
if (isMusl()) {
|
||||
try {
|
||||
return require('./affine.linux-arm64-musl.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-arm64-musl')
|
||||
const bindingPackageVersion = require('@affine/native-linux-arm64-musl/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
return require('./affine.linux-arm64-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-arm64-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-linux-arm64-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
}
|
||||
} else if (process.arch === 'arm') {
|
||||
if (isMusl()) {
|
||||
try {
|
||||
return require('./affine.linux-arm-musleabihf.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-arm-musleabihf')
|
||||
const bindingPackageVersion = require('@affine/native-linux-arm-musleabihf/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
return require('./affine.linux-arm-gnueabihf.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-arm-gnueabihf')
|
||||
const bindingPackageVersion = require('@affine/native-linux-arm-gnueabihf/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
}
|
||||
} else if (process.arch === 'loong64') {
|
||||
if (isMusl()) {
|
||||
try {
|
||||
return require('./affine.linux-loong64-musl.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-loong64-musl')
|
||||
const bindingPackageVersion = require('@affine/native-linux-loong64-musl/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
return require('./affine.linux-loong64-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-loong64-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-linux-loong64-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
}
|
||||
} else if (process.arch === 'riscv64') {
|
||||
if (isMusl()) {
|
||||
try {
|
||||
return require('./affine.linux-riscv64-musl.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-riscv64-musl')
|
||||
const bindingPackageVersion = require('@affine/native-linux-riscv64-musl/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
return require('./affine.linux-riscv64-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-riscv64-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-linux-riscv64-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
}
|
||||
} else if (process.arch === 'ppc64') {
|
||||
try {
|
||||
return require('./affine.linux-ppc64-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-ppc64-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-linux-ppc64-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 's390x') {
|
||||
try {
|
||||
return require('./affine.linux-s390x-gnu.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-linux-s390x-gnu')
|
||||
const bindingPackageVersion = require('@affine/native-linux-s390x-gnu/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`))
|
||||
}
|
||||
} else if (process.platform === 'openharmony') {
|
||||
if (process.arch === 'arm64') {
|
||||
try {
|
||||
return require('./affine.openharmony-arm64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-openharmony-arm64')
|
||||
const bindingPackageVersion = require('@affine/native-openharmony-arm64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 'x64') {
|
||||
try {
|
||||
return require('./affine.openharmony-x64.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-openharmony-x64')
|
||||
const bindingPackageVersion = require('@affine/native-openharmony-x64/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else if (process.arch === 'arm') {
|
||||
try {
|
||||
return require('./affine.openharmony-arm.node')
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
try {
|
||||
const binding = require('@affine/native-openharmony-arm')
|
||||
const bindingPackageVersion = require('@affine/native-openharmony-arm/package.json').version
|
||||
if (bindingPackageVersion !== '0.27.5' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
||||
throw new Error(`Native binding package version mismatch, expected 0.27.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
||||
}
|
||||
return binding
|
||||
} catch (e) {
|
||||
loadErrors.push(e)
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`))
|
||||
}
|
||||
} else {
|
||||
loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`))
|
||||
}
|
||||
}
|
||||
|
||||
nativeBinding = requireNative()
|
||||
|
||||
if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
|
||||
let wasiBinding = null
|
||||
let wasiBindingError = null
|
||||
try {
|
||||
wasiBinding = require('./affine.wasi.cjs')
|
||||
nativeBinding = wasiBinding
|
||||
} catch (err) {
|
||||
if (process.env.NAPI_RS_FORCE_WASI) {
|
||||
wasiBindingError = err
|
||||
}
|
||||
}
|
||||
if (!nativeBinding) {
|
||||
try {
|
||||
wasiBinding = require('@affine/native-wasm32-wasi')
|
||||
nativeBinding = wasiBinding
|
||||
} catch (err) {
|
||||
if (process.env.NAPI_RS_FORCE_WASI) {
|
||||
wasiBindingError.cause = err
|
||||
loadErrors.push(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (process.env.NAPI_RS_FORCE_WASI === 'error' && !wasiBinding) {
|
||||
const error = new Error('WASI binding not found and NAPI_RS_FORCE_WASI is set to error')
|
||||
error.cause = wasiBindingError
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
if (!nativeBinding) {
|
||||
if (loadErrors.length > 0) {
|
||||
throw new Error(
|
||||
`Cannot find native binding. ` +
|
||||
`npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
|
||||
'Please try `npm i` again after removing both package-lock.json and node_modules directory.',
|
||||
{
|
||||
cause: loadErrors.reduce((err, cur) => {
|
||||
cur.cause = err
|
||||
return cur
|
||||
}),
|
||||
},
|
||||
)
|
||||
}
|
||||
throw new Error(`Failed to load native binding`)
|
||||
}
|
||||
|
||||
module.exports = nativeBinding
|
||||
module.exports.ApplicationInfo = nativeBinding.ApplicationInfo
|
||||
module.exports.ApplicationListChangedSubscriber = nativeBinding.ApplicationListChangedSubscriber
|
||||
module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStateChangedSubscriber
|
||||
module.exports.ShareableContent = nativeBinding.ShareableContent
|
||||
module.exports.abortRecording = nativeBinding.abortRecording
|
||||
module.exports.decodeAudio = nativeBinding.decodeAudio
|
||||
module.exports.decodeAudioSync = nativeBinding.decodeAudioSync
|
||||
module.exports.startRecording = nativeBinding.startRecording
|
||||
module.exports.stopRecording = nativeBinding.stopRecording
|
||||
module.exports.cancelImportSession = nativeBinding.cancelImportSession
|
||||
module.exports.createImportSession = nativeBinding.createImportSession
|
||||
module.exports.disposeImportSession = nativeBinding.disposeImportSession
|
||||
module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse
|
||||
module.exports.nextImportBatch = nativeBinding.nextImportBatch
|
||||
module.exports.renderMermaidSvg = nativeBinding.renderMermaidSvg
|
||||
module.exports.renderTypstSvg = nativeBinding.renderTypstSvg
|
||||
module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse
|
||||
module.exports.DocStorage = nativeBinding.DocStorage
|
||||
module.exports.DocStoragePool = nativeBinding.DocStoragePool
|
||||
module.exports.SqliteConnection = nativeBinding.SqliteConnection
|
||||
module.exports.ValidationResult = nativeBinding.ValidationResult
|
||||
46
packages/frontend/native/media_capture/Cargo.toml
Normal file
46
packages/frontend/native/media_capture/Cargo.toml
Normal file
@@ -0,0 +1,46 @@
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "affine_media_capture"
|
||||
publish = false
|
||||
version = "0.0.0"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[[bench]]
|
||||
harness = false
|
||||
name = "mix_audio_samples"
|
||||
|
||||
[dependencies]
|
||||
crossbeam-channel = { workspace = true }
|
||||
napi = { workspace = true, features = ["napi4"] }
|
||||
napi-derive = { workspace = true, features = ["type-def"] }
|
||||
ogg = { workspace = true }
|
||||
opus-codec = { git = "https://github.com/toeverything/opus-codec", rev = "c2afef2" }
|
||||
rand = { workspace = true }
|
||||
rubato = { workspace = true }
|
||||
symphonia = { workspace = true, features = ["all", "opt-simd"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
block2 = { workspace = true }
|
||||
core-foundation = { workspace = true, features = ["with-uuid"] }
|
||||
coreaudio-rs = { workspace = true }
|
||||
dispatch2 = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
objc2 = { workspace = true }
|
||||
objc2-foundation = { workspace = true }
|
||||
screencapturekit = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
cpal = { workspace = true }
|
||||
windows = { workspace = true }
|
||||
windows-core = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion2 = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
napi-build = { workspace = true }
|
||||
@@ -0,0 +1,55 @@
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::hint::black_box;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use affine_media_capture::macos::audio_buffer::{mix_audio_samples, mix_audio_samples_scalar};
|
||||
#[cfg(target_os = "macos")]
|
||||
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn generate_test_samples() -> [f32; 1024] {
|
||||
let mut samples = [0.0; 1024];
|
||||
|
||||
// Generate a simple sine wave with some variation
|
||||
for (i, sample) in samples.iter_mut().enumerate() {
|
||||
let t = i as f32 / 1024.0;
|
||||
// Create a complex waveform with multiple frequencies
|
||||
let value = 0.008 * (2.0 * std::f32::consts::PI * t * 5.0).sin()
|
||||
+ 0.004 * (2.0 * std::f32::consts::PI * t * 10.0).cos()
|
||||
+ 0.002 * (2.0 * std::f32::consts::PI * t * 20.0).sin();
|
||||
*sample = value;
|
||||
}
|
||||
|
||||
samples
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn bench_audio_mix(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("audio mix");
|
||||
|
||||
let input = generate_test_samples();
|
||||
let output = generate_test_samples();
|
||||
|
||||
group.bench_function(BenchmarkId::from_parameter("simd"), |b| {
|
||||
b.iter(|| {
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
black_box(mixed);
|
||||
});
|
||||
});
|
||||
|
||||
group.bench_function(BenchmarkId::from_parameter("scalar"), |b| {
|
||||
b.iter(|| {
|
||||
let mut mixed = vec![0.0; 1024];
|
||||
mix_audio_samples_scalar(&input, &output, &mut mixed, 0, input.len());
|
||||
black_box(mixed);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
criterion_group!(benches, bench_audio_mix);
|
||||
#[cfg(target_os = "macos")]
|
||||
criterion_main!(benches);
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
fn main() {}
|
||||
3
packages/frontend/native/media_capture/build.rs
Normal file
3
packages/frontend/native/media_capture/build.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
napi_build::setup();
|
||||
}
|
||||
65
packages/frontend/native/media_capture/src/audio_callback.rs
Normal file
65
packages/frontend/native/media_capture/src/audio_callback.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
|
||||
use crossbeam_channel::{Sender, TrySendError};
|
||||
use napi::{
|
||||
bindgen_prelude::Float32Array,
|
||||
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
|
||||
};
|
||||
|
||||
/// Internal callback abstraction so audio taps can target JS or native
|
||||
/// pipelines.
|
||||
#[derive(Clone)]
|
||||
pub enum AudioCallback {
|
||||
Js(Arc<ThreadsafeFunction<Float32Array, ()>>),
|
||||
Channel {
|
||||
sender: Sender<Vec<f32>>,
|
||||
overflow_count: Arc<AtomicU64>,
|
||||
},
|
||||
}
|
||||
|
||||
impl AudioCallback {
|
||||
pub fn call(&self, samples: Vec<f32>) {
|
||||
match self {
|
||||
Self::Js(func) => {
|
||||
// Non-blocking call into JS; errors are ignored to avoid blocking the
|
||||
// audio thread.
|
||||
let _ = func.call(Ok(samples.into()), ThreadsafeFunctionCallMode::NonBlocking);
|
||||
}
|
||||
Self::Channel { sender, overflow_count } => match sender.try_send(samples) {
|
||||
Ok(()) => {}
|
||||
Err(TrySendError::Full(_)) => {
|
||||
overflow_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(TrySendError::Disconnected(_)) => {}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use crossbeam_channel::bounded;
|
||||
|
||||
use super::AudioCallback;
|
||||
|
||||
#[test]
|
||||
fn channel_overflow_only_increments_the_counter() {
|
||||
let (sender, _rx) = bounded(1);
|
||||
let overflow_count = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
let callback = AudioCallback::Channel {
|
||||
sender: sender.clone(),
|
||||
overflow_count: overflow_count.clone(),
|
||||
};
|
||||
|
||||
sender.send(vec![0.0]).unwrap();
|
||||
callback.call(vec![1.0]);
|
||||
callback.call(vec![2.0]);
|
||||
|
||||
assert_eq!(overflow_count.load(Ordering::Relaxed), 2);
|
||||
}
|
||||
}
|
||||
175
packages/frontend/native/media_capture/src/audio_decoder.rs
Normal file
175
packages/frontend/native/media_capture/src/audio_decoder.rs
Normal file
@@ -0,0 +1,175 @@
|
||||
use std::{io::Cursor, path::Path};
|
||||
|
||||
use napi::{
|
||||
Task,
|
||||
bindgen_prelude::{AbortSignal, AsyncTask, Float32Array, Result, Status, Uint8Array},
|
||||
};
|
||||
use napi_derive::napi;
|
||||
use rubato::{Resampler, SincFixedIn, SincInterpolationParameters, SincInterpolationType};
|
||||
use symphonia::core::{
|
||||
audio::{AudioBuffer, Signal},
|
||||
codecs::DecoderOptions,
|
||||
errors::Error,
|
||||
formats::FormatOptions,
|
||||
io::MediaSourceStream,
|
||||
meta::MetadataOptions,
|
||||
probe::Hint,
|
||||
};
|
||||
|
||||
fn decode<B: AsRef<[u8]> + Send + Sync + 'static>(
|
||||
buf: B,
|
||||
dest_sample_rate: Option<u32>,
|
||||
filename: Option<&str>,
|
||||
) -> std::result::Result<Vec<f32>, Error> {
|
||||
// Create the media source
|
||||
let mss = MediaSourceStream::new(Box::new(Cursor::new(buf)), Default::default());
|
||||
|
||||
// Create a probe hint using the file extension
|
||||
let mut hint = Hint::new();
|
||||
if let Some(ext) = filename.and_then(|filename| Path::new(filename).extension().and_then(|ext| ext.to_str())) {
|
||||
hint.with_extension(ext);
|
||||
}
|
||||
|
||||
let format_opts = FormatOptions {
|
||||
enable_gapless: true,
|
||||
..Default::default()
|
||||
};
|
||||
let metadata_opts = MetadataOptions::default();
|
||||
let decoder_opts = DecoderOptions::default();
|
||||
let probed = symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts)?;
|
||||
|
||||
let mut format = probed.format;
|
||||
|
||||
let track = format
|
||||
.default_track()
|
||||
.ok_or(Error::Unsupported("No default track found"))?;
|
||||
|
||||
let totol_samples = track
|
||||
.codec_params
|
||||
.n_frames
|
||||
.ok_or(Error::Unsupported("No duration found"))?;
|
||||
let sample_rate = track
|
||||
.codec_params
|
||||
.sample_rate
|
||||
.ok_or(Error::Unsupported("No samplerate found"))?;
|
||||
|
||||
let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts)?;
|
||||
|
||||
let mut output: Vec<f32> = Vec::with_capacity(totol_samples as usize);
|
||||
// Decode loop
|
||||
while let Ok(packet) = format.next_packet() {
|
||||
let decoded = decoder.decode(&packet)?;
|
||||
let spec = decoded.spec();
|
||||
let mut audio_buf: AudioBuffer<f32> = AudioBuffer::new(decoded.capacity() as u64, *spec);
|
||||
decoded.convert(&mut audio_buf);
|
||||
|
||||
if spec.channels.count() > 1 {
|
||||
// Mix all channels into mono
|
||||
for i in 0..audio_buf.chan(0).len() {
|
||||
let mut sample_sum = 0.0;
|
||||
for ch in 0..spec.channels.count() {
|
||||
sample_sum += audio_buf.chan(ch)[i];
|
||||
}
|
||||
output.push(sample_sum / spec.channels.count() as f32);
|
||||
}
|
||||
} else {
|
||||
output.extend_from_slice(audio_buf.chan(0));
|
||||
}
|
||||
}
|
||||
|
||||
let Some(dest_sample_rate) = dest_sample_rate else {
|
||||
return Ok(output);
|
||||
};
|
||||
|
||||
if sample_rate != dest_sample_rate {
|
||||
// Calculate parameters for resampling
|
||||
let params = SincInterpolationParameters {
|
||||
sinc_len: 256,
|
||||
f_cutoff: 0.95,
|
||||
interpolation: SincInterpolationType::Linear,
|
||||
oversampling_factor: 256,
|
||||
window: rubato::WindowFunction::BlackmanHarris2,
|
||||
};
|
||||
|
||||
let mut resampler = SincFixedIn::<f32>::new(
|
||||
dest_sample_rate as f64 / sample_rate as f64,
|
||||
2.0,
|
||||
params,
|
||||
output.len(),
|
||||
1,
|
||||
)
|
||||
.map_err(|_| Error::Unsupported("Failed to create resampler"))?;
|
||||
|
||||
let waves_in = vec![output];
|
||||
let mut waves_out = resampler
|
||||
.process(&waves_in, None)
|
||||
.map_err(|_| Error::Unsupported("Failed to run resampler"))?;
|
||||
output = waves_out.pop().ok_or(Error::Unsupported("No resampled output found"))?;
|
||||
}
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
/// Decode audio file into a Float32Array
|
||||
pub fn decode_audio_sync(
|
||||
buf: Uint8Array,
|
||||
dest_sample_rate: Option<u32>,
|
||||
filename: Option<String>,
|
||||
) -> Result<Float32Array> {
|
||||
decode(buf, dest_sample_rate, filename.as_deref())
|
||||
.map(Float32Array::new)
|
||||
.map_err(|e| {
|
||||
napi::Error::new(
|
||||
Status::InvalidArg,
|
||||
format!("Decode audio into Float32Array failed: {e}"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub struct DecodeAudioTask {
|
||||
buf: Uint8Array,
|
||||
dest_sample_rate: Option<u32>,
|
||||
filename: Option<String>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Task for DecodeAudioTask {
|
||||
type Output = Vec<f32>;
|
||||
type JsValue = Float32Array;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
decode(
|
||||
std::mem::replace(&mut self.buf, Uint8Array::new(vec![])),
|
||||
self.dest_sample_rate,
|
||||
self.filename.as_deref(),
|
||||
)
|
||||
.map_err(|e| {
|
||||
napi::Error::new(
|
||||
Status::InvalidArg,
|
||||
format!("Decode audio into Float32Array failed: {e}"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _: napi::Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(Float32Array::new(output))
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn decode_audio(
|
||||
buf: Uint8Array,
|
||||
dest_sample_rate: Option<u32>,
|
||||
filename: Option<String>,
|
||||
signal: Option<AbortSignal>,
|
||||
) -> AsyncTask<DecodeAudioTask> {
|
||||
AsyncTask::with_optional_signal(
|
||||
DecodeAudioTask {
|
||||
buf,
|
||||
dest_sample_rate,
|
||||
filename,
|
||||
},
|
||||
signal,
|
||||
)
|
||||
}
|
||||
13
packages/frontend/native/media_capture/src/lib.rs
Normal file
13
packages/frontend/native/media_capture/src/lib.rs
Normal file
@@ -0,0 +1,13 @@
|
||||
#[cfg(target_os = "macos")]
|
||||
pub mod macos;
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) use macos::*;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub mod windows;
|
||||
#[cfg(target_os = "windows")]
|
||||
pub use windows::*;
|
||||
|
||||
pub mod audio_callback;
|
||||
pub mod audio_decoder;
|
||||
pub mod recording;
|
||||
@@ -0,0 +1,154 @@
|
||||
use napi::bindgen_prelude::{Buffer, Result};
|
||||
use objc2::{
|
||||
Encode, Encoding, msg_send,
|
||||
rc::{Retained, autoreleasepool},
|
||||
runtime::{AnyClass, AnyObject},
|
||||
};
|
||||
use objc2_foundation::NSString;
|
||||
|
||||
#[repr(C)]
|
||||
struct CGSize {
|
||||
width: f64,
|
||||
height: f64,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct CGPoint {
|
||||
x: f64,
|
||||
y: f64,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct CGRect {
|
||||
origin: CGPoint,
|
||||
size: CGSize,
|
||||
}
|
||||
|
||||
unsafe impl Encode for CGSize {
|
||||
const ENCODING: Encoding = Encoding::Struct("CGSize", &[f64::ENCODING, f64::ENCODING]);
|
||||
}
|
||||
|
||||
unsafe impl Encode for CGPoint {
|
||||
const ENCODING: Encoding = Encoding::Struct("CGPoint", &[f64::ENCODING, f64::ENCODING]);
|
||||
}
|
||||
|
||||
unsafe impl Encode for CGRect {
|
||||
const ENCODING: Encoding = Encoding::Struct("CGRect", &[<CGPoint>::ENCODING, <CGSize>::ENCODING]);
|
||||
}
|
||||
|
||||
pub(super) fn application_icon(process_id: i32) -> Result<Buffer> {
|
||||
autoreleasepool(|_| {
|
||||
let running_app_class = match AnyClass::get(c"NSRunningApplication") {
|
||||
Some(class) => class,
|
||||
None => {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
};
|
||||
|
||||
let running_app: *mut AnyObject = unsafe {
|
||||
msg_send![
|
||||
running_app_class,
|
||||
runningApplicationWithProcessIdentifier: process_id
|
||||
]
|
||||
};
|
||||
if running_app.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
unsafe {
|
||||
let icon: *mut AnyObject = msg_send![running_app, icon];
|
||||
if icon.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let nsimage_class = match AnyClass::get(c"NSImage") {
|
||||
Some(class) => class,
|
||||
None => return Ok(Buffer::from(Vec::<u8>::new())),
|
||||
};
|
||||
|
||||
let resized_image: *mut AnyObject = msg_send![nsimage_class, alloc];
|
||||
if resized_image.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let resized_image: *mut AnyObject = msg_send![resized_image, initWithSize: CGSize { width: 64.0, height: 64.0 }];
|
||||
let Some(resized_image) = Retained::from_raw(resized_image) else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let _: () = msg_send![&*resized_image, lockFocus];
|
||||
|
||||
let draw_rect = CGRect {
|
||||
origin: CGPoint { x: 0.0, y: 0.0 },
|
||||
size: CGSize {
|
||||
width: 64.0,
|
||||
height: 64.0,
|
||||
},
|
||||
};
|
||||
|
||||
let from_rect = CGRect {
|
||||
origin: CGPoint { x: 0.0, y: 0.0 },
|
||||
size: CGSize {
|
||||
width: 0.0,
|
||||
height: 0.0,
|
||||
},
|
||||
};
|
||||
|
||||
let _: () = msg_send![icon, drawInRect: draw_rect, fromRect: from_rect, operation: 2u64, fraction: 1.0];
|
||||
let _: () = msg_send![&*resized_image, unlockFocus];
|
||||
|
||||
let tiff_data: *mut AnyObject = msg_send![&*resized_image, TIFFRepresentation];
|
||||
if tiff_data.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let bitmap_class = match AnyClass::get(c"NSBitmapImageRep") {
|
||||
Some(class) => class,
|
||||
None => return Ok(Buffer::from(Vec::<u8>::new())),
|
||||
};
|
||||
|
||||
let bitmap: *mut AnyObject = msg_send![bitmap_class, imageRepWithData: tiff_data];
|
||||
if bitmap.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let dict_class = match AnyClass::get(c"NSMutableDictionary") {
|
||||
Some(class) => class,
|
||||
None => return Ok(Buffer::from(Vec::<u8>::new())),
|
||||
};
|
||||
|
||||
let properties: *mut AnyObject = msg_send![dict_class, dictionary];
|
||||
if properties.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let compression_key = NSString::from_str("NSImageCompressionFactor");
|
||||
let number_class = match AnyClass::get(c"NSNumber") {
|
||||
Some(class) => class,
|
||||
None => return Ok(Buffer::from(Vec::<u8>::new())),
|
||||
};
|
||||
|
||||
let compression_value: *mut AnyObject = msg_send![number_class, numberWithDouble: 0.8];
|
||||
if compression_value.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let _: () = msg_send![properties, setObject: compression_value, forKey: &*compression_key];
|
||||
|
||||
let png_data: *mut AnyObject = msg_send![bitmap, representationUsingType: 4u64, properties: properties]; // 4 = PNG
|
||||
|
||||
if png_data.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let bytes: *const libc::c_void = msg_send![png_data, bytes];
|
||||
let length: usize = msg_send![png_data, length];
|
||||
|
||||
if bytes.is_null() {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
}
|
||||
|
||||
let data = std::slice::from_raw_parts(bytes as *const u8, length).to_vec();
|
||||
Ok(Buffer::from(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
400
packages/frontend/native/media_capture/src/macos/audio_buffer.rs
Normal file
400
packages/frontend/native/media_capture/src/macos/audio_buffer.rs
Normal file
@@ -0,0 +1,400 @@
|
||||
use std::{cmp::Ordering, ffi::c_void};
|
||||
|
||||
use coreaudio::sys::kAudioHardwareBadStreamError;
|
||||
use objc2::{Encode, Encoding, RefEncode};
|
||||
|
||||
use crate::{error::CoreAudioError, utils::process_audio_frame};
|
||||
|
||||
pub const AUDIO_MIX_OUTPUT_WEIGHT: f32 = 0.75;
|
||||
|
||||
/// Mix audio samples using scalar operations (no SIMD)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input_samples` - Samples from the input stream
|
||||
/// * `output_samples` - Samples from the output stream
|
||||
/// * `mixed_samples` - Buffer to store the result (must be pre-allocated)
|
||||
/// * `start_index` - Starting index in the buffers
|
||||
/// * `end_index` - Ending index in the buffers (exclusive)
|
||||
pub fn mix_audio_samples_scalar(
|
||||
input_samples: &[f32],
|
||||
output_samples: &[f32],
|
||||
mixed_samples: &mut [f32],
|
||||
start_index: usize,
|
||||
end_index: usize,
|
||||
) {
|
||||
// performance downgraded 4x if apply suggestion from this lint rule
|
||||
#[allow(clippy::needless_range_loop)]
|
||||
for sample_index in start_index..end_index {
|
||||
let sample_in = input_samples.get(sample_index).unwrap_or(&0.0);
|
||||
let sample_out = output_samples.get(sample_index).unwrap_or(&0.0);
|
||||
mixed_samples[sample_index] = sample_in + sample_out * AUDIO_MIX_OUTPUT_WEIGHT;
|
||||
}
|
||||
}
|
||||
|
||||
/// Mix audio samples from input and output streams with specified weights
|
||||
/// Uses NEON SIMD acceleration on supported platforms
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input_samples` - Samples from the input stream
|
||||
/// * `output_samples` - Samples from the output stream
|
||||
///
|
||||
/// # Returns
|
||||
/// A vector of mixed audio samples
|
||||
pub fn mix_audio_samples(input_samples: &[f32], output_samples: &[f32]) -> Vec<f32> {
|
||||
let mixed_samples_length = input_samples.len();
|
||||
let mut mixed_samples = vec![0.0; mixed_samples_length];
|
||||
|
||||
// For very small arrays, use scalar implementation
|
||||
if mixed_samples_length < 16 {
|
||||
mix_audio_samples_scalar(
|
||||
input_samples,
|
||||
output_samples,
|
||||
&mut mixed_samples,
|
||||
0,
|
||||
mixed_samples_length,
|
||||
);
|
||||
return mixed_samples;
|
||||
}
|
||||
|
||||
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
|
||||
unsafe {
|
||||
use std::arch::aarch64::{vdupq_n_f32, vld1q_f32, vmlaq_f32, vst1q_f32};
|
||||
|
||||
let output_weight_vec = vdupq_n_f32(AUDIO_MIX_OUTPUT_WEIGHT);
|
||||
// Process the common length where both arrays have data
|
||||
let common_length = input_samples.len();
|
||||
|
||||
// Main SIMD loop - process each block of 4 samples
|
||||
let input_ptr = input_samples.as_ptr();
|
||||
let output_ptr = output_samples.as_ptr();
|
||||
let result_ptr = mixed_samples.as_mut_ptr();
|
||||
|
||||
let mut offset: usize = 0;
|
||||
let mut remaining_offset: Option<usize> = None;
|
||||
|
||||
// Process 16 samples at a time (4 SIMD vectors)
|
||||
while offset < common_length {
|
||||
// Load 4 vectors of 4 floats each
|
||||
let in_vec1 = vld1q_f32(input_ptr.add(offset));
|
||||
let out_vec1 = vld1q_f32(output_ptr.add(offset));
|
||||
let in_vec2 = vld1q_f32(input_ptr.add(offset + 4));
|
||||
let out_vec2 = vld1q_f32(output_ptr.add(offset + 4));
|
||||
let in_vec3 = vld1q_f32(input_ptr.add(offset + 8));
|
||||
let out_vec3 = vld1q_f32(output_ptr.add(offset + 8));
|
||||
let in_vec4 = vld1q_f32(input_ptr.add(offset + 12));
|
||||
let out_vec4 = vld1q_f32(output_ptr.add(offset + 12));
|
||||
|
||||
// Using fused multiply-add: (a * b) + c in one operation
|
||||
// First multiply input by weight
|
||||
let result1 = vmlaq_f32(in_vec1, out_vec1, output_weight_vec);
|
||||
let result2 = vmlaq_f32(in_vec2, out_vec2, output_weight_vec);
|
||||
let result3 = vmlaq_f32(in_vec3, out_vec3, output_weight_vec);
|
||||
|
||||
let result4 = vmlaq_f32(in_vec4, out_vec4, output_weight_vec);
|
||||
|
||||
// Store results
|
||||
vst1q_f32(result_ptr.add(offset), result1);
|
||||
vst1q_f32(result_ptr.add(offset + 4), result2);
|
||||
vst1q_f32(result_ptr.add(offset + 8), result3);
|
||||
vst1q_f32(result_ptr.add(offset + 12), result4);
|
||||
offset += 16;
|
||||
// accept clippy lint suggestion would downgrade the performance by 15%
|
||||
#[allow(clippy::comparison_chain)]
|
||||
// fast path for aligned length
|
||||
if offset == common_length {
|
||||
break;
|
||||
} else if offset > common_length {
|
||||
remaining_offset = Some(offset - 16);
|
||||
} else {
|
||||
let remaining = common_length - offset;
|
||||
if remaining < 16 {
|
||||
remaining_offset = Some(offset);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(remaining_offset) = remaining_offset {
|
||||
mix_audio_samples_scalar(
|
||||
input_samples,
|
||||
output_samples,
|
||||
&mut mixed_samples,
|
||||
remaining_offset,
|
||||
common_length,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))]
|
||||
{
|
||||
// Fallback for non-ARM architectures
|
||||
mix_audio_samples_scalar(
|
||||
input_samples,
|
||||
output_samples,
|
||||
&mut mixed_samples,
|
||||
0,
|
||||
mixed_samples_length,
|
||||
);
|
||||
}
|
||||
|
||||
mixed_samples
|
||||
}
|
||||
|
||||
/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiobuffer?language=objc)
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
#[allow(non_snake_case)]
|
||||
pub struct AudioBuffer {
|
||||
pub mNumberChannels: u32,
|
||||
pub mDataByteSize: u32,
|
||||
pub mData: *mut c_void,
|
||||
}
|
||||
|
||||
unsafe impl Encode for AudioBuffer {
|
||||
const ENCODING: Encoding = Encoding::Struct(
|
||||
"AudioBuffer",
|
||||
&[<u32>::ENCODING, <u32>::ENCODING, <*mut c_void>::ENCODING],
|
||||
);
|
||||
}
|
||||
|
||||
unsafe impl RefEncode for AudioBuffer {
|
||||
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
#[allow(non_snake_case)]
|
||||
pub struct AudioBufferList {
|
||||
pub mNumberBuffers: u32,
|
||||
pub mBuffers: [AudioBuffer; 2],
|
||||
}
|
||||
|
||||
unsafe impl Encode for AudioBufferList {
|
||||
const ENCODING: Encoding = Encoding::Struct("AudioBufferList", &[<u32>::ENCODING, <[AudioBuffer; 1]>::ENCODING]);
|
||||
}
|
||||
|
||||
unsafe impl RefEncode for AudioBufferList {
|
||||
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
|
||||
}
|
||||
|
||||
/// Represent the audio buffer contains input and output PCM buffers
|
||||
#[repr(transparent)]
|
||||
pub struct InputAndOutputAudioBufferList(pub AudioBufferList);
|
||||
|
||||
impl InputAndOutputAudioBufferList {
|
||||
/// # Safety
|
||||
///
|
||||
/// The caller must ensure that the input data is a valid AudioBufferList
|
||||
pub unsafe fn from_raw(in_input_data: *mut c_void) -> Result<Self, i32> {
|
||||
let buffer_list: AudioBufferList = unsafe { *in_input_data.cast() };
|
||||
if buffer_list.mNumberBuffers != 2 {
|
||||
return Err(kAudioHardwareBadStreamError as i32);
|
||||
}
|
||||
Ok(Self(buffer_list))
|
||||
}
|
||||
|
||||
pub fn mix_input_and_output(
|
||||
&self,
|
||||
target_sample_rate: f64,
|
||||
input_sample_rate: f64,
|
||||
output_sample_rate: f64,
|
||||
) -> Result<Vec<f32>, CoreAudioError> {
|
||||
let mut mixed_samples = Vec::new();
|
||||
|
||||
// Directly access buffers from the list
|
||||
let [input_buffer, output_buffer] = self.0.mBuffers;
|
||||
|
||||
if let Some(processed_input) = process_audio_frame(
|
||||
input_buffer.mData,
|
||||
input_buffer.mDataByteSize,
|
||||
input_buffer.mNumberChannels,
|
||||
input_sample_rate,
|
||||
target_sample_rate,
|
||||
) {
|
||||
mixed_samples = processed_input;
|
||||
}
|
||||
|
||||
if let Some(processed_output) = process_audio_frame(
|
||||
output_buffer.mData,
|
||||
output_buffer.mDataByteSize,
|
||||
output_buffer.mNumberChannels,
|
||||
output_sample_rate,
|
||||
target_sample_rate,
|
||||
) {
|
||||
if mixed_samples.is_empty() {
|
||||
mixed_samples = processed_output;
|
||||
} else {
|
||||
let len1 = mixed_samples.len();
|
||||
let len2 = processed_output.len();
|
||||
match len1.cmp(&len2) {
|
||||
Ordering::Less => {
|
||||
mixed_samples.resize(len2, 0.0);
|
||||
}
|
||||
Ordering::Greater => {
|
||||
let mut padded_output = processed_output;
|
||||
padded_output.resize(len1, 0.0);
|
||||
for (sample1, sample2) in mixed_samples.iter_mut().zip(padded_output.iter()) {
|
||||
*sample1 = (*sample1 + *sample2) / 2.0;
|
||||
}
|
||||
return Ok(mixed_samples);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
for (sample1, sample2) in mixed_samples.iter_mut().zip(processed_output.iter()) {
|
||||
*sample1 = (*sample1 + *sample2) / 2.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(mixed_samples)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_mix_audio_samples_empty() {
|
||||
let input: Vec<f32> = vec![];
|
||||
let output: Vec<f32> = vec![];
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
assert_eq!(mixed.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mix_audio_samples_equal_length() {
|
||||
let input = vec![0.1, 0.2, 0.3, 0.4, 0.5];
|
||||
let output = vec![0.5, 0.4, 0.3, 0.2, 0.1];
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
|
||||
assert_eq!(mixed.len(), 5);
|
||||
|
||||
// Verify calculations: (input + output * 0.75)
|
||||
let expected = [
|
||||
(0.1 + 0.5 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.2 + 0.4 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.3 + 0.3 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.4 + 0.2 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.5 + 0.1 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
];
|
||||
|
||||
for i in 0..mixed.len() {
|
||||
assert!(
|
||||
(mixed[i] - expected[i]).abs() < 1e-6,
|
||||
"Mismatch at index {}: expected {}, got {}",
|
||||
i,
|
||||
expected[i],
|
||||
mixed[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mix_audio_samples_input_longer() {
|
||||
let input = vec![0.1, 0.2, 0.3, 0.4, 0.5];
|
||||
let output = vec![0.5, 0.4, 0.3];
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
|
||||
assert_eq!(mixed.len(), 5);
|
||||
|
||||
// Verify calculations
|
||||
let expected = [
|
||||
(0.1 + 0.5 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.2 + 0.4 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.3 + 0.3 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.4 + 0.0 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.5 + 0.0 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
];
|
||||
|
||||
for i in 0..mixed.len() {
|
||||
assert!(
|
||||
(mixed[i] - expected[i]).abs() < 1e-6,
|
||||
"Mismatch at index {}: expected {}, got {}",
|
||||
i,
|
||||
expected[i],
|
||||
mixed[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mix_audio_samples_custom_weights() {
|
||||
// Note: We're using the constant weights so we can't really test custom
|
||||
// values directly
|
||||
let input = vec![0.1, 0.2, 0.3];
|
||||
let output = vec![0.5, 0.4, 0.3];
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
|
||||
// Calculate expected values based on the constants
|
||||
let expected = [
|
||||
(0.1 + 0.5 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.2 + 0.4 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.3 + 0.3 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
];
|
||||
|
||||
for i in 0..mixed.len() {
|
||||
assert!(
|
||||
(mixed[i] - expected[i]).abs() < 1e-6,
|
||||
"Mismatch at index {}: expected {}, got {}",
|
||||
i,
|
||||
expected[i],
|
||||
mixed[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
|
||||
#[test]
|
||||
fn test_simd_implementation_used() {
|
||||
const BUFFER_SIZES: [usize; 4] = [100, 127, 128, 512];
|
||||
for size in BUFFER_SIZES {
|
||||
// Create arrays large enough to trigger SIMD path
|
||||
let input: Vec<f32> = (0..size).map(|i| i as f32 * 0.01).collect();
|
||||
let output: Vec<f32> = (0..size).map(|i| (size - i) as f32 * 0.01).collect();
|
||||
|
||||
// Mix with standard weights
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
|
||||
// Compute the same mix using scalar implementation for comparison
|
||||
let mut expected = vec![0.0; input.len()];
|
||||
mix_audio_samples_scalar(&input, &output, &mut expected, 0, input.len());
|
||||
|
||||
// Verify results match between SIMD and scalar implementations
|
||||
for i in 0..mixed.len() {
|
||||
assert!(
|
||||
(mixed[i] - expected[i]).abs() < 1e-6,
|
||||
"SIMD and scalar implementations should produce identical results at index {}",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_vector_uses_scalar() {
|
||||
// Create small arrays that should use scalar path even with SIMD available
|
||||
let input = vec![0.1, 0.2, 0.3];
|
||||
let output = vec![0.5, 0.4, 0.3];
|
||||
|
||||
// Mix with standard weights
|
||||
let mixed = mix_audio_samples(&input, &output);
|
||||
|
||||
// Calculate expected values manually
|
||||
let expected = [
|
||||
(0.1 + 0.5 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.2 + 0.4 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
(0.3 + 0.3 * AUDIO_MIX_OUTPUT_WEIGHT),
|
||||
];
|
||||
|
||||
// Verify results
|
||||
for i in 0..mixed.len() {
|
||||
assert!(
|
||||
(mixed[i] - expected[i]).abs() < 1e-6,
|
||||
"Small vector mixing should be correct at index {}",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
use std::fmt::Display;
|
||||
|
||||
use coreaudio::sys::{AudioObjectID, kAudioTapPropertyFormat};
|
||||
use objc2::{Encode, Encoding, RefEncode};
|
||||
|
||||
use crate::{error::CoreAudioError, utils::get_global_main_property};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u32)]
|
||||
pub enum AudioFormatID {
|
||||
LinearPcm = 0x6C70636D, // 'lpcm'
|
||||
Ac3 = 0x61632D33, // 'ac-3'
|
||||
Ac360958 = 0x63616333, // 'cac3'
|
||||
AppleIma4 = 0x696D6134, // 'ima4'
|
||||
Mpeg4Aac = 0x61616320, // 'aac '
|
||||
Mpeg4Celp = 0x63656C70, // 'celp'
|
||||
Mpeg4Hvxc = 0x68767863, // 'hvxc'
|
||||
Mpeg4TwinVq = 0x74777671, // 'twvq'
|
||||
Mace3 = 0x4D414333, // 'MAC3'
|
||||
Mace6 = 0x4D414336, // 'MAC6'
|
||||
ULaw = 0x756C6177, // 'ulaw'
|
||||
ALaw = 0x616C6177, // 'alaw'
|
||||
QDesign = 0x51444D43, // 'QDMC'
|
||||
QDesign2 = 0x51444D32, // 'QDM2'
|
||||
Qualcomm = 0x51636C70, // 'Qclp'
|
||||
MpegLayer1 = 0x2E6D7031, // '.mp1'
|
||||
MpegLayer2 = 0x2E6D7032, // '.mp2'
|
||||
MpegLayer3 = 0x2E6D7033, // '.mp3'
|
||||
TimeCode = 0x74696D65, // 'time'
|
||||
MidiStream = 0x6D696469, // 'midi'
|
||||
ParameterValueStream = 0x61707673, // 'apvs'
|
||||
AppleLossless = 0x616C6163, // 'alac'
|
||||
Mpeg4AacHe = 0x61616368, // 'aach'
|
||||
Mpeg4AacLd = 0x6161636C, // 'aacl'
|
||||
Mpeg4AacEld = 0x61616365, // 'aace'
|
||||
Mpeg4AacEldSbr = 0x61616366, // 'aacf'
|
||||
Mpeg4AacEldV2 = 0x61616367, // 'aacg'
|
||||
Mpeg4AacHeV2 = 0x61616370, // 'aacp'
|
||||
Mpeg4AacSpatial = 0x61616373, // 'aacs'
|
||||
MpegdUsac = 0x75736163, // 'usac'
|
||||
Amr = 0x73616D72, // 'samr'
|
||||
AmrWb = 0x73617762, // 'sawb'
|
||||
Audible = 0x41554442, // 'AUDB'
|
||||
ILbc = 0x696C6263, // 'ilbc'
|
||||
DviIntelIma = 0x6D730011,
|
||||
MicrosoftGsm = 0x6D730031,
|
||||
Aes3 = 0x61657333, // 'aes3'
|
||||
EnhancedAc3 = 0x65632D33, // 'ec-3'
|
||||
Flac = 0x666C6163, // 'flac'
|
||||
Opus = 0x6F707573, // 'opus'
|
||||
Apac = 0x61706163, // 'apac'
|
||||
Unknown = 0x00000000,
|
||||
}
|
||||
|
||||
impl From<u32> for AudioFormatID {
|
||||
fn from(value: u32) -> Self {
|
||||
match value {
|
||||
0x6C70636D => Self::LinearPcm,
|
||||
0x61632D33 => Self::Ac3,
|
||||
0x63616333 => Self::Ac360958,
|
||||
0x696D6134 => Self::AppleIma4,
|
||||
0x61616320 => Self::Mpeg4Aac,
|
||||
0x63656C70 => Self::Mpeg4Celp,
|
||||
0x68767863 => Self::Mpeg4Hvxc,
|
||||
0x74777671 => Self::Mpeg4TwinVq,
|
||||
0x4D414333 => Self::Mace3,
|
||||
0x4D414336 => Self::Mace6,
|
||||
0x756C6177 => Self::ULaw,
|
||||
0x616C6177 => Self::ALaw,
|
||||
0x51444D43 => Self::QDesign,
|
||||
0x51444D32 => Self::QDesign2,
|
||||
0x51636C70 => Self::Qualcomm,
|
||||
0x2E6D7031 => Self::MpegLayer1,
|
||||
0x2E6D7032 => Self::MpegLayer2,
|
||||
0x2E6D7033 => Self::MpegLayer3,
|
||||
0x74696D65 => Self::TimeCode,
|
||||
0x6D696469 => Self::MidiStream,
|
||||
0x61707673 => Self::ParameterValueStream,
|
||||
0x616C6163 => Self::AppleLossless,
|
||||
0x61616368 => Self::Mpeg4AacHe,
|
||||
0x6161636C => Self::Mpeg4AacLd,
|
||||
0x61616365 => Self::Mpeg4AacEld,
|
||||
0x61616366 => Self::Mpeg4AacEldSbr,
|
||||
0x61616367 => Self::Mpeg4AacEldV2,
|
||||
0x61616370 => Self::Mpeg4AacHeV2,
|
||||
0x61616373 => Self::Mpeg4AacSpatial,
|
||||
0x75736163 => Self::MpegdUsac,
|
||||
0x73616D72 => Self::Amr,
|
||||
0x73617762 => Self::AmrWb,
|
||||
0x41554442 => Self::Audible,
|
||||
0x696C6263 => Self::ILbc,
|
||||
0x6D730011 => Self::DviIntelIma,
|
||||
0x6D730031 => Self::MicrosoftGsm,
|
||||
0x61657333 => Self::Aes3,
|
||||
0x65632D33 => Self::EnhancedAc3,
|
||||
0x666C6163 => Self::Flac,
|
||||
0x6F707573 => Self::Opus,
|
||||
0x61706163 => Self::Apac,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AudioFormatFlags(pub u32);
|
||||
|
||||
#[allow(unused)]
|
||||
impl AudioFormatFlags {
|
||||
pub const IS_FLOAT: u32 = 1 << 0;
|
||||
pub const IS_BIG_ENDIAN: u32 = 1 << 1;
|
||||
pub const IS_SIGNED_INTEGER: u32 = 1 << 2;
|
||||
pub const IS_PACKED: u32 = 1 << 3;
|
||||
pub const IS_ALIGNED_HIGH: u32 = 1 << 4;
|
||||
pub const IS_NON_INTERLEAVED: u32 = 1 << 5;
|
||||
pub const IS_NON_MIXABLE: u32 = 1 << 6;
|
||||
pub const ARE_ALL_CLEAR: u32 = 0x80000000;
|
||||
|
||||
pub const LINEAR_PCM_IS_FLOAT: u32 = Self::IS_FLOAT;
|
||||
pub const LINEAR_PCM_IS_BIG_ENDIAN: u32 = Self::IS_BIG_ENDIAN;
|
||||
pub const LINEAR_PCM_IS_SIGNED_INTEGER: u32 = Self::IS_SIGNED_INTEGER;
|
||||
pub const LINEAR_PCM_IS_PACKED: u32 = Self::IS_PACKED;
|
||||
pub const LINEAR_PCM_IS_ALIGNED_HIGH: u32 = Self::IS_ALIGNED_HIGH;
|
||||
pub const LINEAR_PCM_IS_NON_INTERLEAVED: u32 = Self::IS_NON_INTERLEAVED;
|
||||
pub const LINEAR_PCM_IS_NON_MIXABLE: u32 = Self::IS_NON_MIXABLE;
|
||||
pub const LINEAR_PCM_SAMPLE_FRACTION_SHIFT: u32 = 7;
|
||||
pub const LINEAR_PCM_SAMPLE_FRACTION_MASK: u32 = 0x3F << Self::LINEAR_PCM_SAMPLE_FRACTION_SHIFT;
|
||||
pub const LINEAR_PCM_ARE_ALL_CLEAR: u32 = Self::ARE_ALL_CLEAR;
|
||||
|
||||
pub const APPLE_LOSSLESS_FORMAT_FLAG_16_BIT_SOURCE_DATA: u32 = 1;
|
||||
pub const APPLE_LOSSLESS_FORMAT_FLAG_20_BIT_SOURCE_DATA: u32 = 2;
|
||||
pub const APPLE_LOSSLESS_FORMAT_FLAG_24_BIT_SOURCE_DATA: u32 = 3;
|
||||
pub const APPLE_LOSSLESS_FORMAT_FLAG_32_BIT_SOURCE_DATA: u32 = 4;
|
||||
}
|
||||
|
||||
impl std::fmt::Display for AudioFormatFlags {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut flags = Vec::new();
|
||||
|
||||
if self.0 & Self::IS_FLOAT != 0 {
|
||||
flags.push("FLOAT");
|
||||
}
|
||||
if self.0 & Self::IS_BIG_ENDIAN != 0 {
|
||||
flags.push("BIG_ENDIAN");
|
||||
}
|
||||
if self.0 & Self::IS_SIGNED_INTEGER != 0 {
|
||||
flags.push("SIGNED_INTEGER");
|
||||
}
|
||||
if self.0 & Self::IS_PACKED != 0 {
|
||||
flags.push("PACKED");
|
||||
}
|
||||
if self.0 & Self::IS_ALIGNED_HIGH != 0 {
|
||||
flags.push("ALIGNED_HIGH");
|
||||
}
|
||||
if self.0 & Self::IS_NON_INTERLEAVED != 0 {
|
||||
flags.push("NON_INTERLEAVED");
|
||||
}
|
||||
if self.0 & Self::IS_NON_MIXABLE != 0 {
|
||||
flags.push("NON_MIXABLE");
|
||||
}
|
||||
if self.0 & Self::ARE_ALL_CLEAR != 0 {
|
||||
flags.push("ALL_CLEAR");
|
||||
}
|
||||
|
||||
if flags.is_empty() {
|
||||
write!(f, "NONE")
|
||||
} else {
|
||||
write!(f, "{}", flags.join(" | "))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AudioFormatFlags {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "AudioFormatFlags({self})")
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u32> for AudioFormatFlags {
|
||||
fn from(value: u32) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
/// [Apple's documentation](https://developer.apple.com/documentation/coreaudiotypes/audiostreambasicdescription?language=objc)
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
#[allow(non_snake_case)]
|
||||
pub struct AudioStreamBasicDescription {
|
||||
pub mSampleRate: f64,
|
||||
pub mFormatID: u32,
|
||||
pub mFormatFlags: u32,
|
||||
pub mBytesPerPacket: u32,
|
||||
pub mFramesPerPacket: u32,
|
||||
pub mBytesPerFrame: u32,
|
||||
pub mChannelsPerFrame: u32,
|
||||
pub mBitsPerChannel: u32,
|
||||
pub mReserved: u32,
|
||||
}
|
||||
|
||||
unsafe impl Encode for AudioStreamBasicDescription {
|
||||
const ENCODING: Encoding = Encoding::Struct(
|
||||
"AudioStreamBasicDescription",
|
||||
&[
|
||||
<f64>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
<u32>::ENCODING,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
unsafe impl RefEncode for AudioStreamBasicDescription {
|
||||
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[repr(transparent)]
|
||||
pub struct AudioStreamDescription(pub(crate) AudioStreamBasicDescription);
|
||||
|
||||
impl Display for AudioStreamDescription {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"AudioStreamBasicDescription {{ mSampleRate: {}, mFormatID: {:?}, mFormatFlags: {}, mBytesPerPacket: {}, \
|
||||
mFramesPerPacket: {}, mBytesPerFrame: {}, mChannelsPerFrame: {}, mBitsPerChannel: {}, mReserved: {} }}",
|
||||
self.0.mSampleRate,
|
||||
AudioFormatID::from(self.0.mFormatID),
|
||||
AudioFormatFlags(self.0.mFormatFlags),
|
||||
self.0.mBytesPerPacket,
|
||||
self.0.mFramesPerPacket,
|
||||
self.0.mBytesPerFrame,
|
||||
self.0.mChannelsPerFrame,
|
||||
self.0.mBitsPerChannel,
|
||||
self.0.mReserved
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn read_audio_stream_basic_description(
|
||||
tap_id: AudioObjectID,
|
||||
) -> std::result::Result<AudioStreamDescription, CoreAudioError> {
|
||||
let mut data = AudioStreamBasicDescription {
|
||||
mSampleRate: 0.0,
|
||||
mFormatID: 0,
|
||||
mFormatFlags: 0,
|
||||
mBytesPerPacket: 0,
|
||||
mFramesPerPacket: 0,
|
||||
mBytesPerFrame: 0,
|
||||
mChannelsPerFrame: 0,
|
||||
mBitsPerChannel: 0,
|
||||
mReserved: 0,
|
||||
};
|
||||
get_global_main_property(tap_id, kAudioTapPropertyFormat, &mut data)?;
|
||||
|
||||
Ok(AudioStreamDescription(data))
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
use std::ptr;
|
||||
|
||||
use objc2::{
|
||||
AllocAnyThread, msg_send,
|
||||
runtime::{AnyClass, AnyObject},
|
||||
};
|
||||
use objc2_foundation::{NSDictionary, NSError, NSNumber, NSString, NSUInteger, NSURL};
|
||||
|
||||
use crate::{av_audio_format::AVAudioFormat, av_audio_pcm_buffer::AVAudioPCMBuffer, error::CoreAudioError};
|
||||
|
||||
#[allow(unused)]
|
||||
pub(crate) struct AVAudioFile {
|
||||
inner: *mut AnyObject,
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
impl AVAudioFile {
|
||||
pub(crate) fn new(url: &str, format: &AVAudioFormat) -> Result<Self, CoreAudioError> {
|
||||
let cls = AnyClass::get(c"AVAudioFile").ok_or(CoreAudioError::AVAudioFileClassNotFound)?;
|
||||
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::AllocAVAudioFileFailed);
|
||||
}
|
||||
let url: &NSURL = &*unsafe { NSURL::fileURLWithPath(&NSString::from_str(url)) };
|
||||
let settings = &*NSDictionary::from_retained_objects(
|
||||
&[
|
||||
&*NSString::from_str("AVFormatIDKey"),
|
||||
&*NSString::from_str("AVSampleRateKey"),
|
||||
&*NSString::from_str("AVNumberOfChannelsKey"),
|
||||
],
|
||||
&[
|
||||
NSNumber::initWithUnsignedInt(NSNumber::alloc(), format.audio_stream_basic_description.0.mFormatID),
|
||||
NSNumber::initWithDouble(NSNumber::alloc(), format.get_sample_rate()),
|
||||
NSNumber::initWithUnsignedInt(NSNumber::alloc(), format.get_channel_count()),
|
||||
],
|
||||
);
|
||||
let is_interleaved = format.is_interleaved();
|
||||
let mut error: *mut NSError = ptr::null_mut();
|
||||
let common_format: NSUInteger = 1;
|
||||
let obj: *mut AnyObject = unsafe {
|
||||
msg_send![
|
||||
obj,
|
||||
initForWriting: url,
|
||||
settings: settings,
|
||||
commonFormat: common_format,
|
||||
interleaved: is_interleaved,
|
||||
error: &mut error
|
||||
]
|
||||
};
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::InitAVAudioFileFailed);
|
||||
}
|
||||
Ok(Self { inner: obj })
|
||||
}
|
||||
|
||||
pub(crate) fn write(&self, buffer: AVAudioPCMBuffer) -> Result<(), CoreAudioError> {
|
||||
let mut error: *mut NSError = ptr::null_mut();
|
||||
let success: bool = unsafe { msg_send![self.inner, writeFromBuffer: buffer.inner, error: &mut error] };
|
||||
if !success {
|
||||
return Err(CoreAudioError::WriteAVAudioFileFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
use objc2::{
|
||||
Encode, Encoding, RefEncode, msg_send,
|
||||
runtime::{AnyClass, AnyObject},
|
||||
};
|
||||
|
||||
use crate::{audio_stream_basic_desc::AudioStreamDescription, error::CoreAudioError};
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(unused)]
|
||||
pub(crate) struct AVAudioFormat {
|
||||
pub(crate) inner: AVAudioFormatRef,
|
||||
pub(crate) audio_stream_basic_description: AudioStreamDescription,
|
||||
}
|
||||
|
||||
#[repr(transparent)]
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct AVAudioFormatRef(pub(crate) *mut AnyObject);
|
||||
|
||||
unsafe impl Encode for AVAudioFormatRef {
|
||||
const ENCODING: Encoding = Encoding::Struct(
|
||||
"AVAudioFormat",
|
||||
&[
|
||||
Encoding::Double,
|
||||
Encoding::UInt,
|
||||
Encoding::Pointer(&Encoding::Struct(
|
||||
"AVAudioChannelLayout",
|
||||
&[
|
||||
Encoding::UInt,
|
||||
Encoding::UInt,
|
||||
Encoding::Pointer(&Encoding::Struct(
|
||||
"AudioChannelLayout",
|
||||
&[
|
||||
Encoding::UInt,
|
||||
Encoding::UInt,
|
||||
Encoding::Array(
|
||||
1,
|
||||
&Encoding::Struct(
|
||||
"AudioChannelDescription",
|
||||
&[Encoding::UInt, Encoding::UInt, Encoding::Array(3, &Encoding::Float)],
|
||||
),
|
||||
),
|
||||
Encoding::UInt,
|
||||
Encoding::UInt,
|
||||
],
|
||||
)),
|
||||
Encoding::UInt,
|
||||
],
|
||||
)),
|
||||
Encoding::Pointer(&Encoding::Object),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
unsafe impl RefEncode for AVAudioFormatRef {
|
||||
const ENCODING_REF: Encoding = Encoding::Pointer(&Self::ENCODING);
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
impl AVAudioFormat {
|
||||
pub fn new(audio_stream_basic_description: AudioStreamDescription) -> Result<Self, CoreAudioError> {
|
||||
let cls = AnyClass::get(c"AVAudioFormat").ok_or(CoreAudioError::AVAudioFormatClassNotFound)?;
|
||||
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::AllocAVAudioFormatFailed);
|
||||
}
|
||||
let obj: *mut AnyObject = unsafe { msg_send![obj, initWithStreamDescription: &audio_stream_basic_description.0] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::InitAVAudioFormatFailed);
|
||||
}
|
||||
Ok(Self {
|
||||
inner: AVAudioFormatRef(obj),
|
||||
audio_stream_basic_description,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn get_sample_rate(&self) -> f64 {
|
||||
unsafe { msg_send![self.inner.0, sampleRate] }
|
||||
}
|
||||
|
||||
pub(crate) fn get_channel_count(&self) -> u32 {
|
||||
unsafe { msg_send![self.inner.0, channelCount] }
|
||||
}
|
||||
|
||||
pub(crate) fn is_interleaved(&self) -> bool {
|
||||
unsafe { msg_send![self.inner.0, isInterleaved] }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
use block2::RcBlock;
|
||||
use objc2::{
|
||||
msg_send,
|
||||
runtime::{AnyClass, AnyObject},
|
||||
};
|
||||
|
||||
use crate::{audio_buffer::AudioBufferList, av_audio_format::AVAudioFormat, error::CoreAudioError};
|
||||
|
||||
#[allow(unused)]
|
||||
pub(crate) struct AVAudioPCMBuffer {
|
||||
pub(crate) inner: *mut AnyObject,
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
impl AVAudioPCMBuffer {
|
||||
pub(crate) fn new(audio_format: &AVAudioFormat, buffer_list: *const AudioBufferList) -> Result<Self, CoreAudioError> {
|
||||
let cls = AnyClass::get(c"AVAudioPCMBuffer").ok_or(CoreAudioError::AVAudioPCMBufferClassNotFound)?;
|
||||
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::AllocAVAudioPCMBufferFailed);
|
||||
}
|
||||
let deallocator = RcBlock::new(|_buffer_list: *const AudioBufferList| {});
|
||||
let obj: *mut AnyObject = unsafe {
|
||||
msg_send![obj, initWithPCMFormat: audio_format.inner.0, bufferListNoCopy: buffer_list, deallocator: &*deallocator]
|
||||
};
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::InitAVAudioPCMBufferFailed);
|
||||
}
|
||||
Ok(Self { inner: obj })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
use core_foundation::{
|
||||
base::{FromVoid, ItemRef},
|
||||
string::CFString,
|
||||
};
|
||||
use coreaudio::sys::AudioObjectID;
|
||||
use objc2::{
|
||||
AllocAnyThread, msg_send,
|
||||
runtime::{AnyClass, AnyObject},
|
||||
};
|
||||
use objc2_foundation::{NSArray, NSNumber, NSString, NSUUID};
|
||||
|
||||
use crate::error::CoreAudioError;
|
||||
|
||||
pub(crate) struct CATapDescription {
|
||||
pub(crate) inner: *mut AnyObject,
|
||||
}
|
||||
|
||||
impl CATapDescription {
|
||||
pub fn init_stereo_mixdown_of_processes(process: AudioObjectID) -> std::result::Result<Self, CoreAudioError> {
|
||||
let cls = AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?;
|
||||
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::AllocCATapDescriptionFailed);
|
||||
}
|
||||
let processes_array = NSArray::from_retained_slice(&[NSNumber::initWithUnsignedInt(NSNumber::alloc(), process)]);
|
||||
let obj: *mut AnyObject = unsafe { msg_send![obj, initStereoMixdownOfProcesses: &*processes_array] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::InitStereoMixdownOfProcessesFailed);
|
||||
}
|
||||
|
||||
unsafe {
|
||||
let _: () = msg_send![obj, setMuteBehavior: 0_i64];
|
||||
}
|
||||
|
||||
Ok(Self { inner: obj })
|
||||
}
|
||||
|
||||
pub fn init_stereo_global_tap_but_exclude_processes(
|
||||
processes: &[AudioObjectID],
|
||||
) -> std::result::Result<Self, CoreAudioError> {
|
||||
let cls = AnyClass::get(c"CATapDescription").ok_or(CoreAudioError::CATapDescriptionClassNotFound)?;
|
||||
let obj: *mut AnyObject = unsafe { msg_send![cls, alloc] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::AllocCATapDescriptionFailed);
|
||||
}
|
||||
let processes_array = NSArray::from_retained_slice(
|
||||
processes
|
||||
.iter()
|
||||
.map(|p| NSNumber::initWithUnsignedInt(NSNumber::alloc(), *p))
|
||||
.collect::<Vec<_>>()
|
||||
.as_slice(),
|
||||
);
|
||||
let obj: *mut AnyObject = unsafe { msg_send![obj, initStereoGlobalTapButExcludeProcesses: &*processes_array] };
|
||||
if obj.is_null() {
|
||||
return Err(CoreAudioError::InitStereoGlobalTapButExcludeProcessesFailed);
|
||||
}
|
||||
|
||||
Ok(Self { inner: obj })
|
||||
}
|
||||
|
||||
pub fn get_uuid(&self) -> std::result::Result<ItemRef<'_, CFString>, CoreAudioError> {
|
||||
let uuid: *mut NSUUID = unsafe { msg_send![self.inner, UUID] };
|
||||
if uuid.is_null() {
|
||||
return Err(CoreAudioError::GetCATapDescriptionUUIDFailed);
|
||||
}
|
||||
let uuid_string: *mut NSString = unsafe { msg_send![uuid, UUIDString] };
|
||||
if uuid_string.is_null() {
|
||||
return Err(CoreAudioError::ConvertUUIDToCFStringFailed);
|
||||
}
|
||||
Ok(unsafe { CFString::from_void(uuid_string.cast()) })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CATapDescription {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _: () = msg_send![self.inner, release];
|
||||
}
|
||||
}
|
||||
}
|
||||
86
packages/frontend/native/media_capture/src/macos/cf_types.rs
Normal file
86
packages/frontend/native/media_capture/src/macos/cf_types.rs
Normal file
@@ -0,0 +1,86 @@
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
boolean::CFBoolean,
|
||||
dictionary::CFDictionary,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
use crate::utils::cfstring_from_bytes_with_nul;
|
||||
|
||||
pub trait ToCoreFoundation {
|
||||
fn to_cf(&self) -> CFType;
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for CFString {
|
||||
fn to_cf(&self) -> CFType {
|
||||
self.as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for &CFString {
|
||||
fn to_cf(&self) -> CFType {
|
||||
self.as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for String {
|
||||
fn to_cf(&self) -> CFType {
|
||||
CFString::new(self).as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for &str {
|
||||
fn to_cf(&self) -> CFType {
|
||||
CFString::new(self).as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for bool {
|
||||
fn to_cf(&self) -> CFType {
|
||||
if *self {
|
||||
CFBoolean::true_value().as_CFType()
|
||||
} else {
|
||||
CFBoolean::false_value().as_CFType()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for &[u8] {
|
||||
fn to_cf(&self) -> CFType {
|
||||
cfstring_from_bytes_with_nul(self).as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCoreFoundation for CFDictionary<CFType, CFType> {
|
||||
fn to_cf(&self) -> CFType {
|
||||
self.as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ToCoreFoundation> ToCoreFoundation for Vec<T> {
|
||||
fn to_cf(&self) -> CFType {
|
||||
CFArray::from_CFTypes(self.iter().map(|t| t.to_cf()).collect::<Vec<_>>().as_slice()).as_CFType()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CFDictionaryBuilder {
|
||||
pairs: Vec<(CFType, CFType)>,
|
||||
}
|
||||
|
||||
impl CFDictionaryBuilder {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
pairs: Vec::with_capacity(16),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add<K: ToCoreFoundation, V: ToCoreFoundation>(&mut self, key: K, value: V) -> &mut Self {
|
||||
self.pairs.push((key.to_cf(), value.to_cf()));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn build(self) -> CFDictionary<CFType, CFType> {
|
||||
CFDictionary::from_CFType_pairs(self.pairs.as_slice())
|
||||
}
|
||||
}
|
||||
28
packages/frontend/native/media_capture/src/macos/device.rs
Normal file
28
packages/frontend/native/media_capture/src/macos/device.rs
Normal file
@@ -0,0 +1,28 @@
|
||||
use std::ptr;
|
||||
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
use coreaudio::sys::{
|
||||
AudioDeviceID, AudioObjectID, CFStringRef, kAudioDevicePropertyDeviceUID, kAudioObjectSystemObject,
|
||||
};
|
||||
|
||||
use crate::{error::CoreAudioError, utils::get_global_main_property};
|
||||
|
||||
pub(crate) fn get_device_uid(
|
||||
device_id: AudioDeviceID,
|
||||
) -> std::result::Result<(AudioObjectID, CFString), CoreAudioError> {
|
||||
let system_audio_id = get_device_audio_id(device_id)?;
|
||||
let mut output_uid: CFStringRef = ptr::null_mut();
|
||||
get_global_main_property(system_audio_id, kAudioDevicePropertyDeviceUID, &mut output_uid)?;
|
||||
|
||||
Ok((system_audio_id, unsafe {
|
||||
CFString::wrap_under_create_rule(output_uid.cast())
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn get_device_audio_id(device_id: AudioDeviceID) -> std::result::Result<AudioObjectID, CoreAudioError> {
|
||||
let mut system_output_id: AudioObjectID = 0;
|
||||
|
||||
get_global_main_property(kAudioObjectSystemObject, device_id, &mut system_output_id)?;
|
||||
|
||||
Ok(system_output_id)
|
||||
}
|
||||
93
packages/frontend/native/media_capture/src/macos/error.rs
Normal file
93
packages/frontend/native/media_capture/src/macos/error.rs
Normal file
@@ -0,0 +1,93 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum CoreAudioError {
|
||||
#[error("Map pid {0} to AudioObjectID failed")]
|
||||
PidNotFound(i32),
|
||||
#[error("Create process tap failed, status: {0}")]
|
||||
CreateProcessTapFailed(i32),
|
||||
#[error("Get default device failed, status: {0}")]
|
||||
GetDefaultDeviceFailed(i32),
|
||||
#[error("Get device uid failed, status: {0}")]
|
||||
GetDeviceUidFailed(i32),
|
||||
#[error("Create aggregate device failed, status: {0}")]
|
||||
CreateAggregateDeviceFailed(i32),
|
||||
#[error("Get process object list size failed, status: {0}")]
|
||||
GetProcessObjectListSizeFailed(i32),
|
||||
#[error("Get process object list failed, status: {0}")]
|
||||
GetProcessObjectListFailed(i32),
|
||||
#[error("AudioObjectGetPropertyDataSize failed, status: {0}")]
|
||||
AudioObjectGetPropertyDataSizeFailed(i32),
|
||||
#[error("CATapDescription class not found")]
|
||||
CATapDescriptionClassNotFound,
|
||||
#[error("Alloc CATapDescription failed")]
|
||||
AllocCATapDescriptionFailed,
|
||||
#[error("Call initStereoMixdownOfProcesses on CATapDescription failed")]
|
||||
InitStereoMixdownOfProcessesFailed,
|
||||
#[error("Call initStereoGlobalTapButExcludeProcesses on CATapDescription failed")]
|
||||
InitStereoGlobalTapButExcludeProcessesFailed,
|
||||
#[error("Get UUID on CATapDescription failed")]
|
||||
GetCATapDescriptionUUIDFailed,
|
||||
#[error("Get mute behavior on CATapDescription failed")]
|
||||
GetMuteBehaviorFailed,
|
||||
#[error("Convert UUID to CFString failed")]
|
||||
ConvertUUIDToCFStringFailed,
|
||||
#[error("Get AudioStreamBasicDescription failed, status: {0}")]
|
||||
GetAudioStreamBasicDescriptionFailed(i32),
|
||||
#[error("AVAudioFormat class not found")]
|
||||
AVAudioFormatClassNotFound,
|
||||
#[error("Alloc AVAudioFormat failed")]
|
||||
AllocAVAudioFormatFailed,
|
||||
#[error("Init AVAudioFormat failed")]
|
||||
InitAVAudioFormatFailed,
|
||||
#[error("Create IOProcIDWithBlock failed, status: {0}")]
|
||||
CreateIOProcIDWithBlockFailed(i32),
|
||||
#[error("Get hardware devices failed, status: {0}")]
|
||||
GetHardwareDevicesFailed(i32),
|
||||
#[error("AudioDeviceStart failed, status: {0}")]
|
||||
AudioDeviceStartFailed(i32),
|
||||
#[error("AudioDeviceStop failed, status: {0}")]
|
||||
AudioDeviceStopFailed(i32),
|
||||
#[error("AudioDeviceDestroyIOProcID failed, status: {0}")]
|
||||
AudioDeviceDestroyIOProcIDFailed(i32),
|
||||
#[error("AudioHardwareDestroyAggregateDevice failed, status: {0}")]
|
||||
AudioHardwareDestroyAggregateDeviceFailed(i32),
|
||||
#[error("AudioHardwareDestroyProcessTap failed, status: {0}")]
|
||||
AudioHardwareDestroyProcessTapFailed(i32),
|
||||
#[error("Get aggregate device property full sub device list failed, status: {0}")]
|
||||
GetAggregateDevicePropertyFullSubDeviceListFailed(i32),
|
||||
#[error("Add property listener block failed, status: {0}")]
|
||||
AddPropertyListenerBlockFailed(i32),
|
||||
#[error("AudioObjectGetPropertyData failed, status: {0}")]
|
||||
AudioObjectGetPropertyDataFailed(i32),
|
||||
#[error("AVAudioFile class not found")]
|
||||
AVAudioFileClassNotFound,
|
||||
#[error("Alloc AVAudioFile failed")]
|
||||
AllocAVAudioFileFailed,
|
||||
#[error("Init AVAudioFile failed")]
|
||||
InitAVAudioFileFailed,
|
||||
#[error("AVAudioPCMBuffer class not found")]
|
||||
AVAudioPCMBufferClassNotFound,
|
||||
#[error("Alloc AVAudioPCMBuffer failed")]
|
||||
AllocAVAudioPCMBufferFailed,
|
||||
#[error("Init AVAudioPCMBuffer failed")]
|
||||
InitAVAudioPCMBufferFailed,
|
||||
#[error("Write AVAudioFile failed")]
|
||||
WriteAVAudioFileFailed,
|
||||
#[error("Input frame contains no data")]
|
||||
InputFrameContainsNoData,
|
||||
#[error("Output frame contains no data")]
|
||||
OutputFrameContainsNoData,
|
||||
#[error("Get property data size failed, status: {0}")]
|
||||
GetPropertyDataSizeFailed(i32),
|
||||
#[error("Get property data failed, status: {0}")]
|
||||
GetPropertyDataFailed(i32),
|
||||
#[error("Process audio {0} frame failed")]
|
||||
ProcessAudioFrameFailed(&'static str),
|
||||
}
|
||||
|
||||
impl From<CoreAudioError> for napi::Error {
|
||||
fn from(value: CoreAudioError) -> Self {
|
||||
napi::Error::new(napi::Status::GenericFailure, value.to_string())
|
||||
}
|
||||
}
|
||||
17
packages/frontend/native/media_capture/src/macos/mod.rs
Normal file
17
packages/frontend/native/media_capture/src/macos/mod.rs
Normal file
@@ -0,0 +1,17 @@
|
||||
mod application_icon;
|
||||
use application_icon::application_icon;
|
||||
|
||||
pub mod audio_buffer;
|
||||
pub mod audio_stream_basic_desc;
|
||||
pub mod av_audio_file;
|
||||
pub mod av_audio_format;
|
||||
pub mod av_audio_pcm_buffer;
|
||||
pub mod ca_tap_description;
|
||||
pub(crate) mod cf_types;
|
||||
pub mod device;
|
||||
pub(crate) mod error;
|
||||
pub mod pid;
|
||||
pub mod queue;
|
||||
pub mod screen_capture_kit;
|
||||
pub mod tap_audio;
|
||||
pub(crate) mod utils;
|
||||
88
packages/frontend/native/media_capture/src/macos/pid.rs
Normal file
88
packages/frontend/native/media_capture/src/macos/pid.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
use std::{mem::MaybeUninit, ptr};
|
||||
|
||||
use coreaudio::sys::{
|
||||
AudioObjectGetPropertyData, AudioObjectGetPropertyDataSize, AudioObjectID, AudioObjectPropertyAddress,
|
||||
AudioObjectPropertySelector, kAudioHardwareNoError, kAudioHardwarePropertyProcessObjectList,
|
||||
kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject,
|
||||
};
|
||||
|
||||
use crate::error::CoreAudioError;
|
||||
|
||||
pub fn audio_process_list() -> Result<Vec<AudioObjectID>, CoreAudioError> {
|
||||
let address = AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyProcessObjectList,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
};
|
||||
|
||||
let mut data_size = 0u32;
|
||||
let status =
|
||||
unsafe { AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &address, 0, ptr::null_mut(), &mut data_size) };
|
||||
|
||||
if status != kAudioHardwareNoError as i32 {
|
||||
return Err(CoreAudioError::GetProcessObjectListSizeFailed(status));
|
||||
}
|
||||
|
||||
let mut process_list: Vec<AudioObjectID> = vec![0; data_size as usize];
|
||||
|
||||
let status = unsafe {
|
||||
AudioObjectGetPropertyData(
|
||||
kAudioObjectSystemObject,
|
||||
&address,
|
||||
0,
|
||||
ptr::null_mut(),
|
||||
(&mut data_size as *mut u32).cast(),
|
||||
process_list.as_mut_ptr().cast(),
|
||||
)
|
||||
};
|
||||
|
||||
if status != kAudioHardwareNoError as i32 {
|
||||
return Err(CoreAudioError::GetProcessObjectListFailed(status));
|
||||
}
|
||||
|
||||
Ok(process_list)
|
||||
}
|
||||
|
||||
pub fn get_process_property<T: Sized>(
|
||||
object: &AudioObjectID,
|
||||
selector: AudioObjectPropertySelector,
|
||||
) -> Result<T, CoreAudioError> {
|
||||
let object_id = *object;
|
||||
let address = AudioObjectPropertyAddress {
|
||||
mSelector: selector,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
};
|
||||
|
||||
let mut data_size = 0u32;
|
||||
let status = unsafe { AudioObjectGetPropertyDataSize(object_id, &address, 0, ptr::null_mut(), &mut data_size) };
|
||||
|
||||
if status != kAudioHardwareNoError as i32 {
|
||||
return Err(CoreAudioError::AudioObjectGetPropertyDataSizeFailed(status));
|
||||
}
|
||||
get_property_data(object_id, &address, &mut data_size)
|
||||
}
|
||||
|
||||
pub fn get_property_data<T: Sized>(
|
||||
object_id: AudioObjectID,
|
||||
address: &AudioObjectPropertyAddress,
|
||||
data_size: &mut u32,
|
||||
) -> Result<T, CoreAudioError> {
|
||||
let mut property = MaybeUninit::<T>::uninit();
|
||||
let status = unsafe {
|
||||
AudioObjectGetPropertyData(
|
||||
object_id,
|
||||
address,
|
||||
0,
|
||||
ptr::null_mut(),
|
||||
(data_size as *mut u32).cast(),
|
||||
property.as_mut_ptr().cast(),
|
||||
)
|
||||
};
|
||||
|
||||
if status != kAudioHardwareNoError as i32 {
|
||||
return Err(CoreAudioError::AudioObjectGetPropertyDataFailed(status));
|
||||
}
|
||||
|
||||
Ok(unsafe { property.assume_init() })
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
pub(crate) fn create_audio_tap_queue() -> dispatch2::DispatchRetained<dispatch2::DispatchQueue> {
|
||||
let queue_attr = dispatch2::DispatchQueueAttr::with_qos_class(
|
||||
dispatch2::DispatchQueueAttr::SERIAL,
|
||||
dispatch2::DispatchQoS::UserInteractive,
|
||||
0,
|
||||
);
|
||||
dispatch2::DispatchQueue::new("ProcessTapRecorder", Some(&queue_attr))
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
ffi::c_void,
|
||||
ptr,
|
||||
sync::{
|
||||
Arc, LazyLock, RwLock,
|
||||
atomic::{AtomicPtr, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
use block2::{Block, RcBlock};
|
||||
use core_foundation::{
|
||||
base::TCFType,
|
||||
string::{CFString, CFStringRef},
|
||||
};
|
||||
use coreaudio::sys::{
|
||||
AudioObjectAddPropertyListenerBlock, AudioObjectID, AudioObjectPropertyAddress,
|
||||
AudioObjectRemovePropertyListenerBlock, kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain,
|
||||
kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, kAudioProcessPropertyBundleID,
|
||||
kAudioProcessPropertyIsRunning, kAudioProcessPropertyIsRunningInput, kAudioProcessPropertyPID,
|
||||
};
|
||||
use libc;
|
||||
use napi::{
|
||||
bindgen_prelude::{Buffer, Error, Result, Status},
|
||||
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
|
||||
};
|
||||
use napi_derive::napi;
|
||||
use objc2::{
|
||||
msg_send,
|
||||
runtime::{AnyClass, AnyObject},
|
||||
};
|
||||
use objc2_foundation::NSString;
|
||||
use screencapturekit::shareable_content::SCShareableContent;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
audio_callback::AudioCallback,
|
||||
error::CoreAudioError,
|
||||
pid::{audio_process_list, get_process_property},
|
||||
tap_audio::{AggregateDeviceManager, AudioCaptureSession},
|
||||
};
|
||||
|
||||
static RUNNING_APPLICATIONS: LazyLock<RwLock<std::result::Result<Vec<AudioObjectID>, CoreAudioError>>> =
|
||||
LazyLock::new(|| RwLock::new(audio_process_list()));
|
||||
|
||||
type ApplicationStateChangedSubscriberMap = HashMap<AudioObjectID, HashMap<Uuid, Arc<ThreadsafeFunction<(), ()>>>>;
|
||||
|
||||
static APPLICATION_STATE_CHANGED_SUBSCRIBERS: LazyLock<RwLock<ApplicationStateChangedSubscriberMap>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
static APPLICATION_STATE_CHANGED_LISTENER_BLOCKS: LazyLock<RwLock<HashMap<AudioObjectID, AtomicPtr<c_void>>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
static NSRUNNING_APPLICATION_CLASS: LazyLock<Option<&'static AnyClass>> =
|
||||
LazyLock::new(|| AnyClass::get(c"NSRunningApplication"));
|
||||
|
||||
#[napi]
|
||||
#[derive(Clone)]
|
||||
pub struct ApplicationInfo {
|
||||
pub process_id: i32,
|
||||
pub name: String,
|
||||
pub object_id: u32,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ApplicationInfo {
|
||||
#[napi(constructor)]
|
||||
pub fn new(process_id: i32, name: String, object_id: u32) -> Self {
|
||||
Self {
|
||||
process_id,
|
||||
name,
|
||||
object_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn process_group_id(&self) -> i32 {
|
||||
if self.process_id > 0 {
|
||||
let pgid = unsafe { libc::getpgid(self.process_id) };
|
||||
if pgid != -1 {
|
||||
return pgid;
|
||||
}
|
||||
// Fall back to process_id if getpgid fails
|
||||
return self.process_id;
|
||||
}
|
||||
-1
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn bundle_identifier(&self) -> String {
|
||||
if self.process_id <= 0 {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// Try to get bundle identifier using NSRunningApplication
|
||||
if let Some(running_app_class) = NSRUNNING_APPLICATION_CLASS.as_ref() {
|
||||
let running_app: *mut AnyObject = unsafe {
|
||||
msg_send![
|
||||
*running_app_class,
|
||||
runningApplicationWithProcessIdentifier: self.process_id
|
||||
]
|
||||
};
|
||||
|
||||
if !running_app.is_null() {
|
||||
unsafe {
|
||||
let bundle_id_ptr: *mut NSString = msg_send![running_app, bundleIdentifier];
|
||||
if !bundle_id_ptr.is_null() {
|
||||
let length: usize = msg_send![bundle_id_ptr, length];
|
||||
let utf8_ptr: *const u8 = msg_send![bundle_id_ptr, UTF8String];
|
||||
|
||||
if !utf8_ptr.is_null() {
|
||||
let bytes = std::slice::from_raw_parts(utf8_ptr, length);
|
||||
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||
return s.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not available, try to get from the audio process property
|
||||
if self.object_id > 0
|
||||
&& let Ok(bundle_id) = get_process_property::<CFStringRef>(&self.object_id, kAudioProcessPropertyBundleID)
|
||||
{
|
||||
// Safely convert CFStringRef to Rust String
|
||||
let cf_string = unsafe { CFString::wrap_under_get_rule(bundle_id) };
|
||||
return cf_string.to_string();
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn icon(&self) -> Result<Buffer> {
|
||||
super::application_icon(self.process_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct ApplicationListChangedSubscriber {
|
||||
listener_block: RcBlock<dyn Fn(u32, *mut c_void)>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ApplicationListChangedSubscriber {
|
||||
#[napi]
|
||||
pub fn unsubscribe(&self) -> Result<()> {
|
||||
// Wrap in catch_unwind to prevent crashes during shutdown
|
||||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
|
||||
AudioObjectRemovePropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyProcessObjectList,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
(&*self.listener_block as *const Block<dyn Fn(u32, *mut c_void)>)
|
||||
.cast_mut()
|
||||
.cast(),
|
||||
)
|
||||
}));
|
||||
|
||||
match result {
|
||||
Ok(status) => {
|
||||
if status != 0 {
|
||||
return Err(Error::new(Status::GenericFailure, "Failed to remove property listener"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
// If we panicked (likely during shutdown), consider it success
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct ApplicationStateChangedSubscriber {
|
||||
id: Uuid,
|
||||
object_id: AudioObjectID,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ApplicationStateChangedSubscriber {
|
||||
#[napi]
|
||||
pub fn unsubscribe(&self) {
|
||||
if let Ok(mut lock) = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write()
|
||||
&& let Some(subscribers) = lock.get_mut(&self.object_id)
|
||||
{
|
||||
subscribers.remove(&self.id);
|
||||
if subscribers.is_empty() {
|
||||
lock.remove(&self.object_id);
|
||||
if let Some(listener_block) = APPLICATION_STATE_CHANGED_LISTENER_BLOCKS
|
||||
.write()
|
||||
.ok()
|
||||
.as_mut()
|
||||
.and_then(|map| map.remove(&self.object_id))
|
||||
{
|
||||
// Wrap in catch_unwind to prevent crashes during shutdown
|
||||
let _ = std::panic::catch_unwind(|| unsafe {
|
||||
AudioObjectRemovePropertyListenerBlock(
|
||||
self.object_id,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioProcessPropertyIsRunning,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
listener_block.load(Ordering::Relaxed),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct ShareableContent {
|
||||
_inner: SCShareableContent,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ShareableContent {
|
||||
#[napi]
|
||||
pub fn on_application_list_changed(callback: ThreadsafeFunction<(), ()>) -> Result<ApplicationListChangedSubscriber> {
|
||||
let callback_arc = Arc::new(callback);
|
||||
let callback_clone = callback_arc.clone();
|
||||
let callback_block: RcBlock<dyn Fn(u32, *mut c_void)> =
|
||||
RcBlock::new(move |_in_number_addresses, _in_addresses: *mut c_void| {
|
||||
if let Err(err) = RUNNING_APPLICATIONS
|
||||
.write()
|
||||
.map_err(|_| {
|
||||
Error::new(
|
||||
Status::GenericFailure,
|
||||
"Poisoned RwLock while writing RunningApplications",
|
||||
)
|
||||
})
|
||||
.map(|mut running_applications| {
|
||||
*running_applications = audio_process_list();
|
||||
})
|
||||
{
|
||||
callback_clone.call(Err(err), ThreadsafeFunctionCallMode::NonBlocking);
|
||||
} else {
|
||||
callback_clone.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);
|
||||
}
|
||||
});
|
||||
|
||||
let status = unsafe {
|
||||
AudioObjectAddPropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyProcessObjectList,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
(&*callback_block as *const Block<dyn Fn(u32, *mut c_void)>)
|
||||
.cast_mut()
|
||||
.cast(),
|
||||
)
|
||||
};
|
||||
if status != 0 {
|
||||
return Err(Error::new(Status::GenericFailure, "Failed to add property listener"));
|
||||
}
|
||||
Ok(ApplicationListChangedSubscriber {
|
||||
listener_block: callback_block,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn on_app_state_changed(
|
||||
app: &ApplicationInfo,
|
||||
callback: ThreadsafeFunction<(), ()>,
|
||||
) -> Result<ApplicationStateChangedSubscriber> {
|
||||
let id = Uuid::new_v4();
|
||||
let object_id = app.object_id;
|
||||
let callback_arc = Arc::new(callback);
|
||||
|
||||
let mut lock = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write().map_err(|_| {
|
||||
Error::new(
|
||||
Status::GenericFailure,
|
||||
"Poisoned RwLock while writing ApplicationStateChangedSubscribers",
|
||||
)
|
||||
})?;
|
||||
|
||||
if let Some(subscribers) = lock.get_mut(&object_id) {
|
||||
subscribers.insert(id, callback_arc.clone());
|
||||
} else {
|
||||
let list_change: RcBlock<dyn Fn(u32, *mut c_void)> =
|
||||
RcBlock::new(move |in_number_addresses, in_addresses: *mut c_void| {
|
||||
let addresses = unsafe {
|
||||
std::slice::from_raw_parts(
|
||||
in_addresses as *mut AudioObjectPropertyAddress,
|
||||
in_number_addresses as usize,
|
||||
)
|
||||
};
|
||||
for address in addresses {
|
||||
if address.mSelector == kAudioProcessPropertyIsRunning
|
||||
&& let Some(subscribers) = APPLICATION_STATE_CHANGED_SUBSCRIBERS
|
||||
.read()
|
||||
.ok()
|
||||
.as_ref()
|
||||
.and_then(|map| map.get(&object_id))
|
||||
{
|
||||
for callback in subscribers.values() {
|
||||
callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let address = AudioObjectPropertyAddress {
|
||||
mSelector: kAudioProcessPropertyIsRunning,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
};
|
||||
let listener_block = &*list_change as *const Block<dyn Fn(u32, *mut c_void)>;
|
||||
let status = unsafe {
|
||||
AudioObjectAddPropertyListenerBlock(object_id, &address, ptr::null_mut(), listener_block.cast_mut().cast())
|
||||
};
|
||||
if status != 0 {
|
||||
return Err(Error::new(Status::GenericFailure, "Failed to add property listener"));
|
||||
}
|
||||
let subscribers = {
|
||||
let mut map = HashMap::new();
|
||||
map.insert(id, callback_arc.clone());
|
||||
map
|
||||
};
|
||||
lock.insert(object_id, subscribers);
|
||||
}
|
||||
Ok(ApplicationStateChangedSubscriber { id, object_id })
|
||||
}
|
||||
|
||||
#[napi(constructor)]
|
||||
pub fn new() -> Result<Self> {
|
||||
Ok(Self {
|
||||
_inner: SCShareableContent::get().map_err(|err| Error::new(Status::GenericFailure, err))?,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn applications() -> Result<Vec<ApplicationInfo>> {
|
||||
let app_list = RUNNING_APPLICATIONS
|
||||
.read()
|
||||
.map_err(|_| {
|
||||
Error::new(
|
||||
Status::GenericFailure,
|
||||
"Poisoned RwLock while reading RunningApplications",
|
||||
)
|
||||
})?
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter_map(|id| {
|
||||
// Get process ID from object_id
|
||||
let process_id = get_process_property(id, kAudioProcessPropertyPID).unwrap_or(-1);
|
||||
|
||||
if process_id <= 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Get application name using NSRunningApplication
|
||||
let mut name = String::new();
|
||||
if let Some(running_app_class) = NSRUNNING_APPLICATION_CLASS.as_ref() {
|
||||
let running_app: *mut AnyObject = unsafe {
|
||||
msg_send![
|
||||
*running_app_class,
|
||||
runningApplicationWithProcessIdentifier: process_id
|
||||
]
|
||||
};
|
||||
|
||||
if !running_app.is_null() {
|
||||
unsafe {
|
||||
let name_ptr: *mut NSString = msg_send![running_app, localizedName];
|
||||
if !name_ptr.is_null() {
|
||||
let length: usize = msg_send![name_ptr, length];
|
||||
let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String];
|
||||
|
||||
if !utf8_ptr.is_null() {
|
||||
let bytes = std::slice::from_raw_parts(utf8_ptr, length);
|
||||
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||
name = s.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let app = ApplicationInfo::new(process_id, name, *id);
|
||||
if !app.bundle_identifier().is_empty() {
|
||||
Some(app)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
Ok(app_list)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn application_with_process_id(process_id: u32) -> Option<ApplicationInfo> {
|
||||
// check if the process is tappable
|
||||
let tappable = ShareableContent::tappable_application_with_process_id(process_id);
|
||||
if let Some(tappable) = tappable {
|
||||
return Some(tappable);
|
||||
}
|
||||
|
||||
// Get NSRunningApplication class
|
||||
let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref()?;
|
||||
|
||||
// Get running application with PID
|
||||
let running_app: *mut AnyObject = unsafe {
|
||||
msg_send![
|
||||
*running_app_class,
|
||||
runningApplicationWithProcessIdentifier: process_id as i32
|
||||
]
|
||||
};
|
||||
|
||||
if running_app.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Get application name
|
||||
let mut name = String::new();
|
||||
unsafe {
|
||||
let name_ptr: *mut NSString = msg_send![running_app, localizedName];
|
||||
if !name_ptr.is_null() {
|
||||
let length: usize = msg_send![name_ptr, length];
|
||||
let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String];
|
||||
|
||||
if !utf8_ptr.is_null() {
|
||||
let bytes = std::slice::from_raw_parts(utf8_ptr, length);
|
||||
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||
name = s.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create an ApplicationInfo with the proper name and object_id 0 (since we
|
||||
// don't have audio object_id from process_id alone)
|
||||
Some(ApplicationInfo::new(process_id as i32, name, 0))
|
||||
}
|
||||
|
||||
pub fn tappable_application_with_process_id(process_id: u32) -> Option<ApplicationInfo> {
|
||||
// Find the TappableApplication with this process ID in the list of running
|
||||
// applications
|
||||
match ShareableContent::applications() {
|
||||
Ok(apps) => {
|
||||
for app in apps {
|
||||
if app.process_id == process_id as i32 {
|
||||
return Some(app);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn is_using_microphone(process_id: u32) -> Result<bool> {
|
||||
if process_id == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Find the audio object ID for this process
|
||||
if let Ok(app_list) = RUNNING_APPLICATIONS.read()
|
||||
&& let Ok(app_list) = app_list.as_ref()
|
||||
{
|
||||
for object_id in app_list {
|
||||
let pid = get_process_property(object_id, kAudioProcessPropertyPID).unwrap_or(-1);
|
||||
if pid == process_id as i32 {
|
||||
// Check if the process is actively using input (microphone)
|
||||
match get_process_property(object_id, kAudioProcessPropertyIsRunningInput) {
|
||||
Ok(is_running) => return Ok(is_running),
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub(crate) fn tap_audio_with_callback(
|
||||
process_id: u32,
|
||||
audio_stream_callback: AudioCallback,
|
||||
) -> Result<AudioCaptureSession> {
|
||||
let app = ShareableContent::applications()?
|
||||
.into_iter()
|
||||
.find(|app| app.process_id == process_id as i32);
|
||||
|
||||
if let Some(app) = app {
|
||||
if app.object_id == 0 {
|
||||
return Err(Error::new(
|
||||
Status::GenericFailure,
|
||||
"Cannot tap audio: invalid object_id",
|
||||
));
|
||||
}
|
||||
|
||||
// Use AggregateDeviceManager instead of AggregateDevice directly
|
||||
// This provides automatic default device change detection
|
||||
let mut device_manager = AggregateDeviceManager::new(&app)?;
|
||||
device_manager.start_capture(audio_stream_callback)?;
|
||||
let boxed_manager = Box::new(device_manager);
|
||||
Ok(AudioCaptureSession::new(boxed_manager))
|
||||
} else {
|
||||
Err(Error::new(
|
||||
Status::GenericFailure,
|
||||
"Application not found or not available for audio tapping",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn tap_global_audio_with_callback(
|
||||
excluded_processes: Option<Vec<&ApplicationInfo>>,
|
||||
audio_stream_callback: AudioCallback,
|
||||
) -> Result<AudioCaptureSession> {
|
||||
let excluded_object_ids = excluded_processes
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|app| app.object_id)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Use the new AggregateDeviceManager for automatic device adaptation
|
||||
let mut device_manager = AggregateDeviceManager::new_global(&excluded_object_ids)?;
|
||||
device_manager.start_capture(audio_stream_callback)?;
|
||||
let boxed_manager = Box::new(device_manager);
|
||||
Ok(AudioCaptureSession::new(boxed_manager))
|
||||
}
|
||||
}
|
||||
984
packages/frontend/native/media_capture/src/macos/tap_audio.rs
Normal file
984
packages/frontend/native/media_capture/src/macos/tap_audio.rs
Normal file
@@ -0,0 +1,984 @@
|
||||
use std::{ffi::c_void, ptr, sync::Arc};
|
||||
|
||||
use block2::{Block, RcBlock};
|
||||
use core_foundation::{
|
||||
base::{CFType, ItemRef, TCFType},
|
||||
dictionary::CFDictionary,
|
||||
string::CFString,
|
||||
uuid::CFUUID,
|
||||
};
|
||||
use coreaudio::sys::{
|
||||
AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID, AudioDeviceStart,
|
||||
AudioDeviceStop, AudioHardwareCreateAggregateDevice, AudioHardwareDestroyAggregateDevice,
|
||||
AudioObjectAddPropertyListenerBlock, AudioObjectGetPropertyDataSize, AudioObjectID, AudioObjectPropertyAddress,
|
||||
AudioObjectRemovePropertyListenerBlock, AudioTimeStamp, OSStatus, kAudioAggregateDeviceClockDeviceKey,
|
||||
kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey, kAudioAggregateDeviceMainSubDeviceKey,
|
||||
kAudioAggregateDeviceNameKey, kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey,
|
||||
kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioDevicePropertyDeviceIsAlive,
|
||||
kAudioDevicePropertyNominalSampleRate, kAudioHardwareBadDeviceError, kAudioHardwareBadStreamError,
|
||||
kAudioHardwareNoError, kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice,
|
||||
kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, kAudioSubDeviceUIDKey,
|
||||
kAudioSubTapUIDKey,
|
||||
};
|
||||
use napi::bindgen_prelude::Result;
|
||||
use objc2::runtime::AnyObject;
|
||||
|
||||
use crate::{
|
||||
audio_buffer::InputAndOutputAudioBufferList,
|
||||
audio_callback::AudioCallback,
|
||||
ca_tap_description::CATapDescription,
|
||||
cf_types::CFDictionaryBuilder,
|
||||
device::get_device_uid,
|
||||
error::CoreAudioError,
|
||||
queue::create_audio_tap_queue,
|
||||
screen_capture_kit::ApplicationInfo,
|
||||
utils::{cfstring_from_bytes_with_nul, get_global_main_property},
|
||||
};
|
||||
|
||||
unsafe extern "C" {
|
||||
fn AudioHardwareCreateProcessTap(inDescription: *mut AnyObject, outTapID: *mut AudioObjectID) -> OSStatus;
|
||||
|
||||
fn AudioHardwareDestroyProcessTap(tapID: AudioObjectID) -> OSStatus;
|
||||
}
|
||||
|
||||
// Audio statistics structure to track audio format information
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct AudioStats {
|
||||
pub sample_rate: f64,
|
||||
pub channels: u32,
|
||||
}
|
||||
|
||||
pub struct AggregateDevice {
|
||||
pub tap_id: AudioObjectID,
|
||||
pub id: AudioObjectID,
|
||||
pub audio_stats: Option<AudioStats>,
|
||||
pub input_device_id: AudioObjectID,
|
||||
pub output_device_id: AudioObjectID,
|
||||
pub input_proc_id: Option<AudioDeviceIOProcID>,
|
||||
pub output_proc_id: Option<AudioDeviceIOProcID>,
|
||||
}
|
||||
|
||||
impl AggregateDevice {
|
||||
pub fn new(app: &ApplicationInfo) -> Result<Self> {
|
||||
let object_id = app.object_id;
|
||||
|
||||
let tap_description = CATapDescription::init_stereo_mixdown_of_processes(object_id)?;
|
||||
let mut tap_id: AudioObjectID = 0;
|
||||
|
||||
let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) };
|
||||
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::CreateProcessTapFailed(status).into());
|
||||
}
|
||||
|
||||
let (input_device_id, default_input_uid) = get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?;
|
||||
|
||||
// Get the default output device ID
|
||||
let (output_device_id, output_device_uid) = get_device_uid(kAudioHardwarePropertyDefaultOutputDevice)?;
|
||||
let description_dict = Self::create_aggregate_description(
|
||||
tap_id,
|
||||
tap_description.get_uuid()?,
|
||||
default_input_uid,
|
||||
output_device_uid,
|
||||
)?;
|
||||
|
||||
let mut aggregate_device_id: AudioObjectID = 0;
|
||||
|
||||
let status = unsafe {
|
||||
AudioHardwareCreateAggregateDevice(description_dict.as_concrete_TypeRef().cast(), &mut aggregate_device_id)
|
||||
};
|
||||
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
tap_id,
|
||||
id: aggregate_device_id,
|
||||
audio_stats: None,
|
||||
input_device_id,
|
||||
output_device_id,
|
||||
input_proc_id: None,
|
||||
output_proc_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn create_global_tap_but_exclude_processes(processes: &[AudioObjectID]) -> Result<Self> {
|
||||
let mut tap_id: AudioObjectID = 0;
|
||||
let tap_description = CATapDescription::init_stereo_global_tap_but_exclude_processes(processes)?;
|
||||
let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) };
|
||||
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::CreateProcessTapFailed(status).into());
|
||||
}
|
||||
|
||||
// Get the default input device (microphone) UID and ID
|
||||
let (input_device_id, default_input_uid) = get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?;
|
||||
|
||||
// Get the default output device ID
|
||||
let (output_device_id, output_device_uid) = get_device_uid(kAudioHardwarePropertyDefaultOutputDevice)?;
|
||||
|
||||
let description_dict = Self::create_aggregate_description(
|
||||
tap_id,
|
||||
tap_description.get_uuid()?,
|
||||
default_input_uid,
|
||||
output_device_uid,
|
||||
)?;
|
||||
|
||||
let mut aggregate_device_id: AudioObjectID = 0;
|
||||
|
||||
let status = unsafe {
|
||||
AudioHardwareCreateAggregateDevice(description_dict.as_concrete_TypeRef().cast(), &mut aggregate_device_id)
|
||||
};
|
||||
|
||||
// Check the status and return the appropriate result
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());
|
||||
}
|
||||
|
||||
// Create a device with stored device IDs
|
||||
let mut device = Self {
|
||||
tap_id,
|
||||
id: aggregate_device_id,
|
||||
audio_stats: None,
|
||||
input_device_id,
|
||||
output_device_id,
|
||||
input_proc_id: None,
|
||||
output_proc_id: None,
|
||||
};
|
||||
|
||||
// Restore the activation logic as it seems necessary for audio flow
|
||||
// Configure the aggregate device to ensure proper handling of both input
|
||||
// and output
|
||||
device.get_aggregate_device_stats()?;
|
||||
|
||||
// Activate both the input and output devices and store their proc IDs
|
||||
let input_proc_id = device.activate_audio_device(input_device_id)?;
|
||||
let output_proc_id = device.activate_audio_device(output_device_id)?;
|
||||
|
||||
device.input_proc_id = Some(input_proc_id);
|
||||
device.output_proc_id = Some(output_proc_id);
|
||||
|
||||
Ok(device)
|
||||
}
|
||||
|
||||
fn get_aggregate_device_stats(&self) -> Result<AudioStats> {
|
||||
let mut sample_rate: f64 = 0.0;
|
||||
get_global_main_property(self.id, kAudioDevicePropertyNominalSampleRate, &mut sample_rate)?;
|
||||
|
||||
let audio_stats = AudioStats {
|
||||
sample_rate,
|
||||
channels: 2,
|
||||
};
|
||||
|
||||
Ok(audio_stats)
|
||||
}
|
||||
|
||||
// Activates an audio device by creating a dummy IO proc
|
||||
fn activate_audio_device(&self, device_id: AudioObjectID) -> Result<AudioDeviceIOProcID> {
|
||||
// Create a simple no-op dummy proc
|
||||
let dummy_block = RcBlock::new(
|
||||
|_: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void| {
|
||||
// No-op function that just returns success
|
||||
kAudioHardwareNoError as i32
|
||||
},
|
||||
);
|
||||
|
||||
let mut dummy_proc_id: AudioDeviceIOProcID = None;
|
||||
|
||||
// Create the IO proc with our dummy block
|
||||
let status = unsafe {
|
||||
AudioDeviceCreateIOProcIDWithBlock(
|
||||
&mut dummy_proc_id,
|
||||
device_id,
|
||||
ptr::null_mut(),
|
||||
(&*dummy_block.copy() as *const Block<dyn Fn(_, _, _, _, _) -> i32>)
|
||||
.cast_mut()
|
||||
.cast(),
|
||||
)
|
||||
};
|
||||
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into());
|
||||
}
|
||||
|
||||
// Start the device to activate it
|
||||
let status = unsafe { AudioDeviceStart(device_id, dummy_proc_id) };
|
||||
if status != 0 {
|
||||
// Clean up the IO proc if AudioDeviceStart fails
|
||||
let _ = unsafe { AudioDeviceDestroyIOProcID(device_id, dummy_proc_id) };
|
||||
return Err(CoreAudioError::AudioDeviceStartFailed(status).into());
|
||||
}
|
||||
|
||||
// Return the proc ID for later cleanup
|
||||
Ok(dummy_proc_id)
|
||||
}
|
||||
|
||||
/// Implementation for the AggregateDevice to start processing audio
|
||||
pub fn start(
|
||||
&mut self,
|
||||
audio_stream_callback: AudioCallback,
|
||||
// Add original_audio_stats to ensure consistent target rate
|
||||
original_audio_stats: AudioStats,
|
||||
) -> Result<AudioTapStream> {
|
||||
let mut current_audio_stats = self.get_aggregate_device_stats()?;
|
||||
|
||||
let queue = create_audio_tap_queue();
|
||||
let mut in_proc_id: AudioDeviceIOProcID = None;
|
||||
|
||||
// Get the current input and output sample rates
|
||||
let mut input_device_sample_rate: f64 = 0.0;
|
||||
get_global_main_property(
|
||||
self.input_device_id,
|
||||
kAudioDevicePropertyNominalSampleRate,
|
||||
&mut input_device_sample_rate,
|
||||
)?;
|
||||
|
||||
let output_sample_rate = current_audio_stats.sample_rate;
|
||||
// Use the consistent original sample rate as the target for the IO block
|
||||
let target_sample_rate = original_audio_stats.sample_rate;
|
||||
|
||||
// Update the device's reported stats to the consistent one
|
||||
current_audio_stats.sample_rate = target_sample_rate;
|
||||
current_audio_stats.channels = original_audio_stats.channels;
|
||||
self.audio_stats = Some(current_audio_stats);
|
||||
|
||||
// Use the consistent stats for the stream object returned
|
||||
let audio_stats_for_stream = current_audio_stats;
|
||||
|
||||
let in_io_block: RcBlock<dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32>;
|
||||
{
|
||||
in_io_block = RcBlock::new(
|
||||
move |_in_now: *mut c_void,
|
||||
in_input_data: *mut c_void,
|
||||
in_input_time: *mut c_void,
|
||||
_in_output_data: *mut c_void,
|
||||
_in_output_time: *mut c_void| {
|
||||
let AudioTimeStamp { mSampleTime, .. } = unsafe { &*in_input_time.cast() };
|
||||
|
||||
// ignore pre-roll
|
||||
if *mSampleTime < 0.0 {
|
||||
return kAudioHardwareNoError as i32;
|
||||
}
|
||||
let Ok(dua_audio_buffer_list) = (unsafe { InputAndOutputAudioBufferList::from_raw(in_input_data) }) else {
|
||||
return kAudioHardwareBadDeviceError as i32;
|
||||
};
|
||||
|
||||
let Ok(mixed_samples) = dua_audio_buffer_list.mix_input_and_output(
|
||||
target_sample_rate,
|
||||
input_device_sample_rate,
|
||||
output_sample_rate,
|
||||
) else {
|
||||
return kAudioHardwareBadStreamError as i32;
|
||||
};
|
||||
|
||||
// Send the processed audio data to the configured sink
|
||||
audio_stream_callback.call(mixed_samples);
|
||||
|
||||
kAudioHardwareNoError as i32
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let status = unsafe {
|
||||
AudioDeviceCreateIOProcIDWithBlock(
|
||||
&mut in_proc_id,
|
||||
self.id,
|
||||
dispatch2::DispatchRetained::as_ptr(&queue).as_ptr().cast(),
|
||||
(&*in_io_block as *const Block<dyn Fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void) -> i32>)
|
||||
.cast_mut()
|
||||
.cast(),
|
||||
)
|
||||
};
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into());
|
||||
}
|
||||
|
||||
let status = unsafe { AudioDeviceStart(self.id, in_proc_id) };
|
||||
if status != 0 {
|
||||
// Attempt to clean up the IO proc if start failed
|
||||
let _cleanup_status = unsafe { AudioDeviceDestroyIOProcID(self.id, in_proc_id) };
|
||||
return Err(CoreAudioError::AudioDeviceStartFailed(status).into());
|
||||
}
|
||||
|
||||
Ok(AudioTapStream {
|
||||
device_id: self.id,
|
||||
in_proc_id,
|
||||
stop_called: false,
|
||||
audio_stats: audio_stats_for_stream,
|
||||
input_device_id: self.input_device_id,
|
||||
output_device_id: self.output_device_id,
|
||||
input_proc_id: self.input_proc_id,
|
||||
output_proc_id: self.output_proc_id,
|
||||
queue: Some(queue),
|
||||
})
|
||||
}
|
||||
|
||||
fn create_aggregate_description(
|
||||
tap_id: AudioObjectID,
|
||||
tap_uuid_string: ItemRef<CFString>,
|
||||
input_device_id: CFString,
|
||||
output_device_id: CFString,
|
||||
) -> Result<CFDictionary<CFType, CFType>> {
|
||||
let aggregate_device_name = CFString::new(&format!("Tap-{tap_id}"));
|
||||
let aggregate_device_uid: uuid::Uuid = CFUUID::new().into();
|
||||
let aggregate_device_uid_string = aggregate_device_uid.to_string();
|
||||
|
||||
let mut sub_device_input_dict = CFDictionaryBuilder::new();
|
||||
sub_device_input_dict.add(kAudioSubDeviceUIDKey.as_slice(), &input_device_id);
|
||||
|
||||
let tap_device_dict = CFDictionary::from_CFType_pairs(&[(
|
||||
cfstring_from_bytes_with_nul(kAudioSubTapUIDKey).as_CFType(),
|
||||
tap_uuid_string.as_CFType(),
|
||||
)]);
|
||||
|
||||
let capture_device_list = vec![sub_device_input_dict.build()];
|
||||
|
||||
// Create the aggregate device description dictionary with a balanced
|
||||
// configuration
|
||||
|
||||
let mut cf_dict_builder = CFDictionaryBuilder::new();
|
||||
|
||||
cf_dict_builder
|
||||
.add(kAudioAggregateDeviceNameKey.as_slice(), aggregate_device_name)
|
||||
.add(kAudioAggregateDeviceUIDKey.as_slice(), aggregate_device_uid_string)
|
||||
.add(kAudioAggregateDeviceMainSubDeviceKey.as_slice(), &output_device_id)
|
||||
.add(kAudioAggregateDeviceIsPrivateKey.as_slice(), true)
|
||||
// can't be stacked because we're using a tap
|
||||
.add(kAudioAggregateDeviceIsStackedKey.as_slice(), false)
|
||||
.add(kAudioAggregateDeviceTapAutoStartKey.as_slice(), true)
|
||||
.add(kAudioAggregateDeviceSubDeviceListKey.as_slice(), capture_device_list)
|
||||
.add(kAudioAggregateDeviceClockDeviceKey.as_slice(), input_device_id)
|
||||
.add(kAudioAggregateDeviceTapListKey.as_slice(), vec![tap_device_dict]);
|
||||
|
||||
Ok(cf_dict_builder.build())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AudioTapStream {
|
||||
device_id: AudioObjectID,
|
||||
in_proc_id: AudioDeviceIOProcID,
|
||||
stop_called: bool,
|
||||
audio_stats: AudioStats,
|
||||
input_device_id: AudioObjectID,
|
||||
output_device_id: AudioObjectID,
|
||||
input_proc_id: Option<AudioDeviceIOProcID>,
|
||||
output_proc_id: Option<AudioDeviceIOProcID>,
|
||||
queue: Option<dispatch2::DispatchRetained<dispatch2::DispatchQueue>>,
|
||||
}
|
||||
|
||||
impl AudioTapStream {
|
||||
pub fn stop(&mut self) -> Result<()> {
|
||||
if self.stop_called {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.stop_called = true;
|
||||
|
||||
// Check if device exists before attempting to stop it
|
||||
let mut device_exists = false;
|
||||
let mut dummy_size: u32 = 0;
|
||||
let device_check_status = unsafe {
|
||||
AudioObjectGetPropertyDataSize(
|
||||
self.device_id,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioDevicePropertyDeviceIsAlive,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
0,
|
||||
ptr::null(),
|
||||
&mut dummy_size,
|
||||
)
|
||||
};
|
||||
|
||||
if device_check_status == 0 {
|
||||
device_exists = true;
|
||||
}
|
||||
|
||||
// Stop the main aggregate device - ignore errors as device might already be
|
||||
// stopped or disconnected
|
||||
if device_exists {
|
||||
let status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) };
|
||||
// Don't fail the whole stop process if this fails, just log the error and
|
||||
// continue cleanup
|
||||
if status != 0 {
|
||||
// kAudioHardwareBadDeviceError (560227702 / 0x2166616E in hex)
|
||||
// indicates the device is gone, which is expected in some
|
||||
// scenarios (like device unplug). Treat this as non-existent.
|
||||
if status == kAudioHardwareBadDeviceError as i32 {
|
||||
device_exists = false; // Treat as non-existent for subsequent steps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the input device if it was activated
|
||||
if let Some(proc_id) = self.input_proc_id {
|
||||
// Ignore errors as device might be disconnected
|
||||
let status = unsafe { AudioDeviceStop(self.input_device_id, proc_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: Input device stop failed with status: {status}");
|
||||
}
|
||||
|
||||
let status = unsafe { AudioDeviceDestroyIOProcID(self.input_device_id, proc_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: Input device destroy IO proc failed with status: {status}");
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the output device if it was activated
|
||||
if let Some(proc_id) = self.output_proc_id {
|
||||
// Ignore errors as device might be disconnected
|
||||
let status = unsafe { AudioDeviceStop(self.output_device_id, proc_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: Output device stop failed with status: {status}");
|
||||
}
|
||||
|
||||
let status = unsafe { AudioDeviceDestroyIOProcID(self.output_device_id, proc_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: Output device destroy IO proc failed with status: {status}");
|
||||
}
|
||||
}
|
||||
|
||||
// Destroy the main IO proc if device still exists
|
||||
if device_exists {
|
||||
let status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: Destroy IO proc failed with status: {status}");
|
||||
}
|
||||
}
|
||||
let status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: AudioHardwareDestroyAggregateDevice failed with status: {status}");
|
||||
}
|
||||
|
||||
// Destroy the process tap - don't fail if this fails
|
||||
let status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) };
|
||||
if status != 0 {
|
||||
println!("DEBUG: WARNING: AudioHardwareDestroyProcessTap failed with status: {status}");
|
||||
}
|
||||
|
||||
// destroy the queue
|
||||
drop(self.queue.take());
|
||||
|
||||
// Always return success to prevent errors from bubbling up to JavaScript
|
||||
// since we've made a best effort to clean up
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_sample_rate(&self) -> f64 {
|
||||
self.audio_stats.sample_rate
|
||||
}
|
||||
|
||||
/// Gets the actual sample rate of the current device
|
||||
///
|
||||
/// This can be different from the original sample rate if the default device
|
||||
/// has changed. The original sample rate is maintained for consistency in
|
||||
/// audio processing, but applications might need to know the actual device
|
||||
/// sample rate for certain operations.
|
||||
pub fn get_actual_sample_rate(&self) -> Result<f64> {
|
||||
let device_id = self.output_device_id;
|
||||
let mut actual_sample_rate: f64 = 0.0;
|
||||
let status = unsafe {
|
||||
let address = AudioObjectPropertyAddress {
|
||||
mSelector: kAudioDevicePropertyNominalSampleRate,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
};
|
||||
|
||||
let mut size = std::mem::size_of::<f64>() as u32;
|
||||
coreaudio::sys::AudioObjectGetPropertyData(
|
||||
device_id,
|
||||
&address,
|
||||
0,
|
||||
ptr::null(),
|
||||
&mut size,
|
||||
&mut actual_sample_rate as *mut f64 as *mut c_void,
|
||||
)
|
||||
};
|
||||
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::GetPropertyDataFailed(status).into());
|
||||
}
|
||||
|
||||
Ok(actual_sample_rate)
|
||||
}
|
||||
|
||||
pub fn get_channels(&self) -> u32 {
|
||||
self.audio_stats.channels
|
||||
}
|
||||
|
||||
/// Mark the stream as stopped without performing actual cleanup
|
||||
/// This is used when the device is known to be in an invalid state
|
||||
pub fn mark_stopped_without_cleanup(&mut self) {
|
||||
self.stop_called = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// A manager for audio device handling that automatically adapts to device
|
||||
/// changes
|
||||
pub struct AggregateDeviceManager {
|
||||
device: AggregateDevice,
|
||||
default_devices_listener: Option<*mut c_void>,
|
||||
is_app_specific: bool,
|
||||
app_id: Option<AudioObjectID>,
|
||||
excluded_processes: Vec<AudioObjectID>,
|
||||
active_stream: Option<Arc<std::sync::Mutex<Option<AudioTapStream>>>>,
|
||||
audio_callback: Option<AudioCallback>,
|
||||
original_audio_stats: Option<AudioStats>,
|
||||
}
|
||||
|
||||
impl AggregateDeviceManager {
|
||||
/// Creates a new AggregateDeviceManager for a specific application
|
||||
pub fn new(app: &ApplicationInfo) -> Result<Self> {
|
||||
let device = AggregateDevice::new(app)?;
|
||||
|
||||
Ok(Self {
|
||||
device,
|
||||
default_devices_listener: None,
|
||||
is_app_specific: true,
|
||||
app_id: Some(app.object_id),
|
||||
excluded_processes: Vec::new(),
|
||||
active_stream: None,
|
||||
audio_callback: None,
|
||||
original_audio_stats: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Creates a new AggregateDeviceManager for global audio with option to
|
||||
/// exclude processes
|
||||
pub fn new_global(excluded_processes: &[AudioObjectID]) -> Result<Self> {
|
||||
let device = AggregateDevice::create_global_tap_but_exclude_processes(excluded_processes)?;
|
||||
Ok(Self {
|
||||
device,
|
||||
default_devices_listener: None,
|
||||
is_app_specific: false,
|
||||
app_id: None,
|
||||
excluded_processes: excluded_processes.to_vec(),
|
||||
active_stream: None,
|
||||
audio_callback: None,
|
||||
original_audio_stats: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// This sets up the initial stream and listeners.
|
||||
pub fn start_capture(&mut self, audio_stream_callback: AudioCallback) -> Result<()> {
|
||||
// Store the callback for potential device switch later
|
||||
self.audio_callback = Some(audio_stream_callback.clone());
|
||||
|
||||
// Create a shared reference for the active stream
|
||||
let stream_mutex = Arc::new(std::sync::Mutex::new(None));
|
||||
self.active_stream = Some(stream_mutex.clone());
|
||||
|
||||
// Start the initial stream
|
||||
// Pass the initially determined consistent audio stats
|
||||
let original_audio_stats = self.device.get_aggregate_device_stats().unwrap_or(AudioStats {
|
||||
sample_rate: 48000.0, // Match fallback in setup_device_change_listeners
|
||||
channels: 2,
|
||||
});
|
||||
self.original_audio_stats = Some(original_audio_stats); // Store for listener use
|
||||
|
||||
let initial_audio_tap_stream = self.device.start(audio_stream_callback.clone(), original_audio_stats)?; // Pass clone of callback
|
||||
|
||||
// Setup device change listeners AFTER getting initial stats and stream
|
||||
self.setup_device_change_listeners()?;
|
||||
|
||||
// Store a reference to the stream
|
||||
if let Ok(mut stream_guard) = stream_mutex.lock() {
|
||||
*stream_guard = Some(initial_audio_tap_stream);
|
||||
} else {
|
||||
println!("DEBUG: Failed to lock stream_mutex to store AudioTapStream reference");
|
||||
// If we can't store the initial stream, something is wrong.
|
||||
// Attempt to stop the stream we just created? Or just return error?
|
||||
// For now, return an error.
|
||||
return Err(napi::Error::from_reason(
|
||||
"Failed to lock internal stream mutex during startup",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sets up listeners for default device changes
|
||||
fn setup_device_change_listeners(&mut self) -> Result<()> {
|
||||
// We need to clean up any existing listeners first
|
||||
self.cleanup_device_listeners();
|
||||
|
||||
// Create a weak reference to self to avoid circular references
|
||||
let stream_arc = self.active_stream.clone();
|
||||
let callback_arc = self.audio_callback.clone();
|
||||
let is_app_specific = self.is_app_specific;
|
||||
let app_id = self.app_id;
|
||||
let excluded_processes = self.excluded_processes.clone();
|
||||
|
||||
// Retrieve the stored original audio stats
|
||||
let Some(original_audio_stats) = self.original_audio_stats else {
|
||||
return Err(napi::Error::from_reason(
|
||||
"Internal error: Original audio stats not available for listener.",
|
||||
));
|
||||
};
|
||||
|
||||
// Create a block that will handle device changes
|
||||
let device_changed_block = RcBlock::new(move |_in_number_addresses: u32, _in_addresses: *mut c_void| {
|
||||
// Skip if we don't have all required information
|
||||
let Some(stream_mutex) = stream_arc.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let Some(callback) = callback_arc.as_ref() else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Try to lock the stream mutex
|
||||
let Ok(mut stream_guard) = stream_mutex.lock() else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Create a new device with updated default devices
|
||||
let result: Result<AggregateDevice> = {
|
||||
if is_app_specific {
|
||||
if let Some(id) = app_id {
|
||||
// For device change listener, we need to create a minimal
|
||||
// ApplicationInfo We don't have the name here, so we'll
|
||||
// use an empty string
|
||||
let app = ApplicationInfo::new(id as i32, String::new(), id);
|
||||
AggregateDevice::new(&app)
|
||||
} else {
|
||||
Err(CoreAudioError::CreateProcessTapFailed(0).into())
|
||||
}
|
||||
} else {
|
||||
AggregateDevice::create_global_tap_but_exclude_processes(&excluded_processes)
|
||||
}
|
||||
};
|
||||
|
||||
// If we successfully created a new device, stop the old stream and start
|
||||
// a new one
|
||||
match result {
|
||||
Ok(mut new_device) => {
|
||||
// Stop and drop the old stream if it exists
|
||||
if let Some(mut old_stream) = stream_guard.take() {
|
||||
// Explicitly drop the old stream's Box before creating the new
|
||||
// device. The drop implementation handles cleanup.
|
||||
// We call stop() directly.
|
||||
let stop_result = old_stream.stop();
|
||||
match stop_result {
|
||||
Ok(_) => {}
|
||||
Err(e) => println!("DEBUG: Error stopping old stream (proceeding anyway): {e}"),
|
||||
};
|
||||
drop(old_stream); // Ensure it's dropped now
|
||||
}
|
||||
|
||||
match new_device.start(callback.clone(), original_audio_stats) {
|
||||
Ok(new_stream) => {
|
||||
// Use the existing stream_guard which already holds the lock
|
||||
*stream_guard = Some(new_stream);
|
||||
}
|
||||
Err(e) => {
|
||||
println!("DEBUG: Failed to start new stream: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
println!("DEBUG: Failed to create new device: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Create pointers to the device_changed_block that can be used in C
|
||||
// functions
|
||||
let block_ptr = &*device_changed_block as *const Block<dyn Fn(u32, *mut c_void)>;
|
||||
let block_ptr_cast = block_ptr.cast_mut().cast();
|
||||
|
||||
// Register listeners for both input and output device changes
|
||||
unsafe {
|
||||
let status = AudioObjectAddPropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyDefaultInputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
block_ptr_cast,
|
||||
);
|
||||
|
||||
if status != 0 {
|
||||
println!("DEBUG: Failed to register input device listener, status: {status}");
|
||||
return Err(CoreAudioError::AddPropertyListenerBlockFailed(status).into());
|
||||
}
|
||||
|
||||
let status = AudioObjectAddPropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
block_ptr_cast,
|
||||
);
|
||||
|
||||
if status != 0 {
|
||||
println!("DEBUG: Failed to register output device listener, status: {status}");
|
||||
// Clean up the first listener if the second one fails
|
||||
AudioObjectRemovePropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyDefaultInputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
block_ptr_cast,
|
||||
);
|
||||
return Err(CoreAudioError::AddPropertyListenerBlockFailed(status).into());
|
||||
}
|
||||
}
|
||||
|
||||
// Store the listener pointer for cleanup
|
||||
self.default_devices_listener = Some(block_ptr_cast);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Cleans up device change listeners
|
||||
fn cleanup_device_listeners(&mut self) {
|
||||
if let Some(listener) = self.default_devices_listener.take() {
|
||||
unsafe {
|
||||
// Add a runtime check to ensure we're not in shutdown
|
||||
let is_system_shutting_down = std::panic::catch_unwind(|| {
|
||||
// Try a simple CoreAudio API call to see if the system is still
|
||||
// responsive
|
||||
let mut size: u32 = 0;
|
||||
AudioObjectGetPropertyDataSize(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyDefaultInputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
0,
|
||||
ptr::null(),
|
||||
&mut size,
|
||||
)
|
||||
})
|
||||
.is_err();
|
||||
|
||||
if is_system_shutting_down {
|
||||
// Don't try to remove listeners if the system is shutting down
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove input device change listener
|
||||
let status = AudioObjectRemovePropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyDefaultInputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
listener,
|
||||
);
|
||||
if status != 0 {
|
||||
// Don't log errors during shutdown to avoid additional issues
|
||||
}
|
||||
|
||||
let status = AudioObjectRemovePropertyListenerBlock(
|
||||
kAudioObjectSystemObject,
|
||||
&AudioObjectPropertyAddress {
|
||||
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
},
|
||||
ptr::null_mut(),
|
||||
listener,
|
||||
);
|
||||
if status != 0 {
|
||||
// Don't log errors during shutdown to avoid additional issues
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops the active stream and cleans up listeners.
|
||||
pub fn stop_capture(&mut self) -> Result<()> {
|
||||
self.cleanup_device_listeners();
|
||||
|
||||
let stream_to_stop = if let Some(stream_mutex) = &self.active_stream {
|
||||
if let Ok(mut stream_guard) = stream_mutex.lock() {
|
||||
stream_guard.take() // Take ownership from the Option<Arc<Mutex<...>>>
|
||||
} else {
|
||||
println!("DEBUG: Failed to lock stream mutex during stop");
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if let Some(mut stream) = stream_to_stop {
|
||||
match stream.stop() {
|
||||
Ok(_) => {}
|
||||
Err(e) => println!("DEBUG: Error stopping stream in stop_capture (ignored): {e}"),
|
||||
}
|
||||
// Explicitly drop here after stopping
|
||||
drop(stream);
|
||||
}
|
||||
|
||||
// Clear related fields
|
||||
self.active_stream = None;
|
||||
self.audio_callback = None;
|
||||
self.original_audio_stats = None;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Gets the stats of the currently active stream, if any.
|
||||
pub fn get_current_stats(&self) -> Option<AudioStats> {
|
||||
if let Some(stream_mutex) = &self.active_stream {
|
||||
if let Ok(stream_guard) = stream_mutex.lock() {
|
||||
// Borrow the stream Option, then map to get stats
|
||||
stream_guard.as_ref().map(|stream| stream.audio_stats)
|
||||
} else {
|
||||
println!("DEBUG: Failed to lock stream mutex for get_current_stats");
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets the actual sample rate of the currently active stream's output
|
||||
/// device.
|
||||
pub fn get_current_actual_sample_rate(&self) -> Result<Option<f64>> {
|
||||
let maybe_stream_ref = if let Some(stream_mutex) = &self.active_stream {
|
||||
match stream_mutex.lock() {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => {
|
||||
println!("DEBUG: Failed to lock stream mutex for get_current_actual_sample_rate");
|
||||
// Return Ok(None) or an error? Let's return None.
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Ok(None); // No active stream manager
|
||||
};
|
||||
|
||||
if let Some(stream) = maybe_stream_ref.as_ref() {
|
||||
// Call the existing non-napi method on AudioTapStream
|
||||
match stream.get_actual_sample_rate() {
|
||||
Ok(rate) => Ok(Some(rate)),
|
||||
Err(e) => {
|
||||
println!("DEBUG: Error getting actual sample rate from stream: {e}");
|
||||
// Propagate the error
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(None) // No active stream
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AggregateDeviceManager {
|
||||
fn drop(&mut self) {
|
||||
// Call stop_capture which handles listener cleanup and stream stopping
|
||||
match self.stop_capture() {
|
||||
Ok(_) => {}
|
||||
Err(e) => println!("DEBUG: Error during stop_capture in Drop (ignored): {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AudioCaptureSession {
|
||||
// Use Option<Box<...>> to allow taking ownership in stop()
|
||||
manager: Option<Box<AggregateDeviceManager>>,
|
||||
sample_rate: Option<f64>,
|
||||
channels: Option<u32>,
|
||||
}
|
||||
|
||||
impl AudioCaptureSession {
|
||||
pub(crate) fn new(manager: Box<AggregateDeviceManager>) -> Self {
|
||||
Self {
|
||||
manager: Some(manager),
|
||||
sample_rate: None,
|
||||
channels: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stop(&mut self) -> Result<()> {
|
||||
if let Some(manager) = self.manager.take() {
|
||||
// Cache the stats before dropping
|
||||
if let Some(stats) = manager.get_current_stats() {
|
||||
self.sample_rate = Some(stats.sample_rate);
|
||||
self.channels = Some(stats.channels);
|
||||
}
|
||||
|
||||
// Drop the manager
|
||||
drop(manager);
|
||||
Ok(())
|
||||
} else {
|
||||
println!("DEBUG: AudioCaptureSession.stop() called, but manager was already taken");
|
||||
// Return Ok even if called multiple times, idempotent behavior
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_sample_rate(&self) -> Result<f64> {
|
||||
if let Some(manager) = &self.manager {
|
||||
manager
|
||||
.get_current_stats()
|
||||
.map(|stats| stats.sample_rate)
|
||||
.ok_or_else(|| napi::Error::from_reason("No active audio stream to get sample rate from"))
|
||||
} else if let Some(cached_rate) = self.sample_rate {
|
||||
// Return cached value when session is stopped
|
||||
Ok(cached_rate)
|
||||
} else {
|
||||
Err(napi::Error::from_reason(
|
||||
"Audio session is stopped and no cached sample rate available",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_channels(&self) -> Result<u32> {
|
||||
if let Some(manager) = &self.manager {
|
||||
manager
|
||||
.get_current_stats()
|
||||
.map(|stats| stats.channels)
|
||||
.ok_or_else(|| napi::Error::from_reason("No active audio stream to get channels from"))
|
||||
} else if let Some(cached_channels) = self.channels {
|
||||
// Return cached value when session is stopped
|
||||
Ok(cached_channels)
|
||||
} else {
|
||||
Err(napi::Error::from_reason(
|
||||
"Audio session is stopped and no cached channels available",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_actual_sample_rate(&self) -> Result<f64> {
|
||||
if let Some(manager) = &self.manager {
|
||||
manager
|
||||
.get_current_actual_sample_rate()? // Propagate CoreAudioError
|
||||
.ok_or_else(|| napi::Error::from_reason("No active audio stream to get actual sample rate from"))
|
||||
} else if let Some(cached_rate) = self.sample_rate {
|
||||
// Return cached sample rate as the best approximation when session is
|
||||
// stopped
|
||||
Ok(cached_rate)
|
||||
} else {
|
||||
Err(napi::Error::from_reason(
|
||||
"Audio session is stopped and no cached sample rate available",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the manager is dropped if the session object is dropped without
|
||||
// calling stop()
|
||||
impl Drop for AudioCaptureSession {
|
||||
fn drop(&mut self) {
|
||||
// Automatically calls drop on self.manager if it's Some
|
||||
if let Some(manager) = self.manager.take() {
|
||||
drop(manager);
|
||||
}
|
||||
}
|
||||
}
|
||||
226
packages/frontend/native/media_capture/src/macos/utils.rs
Normal file
226
packages/frontend/native/media_capture/src/macos/utils.rs
Normal file
@@ -0,0 +1,226 @@
|
||||
use std::{cell::RefCell, collections::HashMap, ffi::c_void, mem::size_of};
|
||||
|
||||
use core_foundation::string::CFString;
|
||||
use coreaudio::sys::{
|
||||
AudioObjectGetPropertyData, AudioObjectID, AudioObjectPropertyAddress, kAudioObjectPropertyElementMain,
|
||||
kAudioObjectPropertyScopeGlobal,
|
||||
};
|
||||
use rubato::{FastFixedIn, PolynomialDegree, Resampler};
|
||||
|
||||
use crate::error::CoreAudioError;
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// A simple wrapper that buffers incoming planar frames so that we always feed
|
||||
// the Rubato resampler its preferred fixed block-length. This avoids the
|
||||
// artefacts caused by recreating the resampler every callback.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
const RESAMPLER_INPUT_CHUNK: usize = 1024; // samples per channel
|
||||
|
||||
struct BufferedResampler {
|
||||
resampler: FastFixedIn<f32>,
|
||||
channels: usize,
|
||||
fifo: Vec<Vec<f32>>, // per-channel queue
|
||||
initial_output_discarded: bool, // Flag to track if the first output has been discarded
|
||||
}
|
||||
|
||||
impl BufferedResampler {
|
||||
fn new(from_sr: f64, to_sr: f64, channels: usize) -> Self {
|
||||
let ratio = to_sr / from_sr;
|
||||
let resampler = FastFixedIn::<f32>::new(
|
||||
ratio,
|
||||
1.0, // max_resample_ratio_relative (must be >= 1.0, use 1.0 for fixed ratio)
|
||||
PolynomialDegree::Linear, // Use Linear interpolation quality
|
||||
RESAMPLER_INPUT_CHUNK,
|
||||
channels,
|
||||
)
|
||||
.expect("Failed to create FastFixedIn resampler (5-arg attempt)");
|
||||
|
||||
BufferedResampler {
|
||||
resampler,
|
||||
channels,
|
||||
fifo: vec![Vec::<f32>::new(); channels],
|
||||
initial_output_discarded: false,
|
||||
}
|
||||
}
|
||||
|
||||
// feed planar samples; returns interleaved output (may be empty if not
|
||||
// enough samples accumulated yet).
|
||||
fn feed(&mut self, planar_in: &[Vec<f32>]) -> Vec<f32> {
|
||||
// Append incoming to fifo
|
||||
for (ch, data) in planar_in.iter().enumerate() {
|
||||
self.fifo[ch].extend_from_slice(data);
|
||||
}
|
||||
|
||||
let mut interleaved_out: Vec<f32> = Vec::new();
|
||||
|
||||
while self.fifo[0].len() >= RESAMPLER_INPUT_CHUNK {
|
||||
// Drain exactly one chunk per channel
|
||||
let mut chunk: Vec<Vec<f32>> = Vec::with_capacity(self.channels);
|
||||
for ch in 0..self.channels {
|
||||
let tail = self.fifo[ch].drain(..RESAMPLER_INPUT_CHUNK).collect::<Vec<_>>();
|
||||
chunk.push(tail);
|
||||
}
|
||||
|
||||
if let Ok(out_blocks) = self.resampler.process(&chunk, None) {
|
||||
// out_blocks is Vec<Vec<f32>> planar
|
||||
if !out_blocks.is_empty() && out_blocks.len() == self.channels {
|
||||
// Check if we should discard the initial output
|
||||
if !self.initial_output_discarded {
|
||||
self.initial_output_discarded = true;
|
||||
} else {
|
||||
// interleave
|
||||
let out_len = out_blocks[0].len();
|
||||
#[allow(clippy::needless_range_loop)]
|
||||
for i in 0..out_len {
|
||||
// apply clippy lint suggestion would regress performance
|
||||
for ch in 0..self.channels {
|
||||
interleaved_out.push(out_blocks[ch][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interleaved_out
|
||||
}
|
||||
}
|
||||
|
||||
// thread-local cache so that each audio‐tap thread keeps its own resamplers
|
||||
thread_local! {
|
||||
static RESAMPLER_CACHE: RefCell<HashMap<(u32,u32,usize), BufferedResampler>> = RefCell::new(HashMap::new());
|
||||
}
|
||||
|
||||
pub fn cfstring_from_bytes_with_nul(bytes: &[u8]) -> CFString {
|
||||
CFString::new(
|
||||
unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(bytes) }
|
||||
.to_string_lossy()
|
||||
.as_ref(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_global_main_property<T: Sized>(
|
||||
id: AudioObjectID,
|
||||
selector: u32,
|
||||
data: *mut T,
|
||||
) -> Result<(), CoreAudioError> {
|
||||
unsafe {
|
||||
let address = AudioObjectPropertyAddress {
|
||||
mSelector: selector,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain,
|
||||
};
|
||||
|
||||
let mut data_size = size_of::<T>() as u32;
|
||||
|
||||
let status = AudioObjectGetPropertyData(
|
||||
id,
|
||||
&address as *const AudioObjectPropertyAddress,
|
||||
0,
|
||||
std::ptr::null(),
|
||||
&mut data_size,
|
||||
data.cast(),
|
||||
);
|
||||
|
||||
if status != 0 {
|
||||
return Err(CoreAudioError::GetPropertyDataFailed(status));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process_audio_frame(
|
||||
m_data: *mut c_void,
|
||||
m_data_byte_size: u32,
|
||||
m_number_channels: u32,
|
||||
current_sample_rate: f64,
|
||||
target_sample_rate: f64,
|
||||
) -> Option<Vec<f32>> {
|
||||
// Only create slice if we have valid data
|
||||
|
||||
if m_data.is_null() || m_data_byte_size == 0 {
|
||||
return None;
|
||||
}
|
||||
// Calculate total number of samples (total bytes / bytes per sample)
|
||||
let total_samples = m_data_byte_size as usize / 4; // 4 bytes per f32
|
||||
|
||||
// Create a slice of all samples
|
||||
let samples: &[f32] = unsafe { std::slice::from_raw_parts(m_data.cast::<f32>(), total_samples) };
|
||||
|
||||
// Check the channel count and data format
|
||||
let channel_count = m_number_channels as usize;
|
||||
|
||||
// If the audio has two or more channels, keep (at most) the first two
|
||||
// channels and return them in interleaved stereo format. Otherwise keep
|
||||
// mono as-is.
|
||||
|
||||
let interleaved_samples: Vec<f32> = if channel_count >= 2 {
|
||||
// Split interleaved input into the first two channels (L, R)
|
||||
let mut left: Vec<f32> = Vec::with_capacity(total_samples / channel_count);
|
||||
let mut right: Vec<f32> = Vec::with_capacity(total_samples / channel_count);
|
||||
|
||||
for chunk in samples.chunks(channel_count) {
|
||||
// SAFETY: chunk has at least 2 items because channel_count >= 2
|
||||
left.push(chunk[0]);
|
||||
right.push(chunk[1]);
|
||||
}
|
||||
|
||||
if current_sample_rate != target_sample_rate {
|
||||
// Use (or create) a persistent BufferedResampler
|
||||
|
||||
RESAMPLER_CACHE.with(|cache| {
|
||||
let mut map = cache.borrow_mut();
|
||||
let key = (current_sample_rate as u32, target_sample_rate as u32, 2usize);
|
||||
let resampler = map
|
||||
.entry(key)
|
||||
.or_insert_with(|| BufferedResampler::new(current_sample_rate, target_sample_rate, 2));
|
||||
resampler.feed(&[left, right])
|
||||
})
|
||||
} else {
|
||||
// No resampling needed, just interleave existing left/right data
|
||||
let mut interleaved: Vec<f32> = Vec::with_capacity(left.len() * 2);
|
||||
for i in 0..left.len() {
|
||||
interleaved.push(left[i]);
|
||||
interleaved.push(right[i]);
|
||||
}
|
||||
|
||||
interleaved
|
||||
}
|
||||
} else {
|
||||
// Mono path – behave as before (optionally resample)
|
||||
let mut mono_samples = samples.to_vec();
|
||||
|
||||
if current_sample_rate != target_sample_rate {
|
||||
let out_vec = RESAMPLER_CACHE.with(|cache| {
|
||||
let mut map = cache.borrow_mut();
|
||||
let key = (current_sample_rate as u32, target_sample_rate as u32, 1usize);
|
||||
let resampler = map
|
||||
.entry(key)
|
||||
.or_insert_with(|| BufferedResampler::new(current_sample_rate, target_sample_rate, 1));
|
||||
resampler.feed(&[mono_samples])
|
||||
});
|
||||
// resampler returns interleaved (1 channel) but we still need planar mono
|
||||
// (vector of samples) before upmix; since feed returns interleaved single
|
||||
// channel, it is planar already.
|
||||
mono_samples = out_vec;
|
||||
}
|
||||
|
||||
// Upmix mono to stereo by duplicating each sample so that mixing with
|
||||
// interleaved stereo streams keeps channel counts aligned.
|
||||
let mut stereo_samples: Vec<f32> = Vec::with_capacity(mono_samples.len() * 2);
|
||||
for s in &mono_samples {
|
||||
stereo_samples.push(*s);
|
||||
stereo_samples.push(*s);
|
||||
}
|
||||
|
||||
stereo_samples
|
||||
};
|
||||
|
||||
if interleaved_samples.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(interleaved_samples)
|
||||
}
|
||||
}
|
||||
1256
packages/frontend/native/media_capture/src/recording.rs
Normal file
1256
packages/frontend/native/media_capture/src/recording.rs
Normal file
File diff suppressed because it is too large
Load Diff
90
packages/frontend/native/media_capture/src/windows/README.md
Normal file
90
packages/frontend/native/media_capture/src/windows/README.md
Normal file
@@ -0,0 +1,90 @@
|
||||
# Windows Audio Recording
|
||||
|
||||
This module provides Windows-specific audio recording functionality using the Windows Audio Session API (WASAPI).
|
||||
|
||||
## Features
|
||||
|
||||
- **Microphone Activity Detection**: Monitor when applications are using the microphone
|
||||
- **Process Identification**: Identify which process is using the microphone
|
||||
- **Real-time Notifications**: Get callbacks when microphone usage starts/stops
|
||||
|
||||
## Usage
|
||||
|
||||
### MicrophoneListener
|
||||
|
||||
The `MicrophoneListener` class provides real-time monitoring of microphone usage:
|
||||
|
||||
```typescript
|
||||
import { MicrophoneListener } from '@affine/native';
|
||||
|
||||
const listener = new MicrophoneListener(
|
||||
(isRunning: boolean, processName: string) => {
|
||||
console.log(
|
||||
`Microphone ${isRunning ? 'started' : 'stopped'} by ${processName}`
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// Check current status
|
||||
console.log('Is microphone currently active:', listener.is_running());
|
||||
```
|
||||
|
||||
### Callback Parameters
|
||||
|
||||
The callback receives two parameters:
|
||||
|
||||
- `isRunning: boolean` - Whether the microphone is currently active
|
||||
- `processName: string` - Name of the process using the microphone
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Audio Session Monitoring
|
||||
|
||||
The implementation uses Windows Audio Session API to:
|
||||
|
||||
1. **Enumerate Audio Sessions**: Get all active audio sessions
|
||||
2. **Monitor Session State**: Track when sessions become active/inactive
|
||||
3. **Process Identification**: Map audio sessions to process names
|
||||
4. **Event Handling**: Provide real-time notifications
|
||||
|
||||
### COM Initialization
|
||||
|
||||
The module automatically initializes COM (Component Object Model) with `COINIT_MULTITHREADED` for proper Windows API interaction.
|
||||
|
||||
### Error Handling
|
||||
|
||||
All Windows API errors are wrapped in `WindowsAudioError` enum and converted to NAPI errors for JavaScript consumption.
|
||||
|
||||
## Cross-Platform Compatibility
|
||||
|
||||
This Windows implementation maintains API compatibility with the macOS version, providing the same JavaScript interface while using Windows-specific APIs underneath.
|
||||
|
||||
## Platform Requirements
|
||||
|
||||
- Windows 10 or later
|
||||
- Microphone access permissions
|
||||
- Audio devices available
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `windows` crate v0.61 with Audio and Process features
|
||||
- `windows-core` crate v0.61
|
||||
- `napi` and `napi-derive` for JavaScript bindings
|
||||
|
||||
## Technical Notes
|
||||
|
||||
### Thread Safety
|
||||
|
||||
The implementation uses thread-safe callbacks to JavaScript with `ThreadsafeFunction<(bool, String), ()>` to ensure proper communication between the Windows audio session monitoring thread and the JavaScript runtime.
|
||||
|
||||
### Process Name Resolution
|
||||
|
||||
Process names are resolved using Windows APIs:
|
||||
|
||||
- `GetModuleFileNameExW` for full executable path
|
||||
- `GetProcessImageFileNameW` as fallback
|
||||
- Automatic extraction of filename from full path
|
||||
|
||||
### Session Filtering
|
||||
|
||||
The implementation automatically filters out system audio sessions (like `AudioSrv`) to focus on user applications.
|
||||
@@ -0,0 +1,491 @@
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
thread::JoinHandle,
|
||||
};
|
||||
|
||||
use cpal::{
|
||||
SampleRate,
|
||||
traits::{DeviceTrait, HostTrait, StreamTrait},
|
||||
};
|
||||
use crossbeam_channel::unbounded;
|
||||
use napi::{Error, Status, bindgen_prelude::Result};
|
||||
use napi_derive::napi;
|
||||
use rubato::{FastFixedIn, PolynomialDegree, Resampler};
|
||||
|
||||
use crate::audio_callback::AudioCallback;
|
||||
|
||||
const RESAMPLER_INPUT_CHUNK: usize = 1024; // samples per channel
|
||||
const TARGET_FRAME_SIZE: usize = 1024; // frame size returned to JS (in mono samples)
|
||||
|
||||
struct BufferedResampler {
|
||||
resampler: FastFixedIn<f32>,
|
||||
channels: usize,
|
||||
fifo: Vec<Vec<f32>>, // per-channel queue
|
||||
initial_output_discarded: bool, // Flag to discard first output block (warm-up)
|
||||
}
|
||||
|
||||
impl BufferedResampler {
|
||||
fn new(from_sr: f64, to_sr: f64, channels: usize) -> Self {
|
||||
let ratio = to_sr / from_sr;
|
||||
let resampler = FastFixedIn::<f32>::new(
|
||||
ratio,
|
||||
1.0, // max_resample_ratio_relative (>= 1.0, fixed ratio)
|
||||
PolynomialDegree::Linear, // balance quality/perf
|
||||
RESAMPLER_INPUT_CHUNK,
|
||||
channels,
|
||||
)
|
||||
.expect("Failed to create FastFixedIn resampler");
|
||||
|
||||
Self {
|
||||
resampler,
|
||||
channels,
|
||||
fifo: vec![Vec::<f32>::new(); channels],
|
||||
initial_output_discarded: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn append_output(&mut self, out_blocks: &[Vec<f32>], interleaved_out: &mut Vec<f32>, final_flush: bool) {
|
||||
if out_blocks.is_empty() || out_blocks.len() != self.channels {
|
||||
return;
|
||||
}
|
||||
|
||||
if !self.initial_output_discarded && !final_flush {
|
||||
self.initial_output_discarded = true;
|
||||
return;
|
||||
}
|
||||
|
||||
self.initial_output_discarded = true;
|
||||
let out_len = out_blocks[0].len();
|
||||
for i in 0..out_len {
|
||||
for channel in out_blocks.iter().take(self.channels) {
|
||||
interleaved_out.push(channel[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process_chunk(&mut self, chunk: Vec<Vec<f32>>, interleaved_out: &mut Vec<f32>, final_flush: bool) {
|
||||
if let Ok(out_blocks) = self.resampler.process(&chunk, None) {
|
||||
self.append_output(&out_blocks, interleaved_out, final_flush);
|
||||
}
|
||||
}
|
||||
|
||||
// Feed planar samples; returns interleaved output (may be empty)
|
||||
fn feed(&mut self, planar_in: &[Vec<f32>]) -> Vec<f32> {
|
||||
for (ch, data) in planar_in.iter().enumerate() {
|
||||
if ch < self.fifo.len() {
|
||||
self.fifo[ch].extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
let mut interleaved_out = Vec::new();
|
||||
|
||||
while self.fifo[0].len() >= RESAMPLER_INPUT_CHUNK {
|
||||
// Take exactly RESAMPLER_INPUT_CHUNK per channel
|
||||
let mut chunk: Vec<Vec<f32>> = Vec::with_capacity(self.channels);
|
||||
for ch in 0..self.channels {
|
||||
let tail: Vec<f32> = self.fifo[ch].drain(..RESAMPLER_INPUT_CHUNK).collect();
|
||||
chunk.push(tail);
|
||||
}
|
||||
|
||||
self.process_chunk(chunk, &mut interleaved_out, false);
|
||||
}
|
||||
|
||||
interleaved_out
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Vec<f32> {
|
||||
let mut interleaved_out = Vec::new();
|
||||
if self.fifo.first().is_none_or(|channel| channel.is_empty()) {
|
||||
return interleaved_out;
|
||||
}
|
||||
|
||||
let mut chunk: Vec<Vec<f32>> = Vec::with_capacity(self.channels);
|
||||
for ch in 0..self.channels {
|
||||
let mut tail = std::mem::take(&mut self.fifo[ch]);
|
||||
tail.resize(RESAMPLER_INPUT_CHUNK, 0.0);
|
||||
chunk.push(tail);
|
||||
}
|
||||
self.process_chunk(chunk, &mut interleaved_out, true);
|
||||
interleaved_out
|
||||
}
|
||||
}
|
||||
|
||||
fn to_mono(frame: &[f32]) -> f32 {
|
||||
if frame.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let sum: f32 = frame.iter().filter(|s| s.is_finite()).copied().sum();
|
||||
let mono = if frame.len() == 1 {
|
||||
sum // already mono, no reduction needed
|
||||
} else {
|
||||
// For multi-channel, take the sum but don't divide by channel count
|
||||
// This preserves more energy while still avoiding simple doubling
|
||||
sum * 0.7 // slight reduction to prevent clipping, but preserve energy
|
||||
};
|
||||
mono.clamp(-1.0, 1.0)
|
||||
}
|
||||
|
||||
fn mix(a: &[f32], b: &[f32]) -> Vec<f32> {
|
||||
let min_len = a.len().min(b.len());
|
||||
if min_len == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
const MIC_GAIN: f32 = 3.0; // Higher gain for microphone input
|
||||
const LOOPBACK_GAIN: f32 = 1.5; // Moderate gain for loopback
|
||||
const OVERALL_GAIN: f32 = 1.2; // Final boost
|
||||
|
||||
a.iter()
|
||||
.take(min_len)
|
||||
.zip(b.iter().take(min_len))
|
||||
.map(|(x, y)| {
|
||||
let x_clean = if x.is_finite() { *x } else { 0.0 };
|
||||
let y_clean = if y.is_finite() { *y } else { 0.0 };
|
||||
|
||||
// Apply individual gains to mic (x) and loopback (y), then mix
|
||||
let mic_boosted = x_clean * MIC_GAIN;
|
||||
let loopback_boosted = y_clean * LOOPBACK_GAIN;
|
||||
let mixed = (mic_boosted + loopback_boosted) * OVERALL_GAIN;
|
||||
|
||||
// Soft limiting using tanh for more natural sound than hard clipping
|
||||
if mixed.abs() > 1.0 {
|
||||
mixed.signum() * (1.0 - (-mixed.abs()).exp())
|
||||
} else {
|
||||
mixed
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
struct AudioBuffer {
|
||||
data: Vec<f32>,
|
||||
}
|
||||
|
||||
fn extend_post_buffer(post_buffer: &mut Vec<f32>, mono_samples: Vec<f32>, resampler: &mut Option<BufferedResampler>) {
|
||||
if let Some(resampler) = resampler.as_mut() {
|
||||
post_buffer.extend(resampler.feed(&[mono_samples]));
|
||||
} else {
|
||||
post_buffer.extend(mono_samples);
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_mixed_frames(
|
||||
post_mic: &mut Vec<f32>,
|
||||
post_lb: &mut Vec<f32>,
|
||||
audio_buffer_callback: &AudioCallback,
|
||||
final_flush: bool,
|
||||
) {
|
||||
while post_mic.len() >= TARGET_FRAME_SIZE && post_lb.len() >= TARGET_FRAME_SIZE {
|
||||
let mic_chunk: Vec<f32> = post_mic.drain(..TARGET_FRAME_SIZE).collect();
|
||||
let lb_chunk: Vec<f32> = post_lb.drain(..TARGET_FRAME_SIZE).collect();
|
||||
let mixed = mix(&mic_chunk, &lb_chunk);
|
||||
if !mixed.is_empty() {
|
||||
audio_buffer_callback.call(mixed);
|
||||
}
|
||||
}
|
||||
|
||||
if final_flush && !post_mic.is_empty() && !post_lb.is_empty() {
|
||||
let tail_len = post_mic.len().min(post_lb.len());
|
||||
let mic_chunk: Vec<f32> = post_mic.drain(..tail_len).collect();
|
||||
let lb_chunk: Vec<f32> = post_lb.drain(..tail_len).collect();
|
||||
let mixed = mix(&mic_chunk, &lb_chunk);
|
||||
if !mixed.is_empty() {
|
||||
audio_buffer_callback.call(mixed);
|
||||
}
|
||||
post_mic.clear();
|
||||
post_lb.clear();
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AudioCaptureSession {
|
||||
mic_stream: Option<cpal::Stream>,
|
||||
lb_stream: Option<cpal::Stream>,
|
||||
stop_requested: Arc<AtomicBool>,
|
||||
sample_rate: SampleRate,
|
||||
channels: u32,
|
||||
jh: Option<JoinHandle<()>>, // background mixing thread
|
||||
}
|
||||
|
||||
fn teardown_audio_capture_resources<S, F>(
|
||||
mic_stream: &mut Option<S>,
|
||||
lb_stream: &mut Option<S>,
|
||||
stop_requested: &Arc<AtomicBool>,
|
||||
jh: &mut Option<JoinHandle<()>>,
|
||||
pause_stream: F,
|
||||
) -> Result<()>
|
||||
where
|
||||
F: Fn(&S) -> std::result::Result<(), String>,
|
||||
{
|
||||
if mic_stream.is_none() && lb_stream.is_none() && jh.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mic_stream = mic_stream.take();
|
||||
let lb_stream = lb_stream.take();
|
||||
let jh = jh.take();
|
||||
|
||||
let mut pause_errors = Vec::new();
|
||||
|
||||
if let Some(stream) = mic_stream.as_ref() {
|
||||
if let Err(error) = pause_stream(stream) {
|
||||
pause_errors.push(format!("pause mic stream: {error}"));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(stream) = lb_stream.as_ref() {
|
||||
if let Err(error) = pause_stream(stream) {
|
||||
pause_errors.push(format!("pause loopback stream: {error}"));
|
||||
}
|
||||
}
|
||||
|
||||
stop_requested.store(true, Ordering::SeqCst);
|
||||
|
||||
drop(mic_stream);
|
||||
drop(lb_stream);
|
||||
|
||||
if let Some(jh) = jh {
|
||||
let _ = jh.join(); // ignore poison
|
||||
}
|
||||
|
||||
if pause_errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::new(Status::GenericFailure, pause_errors.join("; ")))
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioCaptureSession {
|
||||
pub fn get_sample_rate(&self) -> f64 {
|
||||
self.sample_rate.0 as f64
|
||||
}
|
||||
|
||||
pub fn get_channels(&self) -> u32 {
|
||||
self.channels
|
||||
}
|
||||
|
||||
pub fn get_actual_sample_rate(&self) -> f64 {
|
||||
// For CPAL we always operate at the target rate which is sample_rate
|
||||
self.sample_rate.0 as f64
|
||||
}
|
||||
|
||||
pub fn stop(&mut self) -> Result<()> {
|
||||
teardown_audio_capture_resources(
|
||||
&mut self.mic_stream,
|
||||
&mut self.lb_stream,
|
||||
&self.stop_requested,
|
||||
&mut self.jh,
|
||||
|stream| stream.pause().map_err(|error| error.to_string()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AudioCaptureSession {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.stop(); // Ensure cleanup even if JS forgets to call stop()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_recording(
|
||||
audio_buffer_callback: AudioCallback,
|
||||
target_sample_rate: Option<SampleRate>,
|
||||
) -> Result<AudioCaptureSession> {
|
||||
let available_hosts = cpal::available_hosts();
|
||||
let host_id = available_hosts
|
||||
.first()
|
||||
.ok_or_else(|| Error::new(Status::GenericFailure, "No CPAL hosts available"))?;
|
||||
|
||||
let host = cpal::host_from_id(*host_id).map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
|
||||
|
||||
let mic = host
|
||||
.default_input_device()
|
||||
.ok_or_else(|| Error::new(Status::GenericFailure, "No default input device"))?;
|
||||
let loopback_device = host
|
||||
.default_output_device()
|
||||
.ok_or_else(|| Error::new(Status::GenericFailure, "No default output/loopback device"))?;
|
||||
|
||||
let mic_config = mic
|
||||
.default_input_config()
|
||||
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
|
||||
let lb_config = loopback_device
|
||||
.default_output_config()
|
||||
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
|
||||
|
||||
let mic_sample_rate = mic_config.sample_rate();
|
||||
let lb_sample_rate = lb_config.sample_rate();
|
||||
let target_rate = target_sample_rate.unwrap_or(SampleRate(mic_sample_rate.min(lb_sample_rate).0));
|
||||
|
||||
let mic_channels = mic_config.channels();
|
||||
let lb_channels = lb_config.channels();
|
||||
|
||||
// Convert supported configs to concrete StreamConfigs
|
||||
let mic_stream_config: cpal::StreamConfig = mic_config.clone().into();
|
||||
let lb_stream_config: cpal::StreamConfig = lb_config.clone().into();
|
||||
|
||||
let stop_requested = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Channels for passing raw buffers between callback and mixer thread
|
||||
let (tx_mic, rx_mic) = unbounded::<AudioBuffer>();
|
||||
let (tx_lb, rx_lb) = unbounded::<AudioBuffer>();
|
||||
|
||||
// Build microphone input stream
|
||||
let mic_stream = mic
|
||||
.build_input_stream(
|
||||
&mic_stream_config,
|
||||
move |data: &[f32], _| {
|
||||
let _ = tx_mic.send(AudioBuffer { data: data.to_vec() });
|
||||
},
|
||||
|err| eprintln!("CPAL mic stream error: {err}"),
|
||||
None,
|
||||
)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, format!("build_input_stream: {e}")))?;
|
||||
|
||||
// Build loopback stream by creating input stream on output device (WASAPI
|
||||
// supports this)
|
||||
let lb_stream = loopback_device
|
||||
.build_input_stream(
|
||||
&lb_stream_config,
|
||||
move |data: &[f32], _| {
|
||||
let _ = tx_lb.send(AudioBuffer { data: data.to_vec() });
|
||||
},
|
||||
|err| eprintln!("CPAL loopback stream error: {err}"),
|
||||
None,
|
||||
)
|
||||
.map_err(|e| Error::new(Status::GenericFailure, format!("build_lb_stream: {e}")))?;
|
||||
|
||||
let stop_requested_flag = stop_requested.clone();
|
||||
|
||||
let jh = std::thread::spawn(move || {
|
||||
let mut post_mic: Vec<f32> = Vec::new();
|
||||
let mut post_lb: Vec<f32> = Vec::new();
|
||||
let mut mic_resampler = (mic_sample_rate != target_rate)
|
||||
.then(|| BufferedResampler::new(mic_sample_rate.0 as f64, target_rate.0 as f64, 1));
|
||||
let mut lb_resampler =
|
||||
(lb_sample_rate != target_rate).then(|| BufferedResampler::new(lb_sample_rate.0 as f64, target_rate.0 as f64, 1));
|
||||
let mut flushed_tail = false;
|
||||
|
||||
loop {
|
||||
while let Ok(buf) = rx_mic.try_recv() {
|
||||
let mono_samples: Vec<f32> = if mic_channels == 1 {
|
||||
buf.data
|
||||
} else {
|
||||
buf.data.chunks(mic_channels as usize).map(to_mono).collect()
|
||||
};
|
||||
extend_post_buffer(&mut post_mic, mono_samples, &mut mic_resampler);
|
||||
}
|
||||
|
||||
while let Ok(buf) = rx_lb.try_recv() {
|
||||
let mono_samples: Vec<f32> = if lb_channels == 1 {
|
||||
buf.data
|
||||
} else {
|
||||
buf.data.chunks(lb_channels as usize).map(to_mono).collect()
|
||||
};
|
||||
extend_post_buffer(&mut post_lb, mono_samples, &mut lb_resampler);
|
||||
}
|
||||
|
||||
emit_mixed_frames(&mut post_mic, &mut post_lb, &audio_buffer_callback, false);
|
||||
|
||||
const MAX_POST: usize = TARGET_FRAME_SIZE * 10;
|
||||
if post_mic.len() > MAX_POST {
|
||||
post_mic.drain(..post_mic.len() - MAX_POST);
|
||||
}
|
||||
if post_lb.len() > MAX_POST {
|
||||
post_lb.drain(..post_lb.len() - MAX_POST);
|
||||
}
|
||||
|
||||
let stop_requested = stop_requested_flag.load(Ordering::SeqCst);
|
||||
if stop_requested && !flushed_tail && rx_mic.is_empty() && rx_lb.is_empty() {
|
||||
if let Some(mut resampler) = mic_resampler.take() {
|
||||
post_mic.extend(resampler.finish());
|
||||
}
|
||||
if let Some(mut resampler) = lb_resampler.take() {
|
||||
post_lb.extend(resampler.finish());
|
||||
}
|
||||
emit_mixed_frames(&mut post_mic, &mut post_lb, &audio_buffer_callback, true);
|
||||
flushed_tail = true;
|
||||
}
|
||||
|
||||
if stop_requested && flushed_tail && rx_mic.is_empty() && rx_lb.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
if !stop_requested
|
||||
&& rx_mic.is_empty()
|
||||
&& rx_lb.is_empty()
|
||||
&& post_mic.len() < TARGET_FRAME_SIZE
|
||||
&& post_lb.len() < TARGET_FRAME_SIZE
|
||||
{
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
mic_stream
|
||||
.play()
|
||||
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
|
||||
lb_stream
|
||||
.play()
|
||||
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?;
|
||||
|
||||
Ok(AudioCaptureSession {
|
||||
mic_stream: Some(mic_stream),
|
||||
lb_stream: Some(lb_stream),
|
||||
stop_requested,
|
||||
sample_rate: target_rate,
|
||||
channels: 1, // mono output
|
||||
jh: Some(jh),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
use super::teardown_audio_capture_resources;
|
||||
|
||||
#[test]
|
||||
fn teardown_consumes_resources_even_when_pause_fails() {
|
||||
let mut mic_stream = Some("mic");
|
||||
let mut lb_stream = Some("loopback");
|
||||
let stop_requested = Arc::new(AtomicBool::new(false));
|
||||
let joined = Arc::new(AtomicBool::new(false));
|
||||
let joined_flag = joined.clone();
|
||||
let mut jh = Some(std::thread::spawn(move || {
|
||||
joined_flag.store(true, Ordering::SeqCst);
|
||||
}));
|
||||
|
||||
let result = teardown_audio_capture_resources(&mut mic_stream, &mut lb_stream, &stop_requested, &mut jh, |_| {
|
||||
Err("pause failed".to_owned())
|
||||
});
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(stop_requested.load(Ordering::SeqCst));
|
||||
assert!(mic_stream.is_none());
|
||||
assert!(lb_stream.is_none());
|
||||
assert!(jh.is_none());
|
||||
assert!(joined.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn teardown_is_idempotent_after_resources_are_consumed() {
|
||||
let mut mic_stream: Option<&'static str> = None;
|
||||
let mut lb_stream: Option<&'static str> = None;
|
||||
let stop_requested = Arc::new(AtomicBool::new(true));
|
||||
let mut jh = None;
|
||||
|
||||
let result = teardown_audio_capture_resources(
|
||||
&mut mic_stream,
|
||||
&mut lb_stream,
|
||||
&stop_requested,
|
||||
&mut jh,
|
||||
|_| -> std::result::Result<(), String> { panic!("pause should not be called when resources are already gone") },
|
||||
);
|
||||
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
}
|
||||
39
packages/frontend/native/media_capture/src/windows/error.rs
Normal file
39
packages/frontend/native/media_capture/src/windows/error.rs
Normal file
@@ -0,0 +1,39 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum WindowsAudioError {
|
||||
#[error("Failed to initialize COM: {0}")]
|
||||
ComInitializationFailed(String),
|
||||
#[error("Failed to create device enumerator: {0}")]
|
||||
DeviceEnumeratorCreationFailed(String),
|
||||
#[error("Failed to get default audio endpoint: {0}")]
|
||||
DefaultAudioEndpointFailed(String),
|
||||
#[error("Failed to activate audio session manager: {0}")]
|
||||
AudioSessionManagerActivationFailed(String),
|
||||
#[error("Failed to register session notification: {0}")]
|
||||
SessionNotificationRegistrationFailed(String),
|
||||
#[error("Failed to get session enumerator: {0}")]
|
||||
SessionEnumeratorFailed(String),
|
||||
#[error("Failed to get session count: {0}")]
|
||||
SessionCountFailed(String),
|
||||
#[error("Failed to get session: {0}")]
|
||||
GetSessionFailed(String),
|
||||
#[error("Failed to get process ID: {0}")]
|
||||
ProcessIdFailed(String),
|
||||
#[error("Failed to get session state: {0}")]
|
||||
SessionStateFailed(String),
|
||||
#[error("Failed to register audio session notification: {0}")]
|
||||
AudioSessionNotificationFailed(String),
|
||||
#[error("Failed to unregister audio session notification: {0}")]
|
||||
AudioSessionUnregisterFailed(String),
|
||||
#[error("Failed to open process: {0}")]
|
||||
ProcessOpenFailed(String),
|
||||
#[error("Failed to get process name: {0}")]
|
||||
ProcessNameFailed(String),
|
||||
}
|
||||
|
||||
impl From<WindowsAudioError> for napi::Error {
|
||||
fn from(value: WindowsAudioError) -> Self {
|
||||
napi::Error::new(napi::Status::GenericFailure, value.to_string())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
use std::{
|
||||
ffi::OsString,
|
||||
os::windows::ffi::OsStringExt,
|
||||
process,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
use napi::{
|
||||
Result,
|
||||
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
|
||||
};
|
||||
use windows::{
|
||||
Win32::{
|
||||
Foundation::CloseHandle,
|
||||
Media::Audio::{
|
||||
AudioSessionState, AudioSessionStateActive, DEVICE_STATE_ACTIVE, IAudioSessionControl, IAudioSessionControl2,
|
||||
IAudioSessionEnumerator, IAudioSessionEvents, IAudioSessionEvents_Impl, IAudioSessionManager2,
|
||||
IAudioSessionNotification, IAudioSessionNotification_Impl, IMMDevice, IMMDeviceCollection, IMMDeviceEnumerator,
|
||||
MMDeviceEnumerator, eCapture, eCommunications, eConsole,
|
||||
},
|
||||
System::{
|
||||
Com::{CLSCTX_ALL, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx},
|
||||
ProcessStatus::{GetModuleFileNameExW, GetProcessImageFileNameW},
|
||||
Threading::{OpenProcess, PROCESS_QUERY_INFORMATION, PROCESS_VM_READ},
|
||||
},
|
||||
},
|
||||
core::Interface,
|
||||
};
|
||||
use windows_core::implement;
|
||||
|
||||
pub struct AudioProcess {
|
||||
pub process_name: String,
|
||||
pub process_id: u32,
|
||||
pub device_id: String,
|
||||
pub device_name: String,
|
||||
pub is_running: bool,
|
||||
}
|
||||
|
||||
pub struct AudioDevice {
|
||||
pub device_id: String,
|
||||
pub device_name: String,
|
||||
pub is_default_communications: bool,
|
||||
pub is_default_console: bool,
|
||||
pub has_active_sessions: bool,
|
||||
}
|
||||
|
||||
// Simple struct for callback data - not a NAPI object
|
||||
#[derive(Clone)]
|
||||
pub struct MicrophoneActivateCallback {
|
||||
pub is_running: bool,
|
||||
pub process_name: String,
|
||||
pub device_id: String,
|
||||
pub device_name: String,
|
||||
}
|
||||
|
||||
#[implement(IAudioSessionEvents)]
|
||||
struct SessionEvents {
|
||||
process_name: String,
|
||||
device_id: String,
|
||||
device_name: String,
|
||||
callback: Arc<ThreadsafeFunction<(bool, String, String, String)>>,
|
||||
ctrl: IAudioSessionControl,
|
||||
events_ref: Arc<Mutex<Option<IAudioSessionEvents>>>,
|
||||
is_running: Arc<AtomicBool>,
|
||||
active_sessions: Arc<AtomicUsize>,
|
||||
session_is_active: AtomicBool,
|
||||
}
|
||||
|
||||
impl IAudioSessionEvents_Impl for SessionEvents_Impl {
|
||||
fn OnChannelVolumeChanged(
|
||||
&self,
|
||||
_channelcount: u32,
|
||||
_newchannelvolumearray: *const f32,
|
||||
_changedchannel: u32,
|
||||
_eventcontext: *const windows_core::GUID,
|
||||
) -> windows_core::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn OnDisplayNameChanged(
|
||||
&self,
|
||||
_newdisplayname: &windows_core::PCWSTR,
|
||||
_eventcontext: *const windows_core::GUID,
|
||||
) -> windows_core::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn OnGroupingParamChanged(
|
||||
&self,
|
||||
_newgroupingparam: *const windows_core::GUID,
|
||||
_eventcontext: *const windows_core::GUID,
|
||||
) -> windows_core::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn OnIconPathChanged(
|
||||
&self,
|
||||
_newiconpath: &windows_core::PCWSTR,
|
||||
_eventcontext: *const windows_core::GUID,
|
||||
) -> windows_core::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn OnSessionDisconnected(
|
||||
&self,
|
||||
_disconnectreason: windows::Win32::Media::Audio::AudioSessionDisconnectReason,
|
||||
) -> windows_core::Result<()> {
|
||||
if let Some(events) = self.events_ref.lock().unwrap().take() {
|
||||
unsafe { self.ctrl.UnregisterAudioSessionNotification(&events)? };
|
||||
}
|
||||
|
||||
// If this session was active, decrement the global counter
|
||||
if self.session_is_active.swap(false, Ordering::SeqCst) {
|
||||
let prev = self.active_sessions.fetch_sub(1, Ordering::SeqCst);
|
||||
if prev == 1 {
|
||||
// Last active session ended
|
||||
self.is_running.store(false, Ordering::Relaxed);
|
||||
// Notify JS side that recording has stopped
|
||||
self.callback.call(
|
||||
Ok((
|
||||
false,
|
||||
self.process_name.clone(),
|
||||
self.device_id.clone(),
|
||||
self.device_name.clone(),
|
||||
)),
|
||||
ThreadsafeFunctionCallMode::NonBlocking,
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn OnSimpleVolumeChanged(
|
||||
&self,
|
||||
_newvolume: f32,
|
||||
_newmute: windows_core::BOOL,
|
||||
_eventcontext: *const windows_core::GUID,
|
||||
) -> windows_core::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn OnStateChanged(&self, newstate: AudioSessionState) -> windows_core::Result<()> {
|
||||
// Determine the new recording state for this session
|
||||
let currently_recording = newstate == AudioSessionStateActive;
|
||||
|
||||
// Atomically swap the flag tracking this particular session
|
||||
let previously_recording = self.session_is_active.swap(currently_recording, Ordering::SeqCst);
|
||||
|
||||
// Update the global counter accordingly
|
||||
if !previously_recording && currently_recording {
|
||||
// Session started
|
||||
let prev = self.active_sessions.fetch_add(1, Ordering::SeqCst);
|
||||
if prev == 0 {
|
||||
// First active session across the whole system
|
||||
self.is_running.store(true, Ordering::Relaxed);
|
||||
}
|
||||
} else if previously_recording && !currently_recording {
|
||||
// Session stopped
|
||||
let prev = self.active_sessions.fetch_sub(1, Ordering::SeqCst);
|
||||
if prev == 1 {
|
||||
// Last active session just stopped
|
||||
self.is_running.store(false, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
let overall_is_running = self.active_sessions.load(Ordering::SeqCst) > 0;
|
||||
|
||||
// Notify JS side (non-blocking)
|
||||
self.callback.call(
|
||||
Ok((
|
||||
overall_is_running,
|
||||
self.process_name.clone(),
|
||||
self.device_id.clone(),
|
||||
self.device_name.clone(),
|
||||
)),
|
||||
ThreadsafeFunctionCallMode::NonBlocking,
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(IAudioSessionNotification)]
|
||||
struct SessionNotifier {
|
||||
_mgr: IAudioSessionManager2, // keep mgr alive
|
||||
device_id: String,
|
||||
device_name: String,
|
||||
ctrl: Mutex<Option<(IAudioSessionControl2, IAudioSessionEvents)>>, /* keep the ctrl2 and
|
||||
* events alive */
|
||||
callback: Arc<ThreadsafeFunction<(bool, String, String, String)>>,
|
||||
is_running: Arc<AtomicBool>, // Shared is_running flag
|
||||
active_sessions: Arc<AtomicUsize>, // Global counter of active sessions
|
||||
}
|
||||
|
||||
impl SessionNotifier {
|
||||
fn new(
|
||||
mgr: &IAudioSessionManager2,
|
||||
device_id: String,
|
||||
device_name: String,
|
||||
callback: Arc<ThreadsafeFunction<(bool, String, String, String)>>,
|
||||
is_running: Arc<AtomicBool>,
|
||||
active_sessions: Arc<AtomicUsize>,
|
||||
) -> Self {
|
||||
Self {
|
||||
_mgr: mgr.clone(),
|
||||
device_id,
|
||||
device_name,
|
||||
ctrl: Default::default(),
|
||||
callback,
|
||||
is_running,
|
||||
active_sessions,
|
||||
}
|
||||
}
|
||||
|
||||
fn refresh_state(&self, ctrl: &IAudioSessionControl) -> windows_core::Result<()> {
|
||||
let ctrl2: IAudioSessionControl2 = ctrl.cast()?;
|
||||
let process_id = unsafe { ctrl2.GetProcessId()? };
|
||||
|
||||
// Skip current process to avoid self-detection
|
||||
if process_id == process::id() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let process_name = match get_process_name(process_id) {
|
||||
Some(n) => n,
|
||||
None => unsafe { ctrl2.GetDisplayName()?.to_string()? },
|
||||
};
|
||||
// Skip system-sounds session
|
||||
// The `IsSystemSoundsSession` always true for unknown reason
|
||||
if process_name.contains("AudioSrv") {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Active ⇒ microphone is recording
|
||||
if unsafe { ctrl.GetState()? } == AudioSessionStateActive {
|
||||
let mut should_notify = false;
|
||||
if let Ok(mut optional_ctrl) = self.ctrl.lock() {
|
||||
// Increment the active session counter. If this was the first, flip
|
||||
// is_running to true.
|
||||
let prev = self.active_sessions.fetch_add(1, Ordering::SeqCst);
|
||||
if prev == 0 {
|
||||
self.is_running.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let events_ref = Arc::new(Mutex::new(None));
|
||||
let events: IAudioSessionEvents = SessionEvents {
|
||||
callback: self.callback.clone(),
|
||||
process_name: process_name.clone(),
|
||||
device_id: self.device_id.clone(),
|
||||
device_name: self.device_name.clone(),
|
||||
events_ref: events_ref.clone(),
|
||||
ctrl: ctrl.clone(),
|
||||
is_running: self.is_running.clone(),
|
||||
active_sessions: self.active_sessions.clone(),
|
||||
session_is_active: AtomicBool::new(true),
|
||||
}
|
||||
.into();
|
||||
let mut events_mut_ref = events_ref.lock().unwrap();
|
||||
*events_mut_ref = Some(events.clone());
|
||||
unsafe { ctrl.RegisterAudioSessionNotification(&events)? };
|
||||
// keep the ctrl2 alive so that the notification can be called
|
||||
*optional_ctrl = Some((ctrl2, events));
|
||||
|
||||
should_notify = true;
|
||||
}
|
||||
|
||||
if should_notify {
|
||||
self.callback.call(
|
||||
Ok((true, process_name, self.device_id.clone(), self.device_name.clone())),
|
||||
ThreadsafeFunctionCallMode::NonBlocking,
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl IAudioSessionNotification_Impl for SessionNotifier_Impl {
|
||||
fn OnSessionCreated(
|
||||
&self,
|
||||
ctrl_ref: windows_core::Ref<'_, windows::Win32::Media::Audio::IAudioSessionControl>,
|
||||
) -> windows_core::Result<()> {
|
||||
let Some(ctrl) = ctrl_ref.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
self.refresh_state(ctrl)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_audio_device_status_callback(
|
||||
is_running: Arc<AtomicBool>,
|
||||
active_sessions: Arc<AtomicUsize>,
|
||||
callback: Arc<ThreadsafeFunction<(bool, String, String, String)>>,
|
||||
) -> windows_core::Result<Vec<IAudioSessionNotification>> {
|
||||
unsafe {
|
||||
let enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?;
|
||||
|
||||
// Get all active capture devices
|
||||
let device_collection: IMMDeviceCollection = enumerator.EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE)?;
|
||||
|
||||
let device_count = device_collection.GetCount()?;
|
||||
let mut session_notifiers = Vec::new();
|
||||
|
||||
for i in 0..device_count {
|
||||
let device: IMMDevice = device_collection.Item(i)?;
|
||||
|
||||
// Device identifiers
|
||||
let device_id_pwstr = device.GetId()?;
|
||||
let device_id = device_id_pwstr.to_string()?;
|
||||
let device_name = format!("Audio Device {}", i);
|
||||
|
||||
// Activate session manager for this device
|
||||
let mgr: IAudioSessionManager2 = device.Activate(CLSCTX_ALL, None)?;
|
||||
|
||||
// Create notifier for this device
|
||||
let session_notifier = SessionNotifier::new(
|
||||
&mgr,
|
||||
device_id.clone(),
|
||||
device_name.clone(),
|
||||
callback.clone(),
|
||||
is_running.clone(),
|
||||
active_sessions.clone(),
|
||||
);
|
||||
|
||||
// Enumerate existing sessions to update counters and state immediately
|
||||
let list: IAudioSessionEnumerator = mgr.GetSessionEnumerator()?;
|
||||
let sessions = list.GetCount()?;
|
||||
for idx in 0..sessions {
|
||||
let ctrl = list.GetSession(idx)?;
|
||||
session_notifier.refresh_state(&ctrl)?;
|
||||
}
|
||||
|
||||
let session_notifier_impl: IAudioSessionNotification = session_notifier.into();
|
||||
mgr.RegisterSessionNotification(&session_notifier_impl)?;
|
||||
|
||||
session_notifiers.push(session_notifier_impl);
|
||||
}
|
||||
|
||||
Ok(session_notifiers)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MicrophoneListener {
|
||||
_session_notifiers: Vec<IAudioSessionNotification>, // keep the session_notifiers alive
|
||||
is_running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl MicrophoneListener {
|
||||
pub fn new(callback: ThreadsafeFunction<(bool, String, String, String)>) -> Self {
|
||||
unsafe {
|
||||
if CoInitializeEx(None, COINIT_MULTITHREADED).is_err() {
|
||||
// If COM initialization fails, create a listener with empty notifiers
|
||||
return Self {
|
||||
is_running: Arc::new(AtomicBool::new(false)),
|
||||
_session_notifiers: Vec::new(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let is_running = Arc::new(AtomicBool::new(false));
|
||||
let active_sessions = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let session_notifiers =
|
||||
match register_audio_device_status_callback(is_running.clone(), active_sessions.clone(), Arc::new(callback)) {
|
||||
Ok(notifiers) => notifiers,
|
||||
Err(_) => {
|
||||
// If registration fails, create a listener with empty notifiers
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
Self {
|
||||
is_running,
|
||||
_session_notifiers: session_notifiers,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_running(&self) -> bool {
|
||||
self.is_running.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
// Static method to check if a specific process is using microphone
|
||||
// This is used by TappableApplication::is_running()
|
||||
pub fn is_process_using_microphone(process_id: u32) -> bool {
|
||||
// Use the proven get_all_audio_processes logic
|
||||
match get_all_audio_processes() {
|
||||
Ok(processes) => processes.iter().any(|p| p.process_id == process_id && p.is_running),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_mgr_audio_session_running_status(mgr: &IAudioSessionManager2) -> windows_core::Result<(bool, String)> {
|
||||
let list: IAudioSessionEnumerator = unsafe { mgr.GetSessionEnumerator()? };
|
||||
let sessions = unsafe { list.GetCount()? };
|
||||
for idx in 0..sessions {
|
||||
let ctrl = unsafe { list.GetSession(idx)? };
|
||||
let ctrl2: IAudioSessionControl2 = ctrl.cast()?;
|
||||
let process_id = unsafe { ctrl2.GetProcessId()? };
|
||||
|
||||
// Skip current process to avoid self-detection
|
||||
if process_id == process::id() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let process_name = match get_process_name(process_id) {
|
||||
Some(n) => n,
|
||||
None => unsafe { ctrl2.GetDisplayName()?.to_string()? },
|
||||
};
|
||||
// Skip system-sounds session
|
||||
// The `IsSystemSoundsSession` always true for unknown reason
|
||||
if process_name.contains("AudioSrv") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Active ⇒ microphone is recording
|
||||
if unsafe { ctrl.GetState()? } == AudioSessionStateActive {
|
||||
return Ok((true, process_name));
|
||||
}
|
||||
}
|
||||
Ok((false, String::new()))
|
||||
}
|
||||
|
||||
fn get_process_name(pid: u32) -> Option<String> {
|
||||
unsafe {
|
||||
// Open process with required access rights
|
||||
let process_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid).ok()?;
|
||||
|
||||
// Allocate a buffer large enough to hold extended-length paths (up to ~32K
|
||||
// characters) instead of the legacy MAX_PATH (260) limit.
|
||||
let mut buffer: Vec<u16> = std::iter::repeat(0).take(32_768).collect();
|
||||
|
||||
// Try GetModuleFileNameExW first (gives full path with extension)
|
||||
let length = GetModuleFileNameExW(
|
||||
Some(process_handle),
|
||||
None, // NULL for the process executable
|
||||
&mut buffer,
|
||||
);
|
||||
|
||||
// If that fails, try GetProcessImageFileNameW
|
||||
let length = if length == 0 {
|
||||
GetProcessImageFileNameW(process_handle, &mut buffer)
|
||||
} else {
|
||||
length
|
||||
};
|
||||
|
||||
// Clean up
|
||||
CloseHandle(process_handle).ok()?;
|
||||
|
||||
if length == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Convert to OsString then to a regular String. Truncate buffer first.
|
||||
buffer.truncate(length as usize);
|
||||
let os_string = OsString::from_wide(&buffer);
|
||||
|
||||
// Extract the file name from the path
|
||||
let path_str = os_string.to_string_lossy().to_string();
|
||||
path_str.rsplit('\\').next().map(|s| s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn list_audio_processes() -> Result<Vec<AudioProcess>> {
|
||||
unsafe {
|
||||
// Try to initialize COM, but don't fail if it's already initialized
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
};
|
||||
|
||||
let result =
|
||||
get_all_audio_processes().map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn list_audio_devices() -> Result<Vec<AudioDevice>> {
|
||||
unsafe {
|
||||
// Try to initialize COM, but don't fail if it's already initialized
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
};
|
||||
|
||||
let result = get_all_audio_devices().map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn get_all_audio_processes() -> windows_core::Result<Vec<AudioProcess>> {
|
||||
unsafe {
|
||||
let enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?;
|
||||
|
||||
let device_collection: IMMDeviceCollection = enumerator.EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE)?;
|
||||
|
||||
let device_count = device_collection.GetCount()?;
|
||||
let mut all_processes = Vec::new();
|
||||
let current_pid = process::id();
|
||||
|
||||
for i in 0..device_count {
|
||||
let device: IMMDevice = device_collection.Item(i)?;
|
||||
|
||||
let device_id_pwstr = device.GetId()?;
|
||||
let device_id = device_id_pwstr.to_string()?;
|
||||
let device_name = format!("Audio Device {}", i);
|
||||
|
||||
let mgr: IAudioSessionManager2 = device.Activate(CLSCTX_ALL, None)?;
|
||||
let list: IAudioSessionEnumerator = mgr.GetSessionEnumerator()?;
|
||||
let sessions = list.GetCount()?;
|
||||
|
||||
for idx in 0..sessions {
|
||||
let ctrl = list.GetSession(idx)?;
|
||||
let ctrl2: IAudioSessionControl2 = ctrl.cast()?;
|
||||
let process_id = ctrl2.GetProcessId()?;
|
||||
|
||||
// Skip current process to avoid self-detection
|
||||
if process_id == current_pid {
|
||||
continue;
|
||||
}
|
||||
|
||||
let process_name = match get_process_name(process_id) {
|
||||
Some(n) => n,
|
||||
None => ctrl2.GetDisplayName()?.to_string()?,
|
||||
};
|
||||
|
||||
// Skip system-sounds session
|
||||
if process_name.contains("AudioSrv") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let is_running = ctrl.GetState()? == AudioSessionStateActive;
|
||||
|
||||
all_processes.push(AudioProcess {
|
||||
process_name,
|
||||
process_id,
|
||||
device_id: device_id.clone(),
|
||||
device_name: device_name.clone(),
|
||||
is_running,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(all_processes)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_all_audio_devices() -> windows_core::Result<Vec<AudioDevice>> {
|
||||
unsafe {
|
||||
let enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?;
|
||||
|
||||
let device_collection: IMMDeviceCollection = enumerator.EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE)?;
|
||||
|
||||
let device_count = device_collection.GetCount()?;
|
||||
let mut devices = Vec::new();
|
||||
|
||||
// Get default devices for comparison
|
||||
let default_comm_device_id = enumerator
|
||||
.GetDefaultAudioEndpoint(eCapture, eCommunications)
|
||||
.and_then(|d| d.GetId())
|
||||
.and_then(|id| Ok(id.to_string().unwrap_or_default()))
|
||||
.ok();
|
||||
let default_console_device_id = enumerator
|
||||
.GetDefaultAudioEndpoint(eCapture, eConsole)
|
||||
.and_then(|d| d.GetId())
|
||||
.and_then(|id| Ok(id.to_string().unwrap_or_default()))
|
||||
.ok();
|
||||
|
||||
for i in 0..device_count {
|
||||
let device: IMMDevice = device_collection.Item(i)?;
|
||||
|
||||
let device_id_pwstr = device.GetId()?;
|
||||
let device_id = device_id_pwstr.to_string()?;
|
||||
let device_name = format!("Audio Device {}", i);
|
||||
|
||||
let is_default_communications = default_comm_device_id.as_ref() == Some(&device_id);
|
||||
let is_default_console = default_console_device_id.as_ref() == Some(&device_id);
|
||||
|
||||
let mgr: IAudioSessionManager2 = device.Activate(CLSCTX_ALL, None)?;
|
||||
let (has_active_sessions, _) = get_mgr_audio_session_running_status(&mgr)?;
|
||||
|
||||
devices.push(AudioDevice {
|
||||
device_id,
|
||||
device_name,
|
||||
is_default_communications,
|
||||
is_default_console,
|
||||
has_active_sessions,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(devices)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_active_audio_processes() -> Result<Vec<AudioProcess>> {
|
||||
unsafe {
|
||||
// Try to initialize COM, but don't fail if it's already initialized
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
};
|
||||
|
||||
let result =
|
||||
get_all_audio_processes().map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?;
|
||||
|
||||
// Filter to only return active/running processes
|
||||
let active_processes = result.into_iter().filter(|p| p.is_running).collect();
|
||||
Ok(active_processes)
|
||||
}
|
||||
|
||||
pub fn is_process_actively_using_microphone(pid: u32) -> Result<bool> {
|
||||
unsafe {
|
||||
// Try to initialize COM, but don't fail if it's already initialized
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
};
|
||||
|
||||
let result =
|
||||
get_all_audio_processes().map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?;
|
||||
|
||||
// Check if the PID exists in the list of active processes
|
||||
let is_active = result
|
||||
.iter()
|
||||
.any(|process| process.process_id == pid && process.is_running);
|
||||
|
||||
Ok(is_active)
|
||||
}
|
||||
17
packages/frontend/native/media_capture/src/windows/mod.rs
Normal file
17
packages/frontend/native/media_capture/src/windows/mod.rs
Normal file
@@ -0,0 +1,17 @@
|
||||
pub mod audio_capture;
|
||||
pub(crate) mod error;
|
||||
pub mod microphone_listener;
|
||||
pub mod screen_capture_kit;
|
||||
|
||||
pub use audio_capture::*;
|
||||
pub use microphone_listener::*;
|
||||
pub use screen_capture_kit::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn test_windows_module_loads() {
|
||||
// Simple test to ensure the Windows module compiles and loads correctly
|
||||
assert!(true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
ffi::OsString,
|
||||
os::windows::ffi::OsStringExt,
|
||||
sync::{
|
||||
Arc, LazyLock, RwLock,
|
||||
atomic::{AtomicBool, AtomicU32, Ordering},
|
||||
},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use cpal::SampleRate;
|
||||
use napi::{
|
||||
bindgen_prelude::{Buffer, Error, Result, Status},
|
||||
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
|
||||
};
|
||||
use napi_derive::napi;
|
||||
// Windows API imports
|
||||
use windows::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; // HWND removed
|
||||
use windows::Win32::System::{
|
||||
Com::{COINIT_MULTITHREADED, CoInitializeEx},
|
||||
Diagnostics::ToolHelp::{
|
||||
CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW, TH32CS_SNAPPROCESS,
|
||||
},
|
||||
ProcessStatus::{GetModuleFileNameExW, GetProcessImageFileNameW},
|
||||
Threading::{OpenProcess, PROCESS_QUERY_INFORMATION, PROCESS_VM_READ},
|
||||
};
|
||||
|
||||
// Import the function from microphone_listener
|
||||
use crate::audio_callback::AudioCallback;
|
||||
use crate::windows::microphone_listener::is_process_actively_using_microphone;
|
||||
|
||||
// Type alias to match macOS API
|
||||
pub type AudioObjectID = u32;
|
||||
|
||||
// Global storage for running applications (Windows equivalent of macOS audio
|
||||
// process list)
|
||||
static RUNNING_APPLICATIONS: LazyLock<RwLock<Vec<u32>>> = LazyLock::new(|| RwLock::new(get_running_processes()));
|
||||
|
||||
// Simple counter for generating unique handles
|
||||
static NEXT_HANDLE: AtomicU32 = AtomicU32::new(1);
|
||||
|
||||
// Global storage for active watchers
|
||||
static ACTIVE_APP_WATCHERS: LazyLock<RwLock<Vec<(u32, u32, Arc<ThreadsafeFunction<(), ()>>, Arc<AtomicBool>)>>> =
|
||||
LazyLock::new(|| RwLock::new(Vec::new()));
|
||||
|
||||
static ACTIVE_LIST_WATCHERS: LazyLock<RwLock<Vec<(u32, Arc<ThreadsafeFunction<(), ()>>, Arc<AtomicBool>)>>> =
|
||||
LazyLock::new(|| RwLock::new(Vec::new()));
|
||||
|
||||
// Plain struct for efficient transmission via napi-rs
|
||||
#[napi]
|
||||
#[derive(Clone)]
|
||||
pub struct ApplicationInfo {
|
||||
pub process_id: i32,
|
||||
pub name: String,
|
||||
pub object_id: u32,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ApplicationInfo {
|
||||
#[napi(constructor)]
|
||||
pub fn new(process_id: i32, name: String, object_id: u32) -> Self {
|
||||
Self {
|
||||
process_id,
|
||||
name,
|
||||
object_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn process_group_id(&self) -> i32 {
|
||||
// Windows doesn't have process groups like Unix, return the process ID
|
||||
self.process_id
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn bundle_identifier(&self) -> String {
|
||||
// For Windows, return the fully-qualified path to the .exe on disk
|
||||
let path = get_process_executable_path(self.process_id as u32).unwrap_or_default();
|
||||
// Escape invalid filename characters for Windows
|
||||
escape_filename(&path)
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn icon(&self) -> Buffer {
|
||||
// For now, return empty buffer. In a full implementation, you would extract
|
||||
// the icon from the executable file using Windows APIs
|
||||
Buffer::from(Vec::<u8>::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct ApplicationListChangedSubscriber {
|
||||
handle: u32,
|
||||
// We'll store the callback and manage it through a background thread
|
||||
_callback: Arc<ThreadsafeFunction<(), ()>>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ApplicationListChangedSubscriber {
|
||||
#[napi]
|
||||
pub fn unsubscribe(&self) -> Result<()> {
|
||||
if let Ok(mut watchers) = ACTIVE_LIST_WATCHERS.write() {
|
||||
if let Some(pos) = watchers.iter().position(|(handle, _, _)| *handle == self.handle) {
|
||||
let (_, _, should_stop) = &watchers[pos];
|
||||
should_stop.store(true, Ordering::Relaxed);
|
||||
watchers.remove(pos);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct ApplicationStateChangedSubscriber {
|
||||
handle: u32,
|
||||
process_id: u32,
|
||||
_callback: Arc<ThreadsafeFunction<(), ()>>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ApplicationStateChangedSubscriber {
|
||||
pub fn process_id(&self) -> u32 {
|
||||
self.process_id
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn unsubscribe(&self) {
|
||||
if let Ok(mut watchers) = ACTIVE_APP_WATCHERS.write() {
|
||||
if let Some(pos) = watchers.iter().position(|(handle, _, _, _)| *handle == self.handle) {
|
||||
let (_, _, _, should_stop) = &watchers[pos];
|
||||
should_stop.store(true, Ordering::Relaxed);
|
||||
watchers.remove(pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct ShareableContent {
|
||||
// Windows doesn't need an inner SCShareableContent equivalent
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl ShareableContent {
|
||||
#[napi]
|
||||
pub fn on_application_list_changed(callback: ThreadsafeFunction<(), ()>) -> Result<ApplicationListChangedSubscriber> {
|
||||
let handle = NEXT_HANDLE.fetch_add(1, Ordering::Relaxed);
|
||||
let callback_arc = Arc::new(callback);
|
||||
|
||||
// Start monitoring for application list changes
|
||||
start_list_monitoring(handle, callback_arc.clone());
|
||||
|
||||
Ok(ApplicationListChangedSubscriber {
|
||||
handle,
|
||||
_callback: callback_arc,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn on_app_state_changed(
|
||||
app: &ApplicationInfo,
|
||||
callback: ThreadsafeFunction<(), ()>,
|
||||
) -> Result<ApplicationStateChangedSubscriber> {
|
||||
let handle = NEXT_HANDLE.fetch_add(1, Ordering::Relaxed);
|
||||
let process_id = app.process_id as u32;
|
||||
let callback_arc = Arc::new(callback);
|
||||
|
||||
// Start monitoring for this specific process's microphone state
|
||||
start_process_monitoring(handle, process_id, callback_arc.clone());
|
||||
|
||||
Ok(ApplicationStateChangedSubscriber {
|
||||
handle,
|
||||
process_id,
|
||||
_callback: callback_arc,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(constructor)]
|
||||
pub fn new() -> Self {
|
||||
unsafe {
|
||||
CoInitializeEx(None, COINIT_MULTITHREADED).ok().unwrap_or_else(|_| {
|
||||
// COM initialization failed, but we can't return an error from
|
||||
// constructor This is typically not fatal as COM might
|
||||
// already be initialized
|
||||
});
|
||||
}
|
||||
Self {}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn applications() -> Result<Vec<ApplicationInfo>> {
|
||||
let processes = RUNNING_APPLICATIONS
|
||||
.read()
|
||||
.map_err(|_| Error::new(Status::GenericFailure, "Failed to read running applications"))?;
|
||||
|
||||
let mut apps = Vec::new();
|
||||
for &process_id in processes.iter() {
|
||||
let name = get_process_name(process_id).unwrap_or_else(|| format!("Process {}", process_id));
|
||||
if !name.is_empty() && name != format!("Process {}", process_id) {
|
||||
let app_info = ApplicationInfo::new(process_id as i32, name, process_id);
|
||||
apps.push(app_info);
|
||||
}
|
||||
}
|
||||
Ok(apps)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn application_with_process_id(process_id: u32) -> Option<ApplicationInfo> {
|
||||
if is_process_running(process_id) {
|
||||
let name = get_process_name(process_id).unwrap_or_else(|| format!("Process {}", process_id));
|
||||
Some(ApplicationInfo::new(process_id as i32, name, process_id))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn tap_audio_with_callback(
|
||||
_process_id: u32,
|
||||
audio_stream_callback: AudioCallback,
|
||||
target_sample_rate: Option<u32>,
|
||||
) -> Result<AudioCaptureSession> {
|
||||
let target = target_sample_rate.map(SampleRate);
|
||||
crate::windows::audio_capture::start_recording(audio_stream_callback, target)
|
||||
}
|
||||
|
||||
pub(crate) fn tap_global_audio_with_callback(
|
||||
_excluded_processes: Option<Vec<&ApplicationInfo>>,
|
||||
audio_stream_callback: AudioCallback,
|
||||
target_sample_rate: Option<u32>,
|
||||
) -> Result<AudioCaptureSession> {
|
||||
let target = target_sample_rate.map(SampleRate);
|
||||
// Delegate to audio_capture::start_recording which handles mixing mic +
|
||||
// loopback
|
||||
crate::windows::audio_capture::start_recording(audio_stream_callback, target)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn is_using_microphone(process_id: u32) -> Result<bool> {
|
||||
is_process_actively_using_microphone(process_id)
|
||||
}
|
||||
}
|
||||
|
||||
// Re-export the concrete audio capture session implemented in audio_capture.rs
|
||||
pub use crate::windows::audio_capture::AudioCaptureSession;
|
||||
|
||||
// Helper function to escape invalid filename characters
|
||||
fn escape_filename(path: &str) -> String {
|
||||
// Replace invalid filename characters with underscores
|
||||
// Invalid chars on Windows: < > : " | ? * \ spaces and control chars (0-31)
|
||||
path
|
||||
.chars()
|
||||
.map(|c| match c {
|
||||
'<' | '>' | ':' | '"' | '|' | '?' | '*' | '\\' | ' ' => '_',
|
||||
c if c.is_control() => '_',
|
||||
c => c,
|
||||
})
|
||||
.collect::<String>()
|
||||
.to_lowercase()
|
||||
}
|
||||
|
||||
// Helper functions for Windows process management
|
||||
|
||||
fn get_running_processes() -> Vec<u32> {
|
||||
let mut processes_set = HashSet::new(); // Use HashSet to avoid duplicates from the start
|
||||
unsafe {
|
||||
let h_snapshot_result = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
|
||||
|
||||
let h_snapshot = match h_snapshot_result {
|
||||
Ok(handle) => {
|
||||
if handle == INVALID_HANDLE_VALUE {
|
||||
// eprintln!("CreateToolhelp32Snapshot returned
|
||||
// INVALID_HANDLE_VALUE");
|
||||
return Vec::new();
|
||||
}
|
||||
handle
|
||||
}
|
||||
Err(_e) => {
|
||||
// eprintln!("CreateToolhelp32Snapshot failed: {:?}", e);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let mut pe32 = PROCESSENTRY32W::default();
|
||||
pe32.dwSize = std::mem::size_of::<PROCESSENTRY32W>() as u32;
|
||||
|
||||
if Process32FirstW(h_snapshot, &mut pe32).is_ok() {
|
||||
loop {
|
||||
processes_set.insert(pe32.th32ProcessID);
|
||||
if Process32NextW(h_snapshot, &mut pe32).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
CloseHandle(h_snapshot).unwrap_or_else(|_e| {
|
||||
// eprintln!("CloseHandle failed for snapshot: {:?}", e);
|
||||
});
|
||||
}
|
||||
let mut processes_vec: Vec<u32> = processes_set.into_iter().collect();
|
||||
processes_vec.sort_unstable(); // Sort for consistent ordering, though not strictly necessary for functionality
|
||||
processes_vec
|
||||
}
|
||||
|
||||
fn is_process_running(process_id: u32) -> bool {
|
||||
unsafe {
|
||||
match OpenProcess(PROCESS_QUERY_INFORMATION, false, process_id) {
|
||||
Ok(handle) => CloseHandle(handle).is_ok(),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_process_name(pid: u32) -> Option<String> {
|
||||
unsafe {
|
||||
let process_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid).ok()?;
|
||||
// Allocate a buffer large enough to hold extended-length paths (up to ~32K
|
||||
// characters) instead of the legacy MAX_PATH (260) limit. 32 768 is the
|
||||
// maximum length supported by the Win32 APIs when the path is prefixed
|
||||
// with "\\?\".
|
||||
let mut buffer: Vec<u16> = std::iter::repeat(0).take(32_768).collect();
|
||||
|
||||
let length = GetModuleFileNameExW(Some(process_handle), None, &mut buffer);
|
||||
CloseHandle(process_handle).ok()?;
|
||||
|
||||
if length == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Truncate the buffer to the length returned by the Windows API before
|
||||
// doing the UTF-16 → UTF-8 conversion.
|
||||
buffer.truncate(length as usize);
|
||||
let os_string = OsString::from_wide(&buffer);
|
||||
let path_str = os_string.to_string_lossy().to_string();
|
||||
path_str.rsplit('\\').next().map(|s| s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn get_process_executable_path(pid: u32) -> Option<String> {
|
||||
unsafe {
|
||||
let process_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid).ok()?;
|
||||
// Use a buffer that can hold extended-length paths. See rationale above.
|
||||
let mut buffer: Vec<u16> = std::iter::repeat(0).take(32_768).collect();
|
||||
|
||||
let length = GetProcessImageFileNameW(process_handle, &mut buffer);
|
||||
CloseHandle(process_handle).ok()?;
|
||||
|
||||
if length == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
buffer.truncate(length as usize);
|
||||
let os_string = OsString::from_wide(&buffer);
|
||||
let path_str = os_string.to_string_lossy().to_string();
|
||||
Some(path_str)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to start monitoring a specific process
|
||||
fn start_process_monitoring(handle: u32, process_id: u32, callback: Arc<ThreadsafeFunction<(), ()>>) {
|
||||
let should_stop = Arc::new(AtomicBool::new(false));
|
||||
let should_stop_clone = should_stop.clone();
|
||||
|
||||
// Store the watcher info
|
||||
if let Ok(mut watchers) = ACTIVE_APP_WATCHERS.write() {
|
||||
watchers.push((handle, process_id, callback.clone(), should_stop.clone()));
|
||||
}
|
||||
|
||||
// Start monitoring thread
|
||||
thread::spawn(move || {
|
||||
let mut last_state = false;
|
||||
|
||||
loop {
|
||||
if should_stop_clone.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Check current microphone state
|
||||
let current_state = is_process_actively_using_microphone(process_id).unwrap_or(false);
|
||||
|
||||
// If state changed, trigger callback
|
||||
if current_state != last_state {
|
||||
let _ = callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);
|
||||
last_state = current_state;
|
||||
}
|
||||
|
||||
// Sleep for a short interval before checking again
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Helper function to start monitoring application list changes
|
||||
fn start_list_monitoring(handle: u32, callback: Arc<ThreadsafeFunction<(), ()>>) {
|
||||
let should_stop = Arc::new(AtomicBool::new(false));
|
||||
let should_stop_clone = should_stop.clone();
|
||||
|
||||
// Store the watcher info
|
||||
if let Ok(mut watchers) = ACTIVE_LIST_WATCHERS.write() {
|
||||
watchers.push((handle, callback.clone(), should_stop.clone()));
|
||||
}
|
||||
|
||||
// Start monitoring thread
|
||||
thread::spawn(move || {
|
||||
let mut last_processes = get_running_processes();
|
||||
|
||||
loop {
|
||||
if should_stop_clone.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Check current process list
|
||||
let current_processes = get_running_processes();
|
||||
|
||||
// If process list changed, trigger callback
|
||||
if current_processes != last_processes {
|
||||
let _ = callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking);
|
||||
last_processes = current_processes;
|
||||
|
||||
// Update global process list
|
||||
if let Ok(mut apps) = RUNNING_APPLICATIONS.write() {
|
||||
*apps = last_processes.clone();
|
||||
}
|
||||
}
|
||||
|
||||
// Sleep for a longer interval for process list changes
|
||||
thread::sleep(Duration::from_millis(2000));
|
||||
}
|
||||
});
|
||||
}
|
||||
58
packages/frontend/native/nbstore/Cargo.toml
Normal file
58
packages/frontend/native/nbstore/Cargo.toml
Normal file
@@ -0,0 +1,58 @@
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "affine_nbstore"
|
||||
publish = false
|
||||
version = "0.0.0"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
napi = ["affine_common/napi"]
|
||||
use-as-lib = ["napi-derive/noop", "napi/noop"]
|
||||
|
||||
[dependencies]
|
||||
affine_common = { workspace = true, features = ["napi"] }
|
||||
affine_doc_loader = { workspace = true }
|
||||
affine_schema = { path = "../schema" }
|
||||
anyhow = { workspace = true }
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
chrono = { workspace = true }
|
||||
memory-indexer = { workspace = true }
|
||||
napi = { workspace = true }
|
||||
napi-derive = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sqlx = { workspace = true, default-features = false, features = [
|
||||
"chrono",
|
||||
"macros",
|
||||
"migrate",
|
||||
"runtime-tokio",
|
||||
"sqlite",
|
||||
] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
y-octo = { workspace = true }
|
||||
zstd = "0.13"
|
||||
|
||||
[target.'cfg(any(target_os = "ios", target_os = "android"))'.dependencies]
|
||||
uniffi = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
affine_schema = { path = "../schema" }
|
||||
dotenvy = { workspace = true }
|
||||
napi-build = { workspace = true }
|
||||
sqlx = { workspace = true, default-features = false, features = [
|
||||
"chrono",
|
||||
"macros",
|
||||
"migrate",
|
||||
"runtime-tokio",
|
||||
"sqlite",
|
||||
] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
assert-json-diff = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
34
packages/frontend/native/nbstore/build.rs
Normal file
34
packages/frontend/native/nbstore/build.rs
Normal file
@@ -0,0 +1,34 @@
|
||||
use std::fs;
|
||||
|
||||
use affine_schema::get_migrator;
|
||||
use sqlx::sqlite::SqliteConnectOptions;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), std::io::Error> {
|
||||
napi_build::setup();
|
||||
|
||||
// always start with a fresh database to have latest db schema
|
||||
let cwd = std::env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
let db_path = format!("{cwd}/affine.db");
|
||||
|
||||
if fs::metadata(&db_path).is_ok() {
|
||||
fs::remove_file(&db_path)?;
|
||||
}
|
||||
|
||||
let options = SqliteConnectOptions::new()
|
||||
.filename(&db_path)
|
||||
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
|
||||
.locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
|
||||
.create_if_missing(true);
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
get_migrator().run(&pool).await.unwrap();
|
||||
|
||||
println!("cargo::rustc-env=DATABASE_URL=sqlite://{db_path}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
186
packages/frontend/native/nbstore/src/blob.rs
Normal file
186
packages/frontend/native/nbstore/src/blob.rs
Normal file
@@ -0,0 +1,186 @@
|
||||
use std::ops::Deref;
|
||||
|
||||
use super::{Blob, ListedBlob, SetBlob, error::Result, storage::SqliteDocStorage};
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn get_blob(&self, key: String) -> Result<Option<Blob>> {
|
||||
let result = sqlx::query_as!(
|
||||
Blob,
|
||||
"SELECT key, data, size, mime, created_at FROM blobs WHERE key = ? AND deleted_at IS NULL",
|
||||
key
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn set_blob(&self, blob: SetBlob) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO blobs (key, data, mime, size)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT(key)
|
||||
DO UPDATE SET data=$2, mime=$3, size=$4, deleted_at=NULL;"#,
|
||||
)
|
||||
.bind(blob.key)
|
||||
.bind(blob.data.deref())
|
||||
.bind(blob.mime)
|
||||
.bind(blob.data.len() as i64)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_blob(&self, key: String, permanently: bool) -> Result<()> {
|
||||
if permanently {
|
||||
sqlx::query("DELETE FROM blobs WHERE key = ?")
|
||||
.bind(&key)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
} else {
|
||||
sqlx::query("UPDATE blobs SET deleted_at = CURRENT_TIMESTAMP WHERE key = ?")
|
||||
.bind(&key)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn release_blobs(&self) -> Result<()> {
|
||||
sqlx::query("DELETE FROM blobs WHERE deleted_at IS NOT NULL;")
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list_blobs(&self) -> Result<Vec<ListedBlob>> {
|
||||
let result = sqlx::query_as!(
|
||||
ListedBlob,
|
||||
"SELECT key, size, mime, created_at FROM blobs WHERE deleted_at IS NULL ORDER BY created_at DESC;"
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use sqlx::Row;
|
||||
|
||||
use super::{super::Data, *};
|
||||
|
||||
async fn get_storage() -> SqliteDocStorage {
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_blob() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
for i in 1..5u32 {
|
||||
storage
|
||||
.set_blob(SetBlob {
|
||||
key: format!("test_{}", i),
|
||||
data: Into::<Data>::into(vec![0, 0]),
|
||||
mime: "text/plain".to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let result = storage.get_blob("test_1".to_string()).await.unwrap();
|
||||
|
||||
assert!(result.is_some());
|
||||
|
||||
storage.delete_blob("test_".to_string(), false).await.unwrap();
|
||||
|
||||
let result = storage.get_blob("test".to_string()).await.unwrap();
|
||||
assert!(result.is_none());
|
||||
|
||||
storage.delete_blob("test_2".to_string(), true).await.unwrap();
|
||||
|
||||
let result = storage.get_blob("test".to_string()).await.unwrap();
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_blobs() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
let blobs = storage.list_blobs().await.unwrap();
|
||||
|
||||
assert_eq!(blobs.len(), 0);
|
||||
|
||||
for i in 1..5u32 {
|
||||
storage
|
||||
.set_blob(SetBlob {
|
||||
key: format!("test_{}", i),
|
||||
data: Into::<Data>::into(vec![0, 0]),
|
||||
mime: "text/plain".to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let blobs = storage.list_blobs().await.unwrap();
|
||||
|
||||
assert_eq!(blobs.len(), 4);
|
||||
assert_eq!(
|
||||
blobs.iter().map(|b| b.key.as_str()).collect::<Vec<_>>(),
|
||||
vec!["test_1", "test_2", "test_3", "test_4"]
|
||||
);
|
||||
|
||||
storage.delete_blob("test_2".to_string(), false).await.unwrap();
|
||||
|
||||
storage.delete_blob("test_3".to_string(), true).await.unwrap();
|
||||
|
||||
let query = sqlx::query("SELECT COUNT(*) as len FROM blobs;")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(query.get::<i64, &str>("len"), 3);
|
||||
|
||||
let blobs = storage.list_blobs().await.unwrap();
|
||||
assert_eq!(blobs.len(), 2);
|
||||
assert_eq!(
|
||||
blobs.iter().map(|b| b.key.as_str()).collect::<Vec<_>>(),
|
||||
vec!["test_1", "test_4"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn release_blobs() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
for i in 1..5u32 {
|
||||
storage
|
||||
.set_blob(SetBlob {
|
||||
key: format!("test_{}", i),
|
||||
data: Into::<Data>::into(vec![0, 0]),
|
||||
mime: "text/plain".to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
storage.delete_blob("test_2".to_string(), false).await.unwrap();
|
||||
storage.release_blobs().await.unwrap();
|
||||
|
||||
let query = sqlx::query("SELECT COUNT(*) as len FROM blobs;")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(query.get::<i64, &str>("len"), 3);
|
||||
}
|
||||
}
|
||||
88
packages/frontend/native/nbstore/src/blob_sync.rs
Normal file
88
packages/frontend/native/nbstore/src/blob_sync.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
use chrono::NaiveDateTime;
|
||||
|
||||
use super::{error::Result, storage::SqliteDocStorage};
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn set_blob_uploaded_at(
|
||||
&self,
|
||||
peer: String,
|
||||
blob_id: String,
|
||||
uploaded_at: Option<NaiveDateTime>,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO peer_blob_sync (peer, blob_id, uploaded_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(peer, blob_id)
|
||||
DO UPDATE SET uploaded_at=$3;"#,
|
||||
)
|
||||
.bind(peer)
|
||||
.bind(blob_id)
|
||||
.bind(uploaded_at)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_blob_uploaded_at(&self, peer: String, blob_id: String) -> Result<Option<NaiveDateTime>> {
|
||||
let result = sqlx::query_scalar!(
|
||||
"SELECT uploaded_at FROM peer_blob_sync WHERE peer = ? AND blob_id = ?",
|
||||
peer,
|
||||
blob_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result.flatten())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::Utc;
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn get_storage() -> SqliteDocStorage {
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blob_uploaded_at() {
|
||||
let storage = get_storage().await;
|
||||
let peer = String::from("peer1");
|
||||
let blob_id = String::from("blob1");
|
||||
|
||||
let uploaded_at = storage
|
||||
.get_blob_uploaded_at(peer.clone(), blob_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(uploaded_at.is_none());
|
||||
|
||||
let now = Utc::now().naive_utc();
|
||||
storage
|
||||
.set_blob_uploaded_at(peer.clone(), blob_id.clone(), Some(now))
|
||||
.await
|
||||
.unwrap();
|
||||
let uploaded_at = storage
|
||||
.get_blob_uploaded_at(peer.clone(), blob_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(uploaded_at.is_some());
|
||||
assert_eq!(uploaded_at.unwrap(), now);
|
||||
|
||||
storage
|
||||
.set_blob_uploaded_at(peer.clone(), blob_id.clone(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let uploaded_at = storage
|
||||
.get_blob_uploaded_at(peer.clone(), blob_id.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(uploaded_at.is_none());
|
||||
}
|
||||
}
|
||||
465
packages/frontend/native/nbstore/src/doc.rs
Normal file
465
packages/frontend/native/nbstore/src/doc.rs
Normal file
@@ -0,0 +1,465 @@
|
||||
use std::ops::Deref;
|
||||
|
||||
use chrono::{DateTime, NaiveDateTime};
|
||||
use sqlx::{QueryBuilder, Row};
|
||||
|
||||
use super::{DocClock, DocRecord, DocUpdate, error::Result, storage::SqliteDocStorage};
|
||||
|
||||
struct Meta {
|
||||
space_id: String,
|
||||
}
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn set_space_id(&self, space_id: String) -> Result<()> {
|
||||
// ensure only one record exists in table
|
||||
let result = sqlx::query_as!(Meta, "SELECT * FROM meta;")
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
match result {
|
||||
Some(meta) => {
|
||||
if meta.space_id != space_id {
|
||||
sqlx::query("UPDATE meta SET space_id = $1;")
|
||||
.bind(&space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query("UPDATE updates SET doc_id = $1 WHERE doc_id = $2;")
|
||||
.bind(&space_id)
|
||||
.bind(&meta.space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query("UPDATE snapshots SET doc_id = $1 WHERE doc_id = $2;")
|
||||
.bind(&space_id)
|
||||
.bind(&meta.space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query("UPDATE clocks SET doc_id = $1 WHERE doc_id = $2;")
|
||||
.bind(&space_id)
|
||||
.bind(&meta.space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
sqlx::query("UPDATE indexer_sync SET doc_id = $1 WHERE doc_id = $2;")
|
||||
.bind(&space_id)
|
||||
.bind(&meta.space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query("UPDATE peer_clocks SET doc_id = $1 WHERE doc_id = $2;")
|
||||
.bind(&space_id)
|
||||
.bind(&meta.space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
sqlx::query("INSERT INTO meta (space_id) VALUES ($1);")
|
||||
.bind(&space_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn push_update<Update: AsRef<[u8]>>(&self, doc_id: String, update: Update) -> Result<NaiveDateTime> {
|
||||
let mut timestamp = DateTime::from_timestamp_millis(chrono::Utc::now().timestamp_millis())
|
||||
.unwrap()
|
||||
.naive_utc();
|
||||
|
||||
let mut tried = 0;
|
||||
|
||||
// Keep trying with incremented timestamps until success
|
||||
loop {
|
||||
match self
|
||||
.try_insert_update_with_timestamp(&doc_id, update.as_ref(), timestamp)
|
||||
.await
|
||||
{
|
||||
Ok(()) => break,
|
||||
Err(e) => {
|
||||
if tried > 10 {
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
// Increment timestamp by 1ms and retry
|
||||
timestamp += chrono::Duration::milliseconds(1);
|
||||
tried += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(timestamp)
|
||||
}
|
||||
|
||||
async fn try_insert_update_with_timestamp(
|
||||
&self,
|
||||
doc_id: &str,
|
||||
update: &[u8],
|
||||
timestamp: NaiveDateTime,
|
||||
) -> sqlx::Result<()> {
|
||||
let mut tx = self.pool.begin().await?;
|
||||
|
||||
sqlx::query(r#"INSERT INTO updates (doc_id, data, created_at) VALUES ($1, $2, $3);"#)
|
||||
.bind(doc_id)
|
||||
.bind(update)
|
||||
.bind(timestamp)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO clocks (doc_id, timestamp) VALUES ($1, $2)
|
||||
ON CONFLICT(doc_id)
|
||||
DO UPDATE SET timestamp=$2;"#,
|
||||
)
|
||||
.bind(doc_id)
|
||||
.bind(timestamp)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_doc_snapshot(&self, doc_id: String) -> Result<Option<DocRecord>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocRecord,
|
||||
"SELECT doc_id, data as bin, updated_at as timestamp FROM snapshots WHERE doc_id = ?",
|
||||
doc_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn set_doc_snapshot(&self, snapshot: DocRecord) -> Result<bool> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO snapshots (doc_id, data, updated_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(doc_id)
|
||||
DO UPDATE SET data=$2, updated_at=$3
|
||||
WHERE updated_at <= $3;"#,
|
||||
)
|
||||
.bind(snapshot.doc_id)
|
||||
.bind(snapshot.bin.deref())
|
||||
.bind(snapshot.timestamp)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
|
||||
pub async fn get_doc_updates(&self, doc_id: String) -> Result<Vec<DocUpdate>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocUpdate,
|
||||
"SELECT doc_id, created_at as timestamp, data as bin FROM updates WHERE doc_id = ?",
|
||||
doc_id
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn mark_updates_merged(&self, doc_id: String, updates: Vec<NaiveDateTime>) -> Result<u32> {
|
||||
let mut qb = QueryBuilder::new("DELETE FROM updates");
|
||||
|
||||
qb.push(" WHERE doc_id = ");
|
||||
qb.push_bind(doc_id);
|
||||
qb.push(" AND created_at IN (");
|
||||
let mut separated = qb.separated(", ");
|
||||
updates.iter().for_each(|update| {
|
||||
separated.push_bind(update);
|
||||
});
|
||||
qb.push(");");
|
||||
|
||||
let query = qb.build();
|
||||
|
||||
let result = query.execute(&self.pool).await?;
|
||||
|
||||
Ok(result.rows_affected() as u32)
|
||||
}
|
||||
|
||||
pub async fn delete_doc(&self, doc_id: String) -> Result<()> {
|
||||
let mut tx = self.pool.begin().await?;
|
||||
|
||||
sqlx::query("DELETE FROM updates WHERE doc_id = ?;")
|
||||
.bind(&doc_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query("DELETE FROM snapshots WHERE doc_id = ?;")
|
||||
.bind(&doc_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query("DELETE FROM clocks WHERE doc_id = ?;")
|
||||
.bind(&doc_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query("DELETE FROM indexer_sync WHERE doc_id = ?;")
|
||||
.bind(&doc_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_doc_clocks(&self, after: Option<NaiveDateTime>) -> Result<Vec<DocClock>> {
|
||||
let query = if let Some(after) = after {
|
||||
sqlx::query("SELECT doc_id, timestamp FROM clocks WHERE timestamp > $1").bind(after)
|
||||
} else {
|
||||
sqlx::query("SELECT doc_id, timestamp FROM clocks")
|
||||
};
|
||||
|
||||
let clocks = query.fetch_all(&self.pool).await?;
|
||||
|
||||
Ok(
|
||||
clocks
|
||||
.iter()
|
||||
.map(|row| DocClock {
|
||||
doc_id: row.get("doc_id"),
|
||||
timestamp: row.get("timestamp"),
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn get_doc_clock(&self, doc_id: String) -> Result<Option<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
"SELECT doc_id, timestamp FROM clocks WHERE doc_id = ?",
|
||||
doc_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::*;
|
||||
use crate::Data;
|
||||
|
||||
async fn get_storage() -> SqliteDocStorage {
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_space_id() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
storage.set_space_id("test".to_string()).await.unwrap();
|
||||
|
||||
let result = sqlx::query!("SELECT space_id FROM meta;")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.space_id, "test");
|
||||
|
||||
storage.set_space_id("test2".to_string()).await.unwrap();
|
||||
|
||||
let result = sqlx::query!("SELECT space_id FROM meta;")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.space_id, "test2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_space_id_with_existing_doc() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
storage.set_space_id("test".to_string()).await.unwrap();
|
||||
storage.push_update("test".to_string(), vec![0, 0]).await.unwrap();
|
||||
storage
|
||||
.set_doc_snapshot(DocRecord {
|
||||
doc_id: "test".to_string(),
|
||||
bin: Into::<Data>::into(vec![0, 0]),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
storage
|
||||
.set_peer_pulled_remote_clock("remote".to_string(), "test".to_string(), Utc::now().naive_utc())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
storage.set_space_id("new_id".to_string()).await.unwrap();
|
||||
|
||||
let result = sqlx::query!("SELECT space_id FROM meta;")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.space_id, "new_id");
|
||||
|
||||
let clocks = storage.get_doc_clocks(None).await.unwrap();
|
||||
|
||||
assert_eq!(clocks[0].doc_id, "new_id");
|
||||
|
||||
let clocks = storage
|
||||
.get_peer_pulled_remote_clock("remote".to_string(), "new_id".to_string())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(clocks.doc_id, "new_id");
|
||||
|
||||
let updates = storage.get_doc_updates("new_id".to_string()).await.unwrap();
|
||||
|
||||
assert_eq!(updates.len(), 1);
|
||||
|
||||
let snapshot = storage.get_doc_snapshot("new_id".to_string()).await.unwrap();
|
||||
|
||||
assert!(snapshot.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn push_updates() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
let updates = vec![vec![0, 0], vec![0, 1], vec![1, 0], vec![1, 1]];
|
||||
|
||||
for update in updates.iter() {
|
||||
storage.push_update("test".to_string(), update).await.unwrap();
|
||||
}
|
||||
|
||||
let result = storage.get_doc_updates("test".to_string()).await.unwrap();
|
||||
|
||||
assert_eq!(result.len(), 4);
|
||||
assert_eq!(result.iter().map(|u| u.bin.to_vec()).collect::<Vec<_>>(), updates);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_doc_snapshot() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
let none = storage.get_doc_snapshot("test".to_string()).await.unwrap();
|
||||
|
||||
assert!(none.is_none());
|
||||
|
||||
let snapshot = DocRecord {
|
||||
doc_id: "test".to_string(),
|
||||
bin: Into::<Data>::into(vec![0, 0]),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
};
|
||||
|
||||
storage.set_doc_snapshot(snapshot).await.unwrap();
|
||||
|
||||
let result = storage.get_doc_snapshot("test".to_string()).await.unwrap();
|
||||
|
||||
assert!(result.is_some());
|
||||
assert_eq!(result.unwrap().bin.to_vec(), vec![0, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_doc_snapshot() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
let snapshot = DocRecord {
|
||||
doc_id: "test".to_string(),
|
||||
bin: Into::<Data>::into(vec![0, 0]),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
};
|
||||
|
||||
storage.set_doc_snapshot(snapshot).await.unwrap();
|
||||
|
||||
let result = storage.get_doc_snapshot("test".to_string()).await.unwrap();
|
||||
|
||||
assert!(result.is_some());
|
||||
assert_eq!(result.unwrap().bin.to_vec(), vec![0, 0]);
|
||||
|
||||
let snapshot = DocRecord {
|
||||
doc_id: "test".to_string(),
|
||||
bin: Into::<Data>::into(vec![0, 1]),
|
||||
timestamp: DateTime::from_timestamp_millis(Utc::now().timestamp_millis() - 1000)
|
||||
.unwrap()
|
||||
.naive_utc(),
|
||||
};
|
||||
|
||||
// can't update because it's tempstamp is older
|
||||
storage.set_doc_snapshot(snapshot).await.unwrap();
|
||||
|
||||
let result = storage.get_doc_snapshot("test".to_string()).await.unwrap();
|
||||
|
||||
assert!(result.is_some());
|
||||
assert_eq!(result.unwrap().bin.to_vec(), vec![0, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_doc_clocks() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
let clocks = storage.get_doc_clocks(None).await.unwrap();
|
||||
|
||||
assert_eq!(clocks.len(), 0);
|
||||
|
||||
for i in 1..5u32 {
|
||||
storage.push_update(format!("test_{i}"), vec![0, 0]).await.unwrap();
|
||||
}
|
||||
|
||||
let clocks = storage.get_doc_clocks(None).await.unwrap();
|
||||
|
||||
assert_eq!(clocks.len(), 4);
|
||||
assert_eq!(
|
||||
clocks.iter().map(|c| c.doc_id.as_str()).collect::<Vec<_>>(),
|
||||
vec!["test_1", "test_2", "test_3", "test_4"]
|
||||
);
|
||||
|
||||
let clocks = storage.get_doc_clocks(Some(Utc::now().naive_utc())).await.unwrap();
|
||||
|
||||
assert_eq!(clocks.len(), 0);
|
||||
|
||||
let clock = storage.get_doc_clock("test_1".to_string()).await.unwrap();
|
||||
|
||||
assert!(clock.is_some());
|
||||
assert_eq!(clock.unwrap().doc_id, "test_1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mark_updates_merged() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
let updates = [vec![0, 0], vec![0, 1], vec![1, 0], vec![1, 1]];
|
||||
|
||||
for update in updates.iter() {
|
||||
storage.push_update("test".to_string(), update).await.unwrap();
|
||||
}
|
||||
|
||||
let updates = storage.get_doc_updates("test".to_string()).await.unwrap();
|
||||
|
||||
let result = storage
|
||||
.mark_updates_merged(
|
||||
"test".to_string(),
|
||||
updates.iter().skip(1).map(|u| u.timestamp).collect::<Vec<_>>(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, 3);
|
||||
|
||||
let updates = storage.get_doc_updates("test".to_string()).await.unwrap();
|
||||
|
||||
assert_eq!(updates.len(), 1);
|
||||
}
|
||||
}
|
||||
279
packages/frontend/native/nbstore/src/doc_sync.rs
Normal file
279
packages/frontend/native/nbstore/src/doc_sync.rs
Normal file
@@ -0,0 +1,279 @@
|
||||
use chrono::NaiveDateTime;
|
||||
|
||||
use super::{DocClock, error::Result, storage::SqliteDocStorage};
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn get_peer_remote_clocks(&self, peer: String) -> Result<Vec<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
"SELECT doc_id, remote_clock as timestamp FROM peer_clocks WHERE peer = ?",
|
||||
peer
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn get_peer_remote_clock(&self, peer: String, doc_id: String) -> Result<Option<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
"SELECT doc_id, remote_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?",
|
||||
peer,
|
||||
doc_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn set_peer_remote_clock(&self, peer: String, doc_id: String, clock: NaiveDateTime) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO peer_clocks (peer, doc_id, remote_clock)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(peer, doc_id)
|
||||
DO UPDATE SET remote_clock=$3 WHERE remote_clock < $3;"#,
|
||||
)
|
||||
.bind(peer)
|
||||
.bind(doc_id)
|
||||
.bind(clock)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_peer_pulled_remote_clocks(&self, peer: String) -> Result<Vec<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
"SELECT doc_id, pulled_remote_clock as timestamp FROM peer_clocks WHERE peer = ?",
|
||||
peer
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn get_peer_pulled_remote_clock(&self, peer: String, doc_id: String) -> Result<Option<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
r#"SELECT doc_id, pulled_remote_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?"#,
|
||||
peer,
|
||||
doc_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn set_peer_pulled_remote_clock(&self, peer: String, doc_id: String, clock: NaiveDateTime) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO peer_clocks (peer, doc_id, pulled_remote_clock)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(peer, doc_id)
|
||||
DO UPDATE SET pulled_remote_clock=$3 WHERE pulled_remote_clock < $3;"#,
|
||||
)
|
||||
.bind(peer)
|
||||
.bind(doc_id)
|
||||
.bind(clock)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_peer_pushed_clocks(&self, peer: String) -> Result<Vec<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
"SELECT doc_id, pushed_clock as timestamp FROM peer_clocks WHERE peer = ?",
|
||||
peer
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn get_peer_pushed_clock(&self, peer: String, doc_id: String) -> Result<Option<DocClock>> {
|
||||
let result = sqlx::query_as!(
|
||||
DocClock,
|
||||
"SELECT doc_id, pushed_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?",
|
||||
peer,
|
||||
doc_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn set_peer_pushed_clock(&self, peer: String, doc_id: String, clock: NaiveDateTime) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO peer_clocks (peer, doc_id, pushed_clock)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(peer, doc_id)
|
||||
DO UPDATE SET pushed_clock=$3 WHERE pushed_clock < $3;"#,
|
||||
)
|
||||
.bind(peer)
|
||||
.bind(doc_id)
|
||||
.bind(clock)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn clear_clocks(&self) -> Result<()> {
|
||||
sqlx::query("DELETE FROM peer_clocks;").execute(&self.pool).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::Row;
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn get_storage() -> SqliteDocStorage {
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_peer_clock() {
|
||||
let storage = get_storage().await;
|
||||
let peer = String::from("peer1");
|
||||
|
||||
let clocks = storage.get_peer_remote_clocks(peer.clone()).await.unwrap();
|
||||
|
||||
assert!(clocks.is_empty());
|
||||
|
||||
let clock = Utc::now().naive_utc();
|
||||
storage
|
||||
.set_peer_remote_clock(peer.clone(), "doc1".to_string(), clock)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let clocks = storage.get_peer_remote_clocks(peer.clone()).await.unwrap();
|
||||
|
||||
assert_eq!(clocks.len(), 1);
|
||||
assert_eq!(clocks.first().unwrap().doc_id, "doc1");
|
||||
assert_eq!(clocks.first().unwrap().timestamp, clock);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_peer_pushed_clock() {
|
||||
let storage = get_storage().await;
|
||||
let peer = String::from("peer1");
|
||||
|
||||
let clocks = storage.get_peer_pushed_clocks(peer.clone()).await.unwrap();
|
||||
|
||||
assert!(clocks.is_empty());
|
||||
|
||||
let clock = Utc::now().naive_utc();
|
||||
storage
|
||||
.set_peer_pushed_clock(peer.clone(), "doc1".to_string(), clock)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let clocks = storage.get_peer_pushed_clocks(peer.clone()).await.unwrap();
|
||||
|
||||
assert_eq!(clocks.len(), 1);
|
||||
assert_eq!(clocks.first().unwrap().doc_id, "doc1");
|
||||
assert_eq!(clocks.first().unwrap().timestamp, clock);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_clocks() {
|
||||
let storage = get_storage().await;
|
||||
let peer = String::from("peer1");
|
||||
|
||||
storage
|
||||
.set_peer_remote_clock(peer.clone(), "doc1".to_string(), Utc::now().naive_utc())
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.set_peer_pushed_clock(peer.clone(), "doc2".to_string(), Utc::now().naive_utc())
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.set_peer_pulled_remote_clock(peer.clone(), "doc3".to_string(), Utc::now().naive_utc())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let record = sqlx::query("SELECT * FROM peer_clocks WHERE peer = ? AND doc_id = ?")
|
||||
.bind(peer.clone())
|
||||
.bind("doc1")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
record.get::<NaiveDateTime, &str>("pushed_clock"),
|
||||
DateTime::from_timestamp(0, 0).unwrap().naive_utc()
|
||||
);
|
||||
|
||||
let record = sqlx::query("SELECT * FROM peer_clocks WHERE peer = ? AND doc_id = ?")
|
||||
.bind(peer.clone())
|
||||
.bind("doc2")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
record.get::<NaiveDateTime, &str>("remote_clock"),
|
||||
DateTime::from_timestamp(0, 0).unwrap().naive_utc()
|
||||
);
|
||||
|
||||
let record = sqlx::query("SELECT * FROM peer_clocks WHERE peer = ? AND doc_id = ?")
|
||||
.bind(peer.clone())
|
||||
.bind("doc3")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
record.get::<NaiveDateTime, &str>("remote_clock"),
|
||||
DateTime::from_timestamp(0, 0).unwrap().naive_utc()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clear_clocks() {
|
||||
let storage = get_storage().await;
|
||||
let peer = String::from("peer1");
|
||||
|
||||
storage
|
||||
.set_peer_remote_clock(peer.clone(), "doc1".to_string(), Utc::now().naive_utc())
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.set_peer_pushed_clock(peer.clone(), "doc2".to_string(), Utc::now().naive_utc())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let clocks = storage.get_peer_remote_clocks(peer.clone()).await.unwrap();
|
||||
assert_eq!(clocks.len(), 2);
|
||||
|
||||
let clocks = storage.get_peer_pushed_clocks(peer.clone()).await.unwrap();
|
||||
assert_eq!(clocks.len(), 2);
|
||||
|
||||
storage.clear_clocks().await.unwrap();
|
||||
|
||||
let clocks = storage.get_peer_remote_clocks(peer.clone()).await.unwrap();
|
||||
assert!(clocks.is_empty());
|
||||
|
||||
let clocks = storage.get_peer_pushed_clocks(peer.clone()).await.unwrap();
|
||||
assert!(clocks.is_empty());
|
||||
}
|
||||
}
|
||||
23
packages/frontend/native/nbstore/src/error.rs
Normal file
23
packages/frontend/native/nbstore/src/error.rs
Normal file
@@ -0,0 +1,23 @@
|
||||
use affine_doc_loader::ParseError;
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("Sqlite Error: {0}")]
|
||||
SqlxError(#[from] sqlx::Error),
|
||||
#[error("Migrate Error: {0}")]
|
||||
MigrateError(#[from] sqlx::migrate::MigrateError),
|
||||
#[error("Connection in progress")]
|
||||
ConnectionInProgress,
|
||||
#[error("Invalid operation")]
|
||||
InvalidOperation,
|
||||
#[error("Index is rebuilding")]
|
||||
IndexNotReady,
|
||||
#[error("Serialization Error: {0}")]
|
||||
Serialization(String),
|
||||
#[error(transparent)]
|
||||
Indexer(#[from] memory_indexer::Error),
|
||||
#[error(transparent)]
|
||||
Parse(#[from] ParseError),
|
||||
}
|
||||
39
packages/frontend/native/nbstore/src/indexer/crawl.rs
Normal file
39
packages/frontend/native/nbstore/src/indexer/crawl.rs
Normal file
@@ -0,0 +1,39 @@
|
||||
use affine_doc_loader::{ParseError, parse_doc_from_binary};
|
||||
use y_octo::merge_updates_v1;
|
||||
|
||||
use super::{NativeCrawlResult, SqliteDocStorage, error::Result};
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn crawl_doc_data(&self, doc_id: &str) -> Result<NativeCrawlResult> {
|
||||
let doc_bin = self.load_doc_binary(doc_id).await?.ok_or(ParseError::DocNotFound)?;
|
||||
Ok(parse_doc_from_binary(doc_bin, doc_id.to_string())?.into())
|
||||
}
|
||||
|
||||
async fn load_doc_binary(&self, doc_id: &str) -> Result<Option<Vec<u8>>> {
|
||||
let snapshot = self.get_doc_snapshot(doc_id.to_string()).await?;
|
||||
let mut updates = self.get_doc_updates(doc_id.to_string()).await?;
|
||||
if snapshot.is_none() && updates.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
updates.sort_by_key(|update| update.timestamp);
|
||||
let mut segments = Vec::with_capacity(snapshot.as_ref().map(|_| 1).unwrap_or(0) + updates.len());
|
||||
if let Some(record) = snapshot {
|
||||
segments.push(record.bin.to_vec());
|
||||
}
|
||||
segments.extend(updates.into_iter().map(|update| update.bin.to_vec()));
|
||||
merge_updates(segments).map(Some)
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_updates(mut segments: Vec<Vec<u8>>) -> Result<Vec<u8>> {
|
||||
if segments.is_empty() {
|
||||
return Err(ParseError::DocNotFound.into());
|
||||
}
|
||||
if segments.len() == 1 {
|
||||
return segments.pop().ok_or(ParseError::DocNotFound.into());
|
||||
}
|
||||
let update = merge_updates_v1(segments).map_err(|_| ParseError::InvalidBinary)?;
|
||||
update
|
||||
.encode_v1()
|
||||
.map_err(|error| ParseError::ParserError(error.to_string()).into())
|
||||
}
|
||||
142
packages/frontend/native/nbstore/src/indexer/dto.rs
Normal file
142
packages/frontend/native/nbstore/src/indexer/dto.rs
Normal file
@@ -0,0 +1,142 @@
|
||||
use affine_doc_loader::{BlockInfo, CrawlResult};
|
||||
use napi_derive::napi;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeBlockInfo {
|
||||
pub block_id: String,
|
||||
pub flavour: String,
|
||||
pub content: Option<Vec<String>>,
|
||||
pub blob: Option<Vec<String>>,
|
||||
pub ref_doc_id: Option<Vec<String>>,
|
||||
pub ref_info: Option<Vec<String>>,
|
||||
pub parent_flavour: Option<String>,
|
||||
pub parent_block_id: Option<String>,
|
||||
pub additional: Option<String>,
|
||||
}
|
||||
|
||||
impl From<BlockInfo> for NativeBlockInfo {
|
||||
fn from(value: BlockInfo) -> Self {
|
||||
Self {
|
||||
block_id: value.block_id,
|
||||
flavour: value.flavour,
|
||||
content: value.content,
|
||||
blob: value.blob,
|
||||
ref_doc_id: value.ref_doc_id,
|
||||
ref_info: value.ref_info,
|
||||
parent_flavour: value.parent_flavour,
|
||||
parent_block_id: value.parent_block_id,
|
||||
additional: value.additional,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeCrawlResult {
|
||||
pub blocks: Vec<NativeBlockInfo>,
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
impl From<CrawlResult> for NativeCrawlResult {
|
||||
fn from(value: CrawlResult) -> Self {
|
||||
Self {
|
||||
blocks: value.blocks.into_iter().map(Into::into).collect(),
|
||||
title: value.title,
|
||||
summary: value.summary,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NativeIndexField {
|
||||
pub field: String,
|
||||
pub values: Vec<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NativeIndexDocument {
|
||||
pub id: String,
|
||||
pub fields: Vec<NativeIndexField>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NativeIndexQuery {
|
||||
pub kind: String,
|
||||
pub field: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub occur: Option<String>,
|
||||
pub clauses: Option<Vec<NativeIndexQuery>>,
|
||||
pub boost: Option<f64>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NativeIndexSearchOptions {
|
||||
pub limit: u32,
|
||||
pub offset: u32,
|
||||
pub fields: Vec<String>,
|
||||
pub highlights: Vec<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexSpan {
|
||||
pub start: u32,
|
||||
pub end: u32,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexHighlightValue {
|
||||
pub value_index: u32,
|
||||
pub spans: Vec<NativeIndexSpan>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexHighlight {
|
||||
pub field: String,
|
||||
pub values: Vec<NativeIndexHighlightValue>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexHit {
|
||||
pub id: String,
|
||||
pub score: f64,
|
||||
pub fields: Vec<NativeIndexField>,
|
||||
pub highlights: Vec<NativeIndexHighlight>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexSearchResult {
|
||||
pub total: u32,
|
||||
pub hits: Vec<NativeIndexHit>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexBucket {
|
||||
pub key: String,
|
||||
pub count: u32,
|
||||
pub score: f64,
|
||||
pub hits: Vec<NativeIndexHit>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct NativeIndexAggregateResult {
|
||||
pub total: u32,
|
||||
pub buckets: Vec<NativeIndexBucket>,
|
||||
}
|
||||
71
packages/frontend/native/nbstore/src/indexer/mod.rs
Normal file
71
packages/frontend/native/nbstore/src/indexer/mod.rs
Normal file
@@ -0,0 +1,71 @@
|
||||
mod crawl;
|
||||
mod dto;
|
||||
mod persistence;
|
||||
mod table;
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
pub use dto::{
|
||||
NativeBlockInfo, NativeCrawlResult, NativeIndexAggregateResult, NativeIndexBucket, NativeIndexDocument,
|
||||
NativeIndexField, NativeIndexHighlight, NativeIndexHighlightValue, NativeIndexHit, NativeIndexQuery,
|
||||
NativeIndexSearchOptions, NativeIndexSearchResult, NativeIndexSpan,
|
||||
};
|
||||
use error::{Error, Result};
|
||||
use memory_indexer::MemoryIndex;
|
||||
pub(super) use table::{TableIndex, string_values};
|
||||
|
||||
pub(super) use super::{DocIndexedClock, error, storage::SqliteDocStorage};
|
||||
|
||||
const NBSTORE_INDEXER_VERSION: u32 = 7;
|
||||
|
||||
pub struct IndexManager {
|
||||
pub(super) doc: TableIndex,
|
||||
pub(super) block: TableIndex,
|
||||
pub(super) ready: AtomicBool,
|
||||
}
|
||||
|
||||
impl IndexManager {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
doc: TableIndex::doc(),
|
||||
block: TableIndex::block(),
|
||||
ready: AtomicBool::new(true),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn table(&self, table: &str) -> Result<&TableIndex> {
|
||||
match table {
|
||||
"doc" => Ok(&self.doc),
|
||||
"block" => Ok(&self.block),
|
||||
_ => Err(Error::Serialization(format!("unknown index table {table}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn tables(&self) -> [(&'static str, &TableIndex); 2] {
|
||||
[("doc", &self.doc), ("block", &self.block)]
|
||||
}
|
||||
|
||||
pub(super) fn ensure_ready(&self) -> Result<()> {
|
||||
self
|
||||
.ready
|
||||
.load(Ordering::Acquire)
|
||||
.then_some(())
|
||||
.ok_or(Error::IndexNotReady)
|
||||
}
|
||||
|
||||
pub(super) async fn reset(&self) {
|
||||
for (_, table) in self.tables() {
|
||||
*table.index.write().await = MemoryIndex::new(table.schema.clone());
|
||||
}
|
||||
self.ready.store(false, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for IndexManager {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
193
packages/frontend/native/nbstore/src/indexer/persistence.rs
Normal file
193
packages/frontend/native/nbstore/src/indexer/persistence.rs
Normal file
@@ -0,0 +1,193 @@
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use memory_indexer::{Document, MemoryIndex, TermsAggregation, Value};
|
||||
use sqlx::Row;
|
||||
|
||||
use super::{
|
||||
DocIndexedClock, NBSTORE_INDEXER_VERSION, NativeIndexAggregateResult, NativeIndexBucket, NativeIndexDocument,
|
||||
NativeIndexQuery, NativeIndexSearchOptions, NativeIndexSearchResult, SqliteDocStorage, error::Result, string_values,
|
||||
};
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn init_index(&self) -> Result<()> {
|
||||
let snapshots = sqlx::query("SELECT index_name, data FROM idx_snapshots WHERE index_name IN ('doc', 'block')")
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
let mut corrupted = false;
|
||||
for row in snapshots {
|
||||
let name: String = row.get("index_name");
|
||||
let data: Vec<u8> = row.get("data");
|
||||
let table = self.indexes.table(&name)?;
|
||||
if let Ok(index) = MemoryIndex::from_checkpoint(table.schema.clone(), &data) {
|
||||
*table.index.write().await = index;
|
||||
} else {
|
||||
corrupted = true;
|
||||
}
|
||||
}
|
||||
if corrupted {
|
||||
self.indexes.reset().await;
|
||||
let mut tx = self.pool.begin().await?;
|
||||
sqlx::query("DELETE FROM idx_snapshots").execute(&mut *tx).await?;
|
||||
sqlx::query("DELETE FROM indexer_sync").execute(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
}
|
||||
sqlx::query("DELETE FROM idx_snapshots WHERE index_name NOT IN ('doc', 'block')")
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn flush_index(&self) -> Result<()> {
|
||||
let mut checkpoints = Vec::new();
|
||||
for (name, table) in self.indexes.tables() {
|
||||
let index = table.index.read().await;
|
||||
if index.has_unpersisted_changes() {
|
||||
checkpoints.push((name, index.checkpoint()?));
|
||||
}
|
||||
}
|
||||
if checkpoints.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut tx = self.pool.begin().await?;
|
||||
for (name, checkpoint) in &checkpoints {
|
||||
sqlx::query("INSERT OR REPLACE INTO idx_snapshots (index_name, data) VALUES (?, ?)")
|
||||
.bind(name)
|
||||
.bind(&checkpoint.bytes)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
for (name, checkpoint) in checkpoints {
|
||||
self
|
||||
.indexes
|
||||
.table(name)?
|
||||
.index
|
||||
.write()
|
||||
.await
|
||||
.mark_checkpoint_persisted(checkpoint.sequence)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn commit_indexed_clocks(&self, clocks: &[DocIndexedClock]) -> Result<()> {
|
||||
let mut checkpoints = Vec::new();
|
||||
for (name, table) in self.indexes.tables() {
|
||||
checkpoints.push((name, table.index.read().await.checkpoint()?));
|
||||
}
|
||||
let mut tx = self.pool.begin().await?;
|
||||
for (name, checkpoint) in &checkpoints {
|
||||
sqlx::query("INSERT OR REPLACE INTO idx_snapshots (index_name, data) VALUES (?, ?)")
|
||||
.bind(name)
|
||||
.bind(&checkpoint.bytes)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
for clock in clocks {
|
||||
sqlx::query(
|
||||
r#"INSERT INTO indexer_sync (doc_id, indexed_clock, indexer_version)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(doc_id)
|
||||
DO UPDATE SET indexed_clock=$2, indexer_version=$3"#,
|
||||
)
|
||||
.bind(&clock.doc_id)
|
||||
.bind(clock.timestamp)
|
||||
.bind(clock.indexer_version)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
for (name, checkpoint) in checkpoints {
|
||||
self
|
||||
.indexes
|
||||
.table(name)?
|
||||
.index
|
||||
.write()
|
||||
.await
|
||||
.mark_checkpoint_persisted(checkpoint.sequence)?;
|
||||
}
|
||||
self.indexes.ready.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn index_version() -> u32 {
|
||||
NBSTORE_INDEXER_VERSION
|
||||
}
|
||||
|
||||
pub async fn index_upsert(&self, table: &str, document: NativeIndexDocument) -> Result<()> {
|
||||
let table = self.indexes.table(table)?;
|
||||
let mut index_document = Document::new(document.id);
|
||||
for field in document.fields {
|
||||
let field_id = table.field(&field.field)?;
|
||||
index_document.add_values(field_id, field.values.into_iter().map(Value::String));
|
||||
}
|
||||
table.index.write().await.upsert(index_document)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn index_delete(&self, table: &str, id: &str) -> Result<()> {
|
||||
self.indexes.table(table)?.index.write().await.delete(id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn index_search(
|
||||
&self,
|
||||
table: &str,
|
||||
query: NativeIndexQuery,
|
||||
options: NativeIndexSearchOptions,
|
||||
) -> Result<NativeIndexSearchResult> {
|
||||
self.indexes.ensure_ready()?;
|
||||
let table = self.indexes.table(table)?;
|
||||
let query = table.compile_query(query)?;
|
||||
let result = table
|
||||
.index
|
||||
.read()
|
||||
.await
|
||||
.search(&query, table.compile_options(options)?)?;
|
||||
Ok(NativeIndexSearchResult {
|
||||
total: result.total as u32,
|
||||
hits: result.hits.into_iter().map(|hit| table.hit(hit)).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn index_aggregate(
|
||||
&self,
|
||||
table: &str,
|
||||
query: NativeIndexQuery,
|
||||
field: &str,
|
||||
limit: u32,
|
||||
offset: u32,
|
||||
hits: Option<NativeIndexSearchOptions>,
|
||||
) -> Result<NativeIndexAggregateResult> {
|
||||
self.indexes.ensure_ready()?;
|
||||
let table = self.indexes.table(table)?;
|
||||
let query = table.compile_query(query)?;
|
||||
let result = table.index.read().await.aggregate(
|
||||
&query,
|
||||
TermsAggregation {
|
||||
field: table.field(field)?,
|
||||
limit: limit as usize,
|
||||
offset: offset as usize,
|
||||
top_hits: hits.map(|options| table.compile_options(options)).transpose()?,
|
||||
},
|
||||
)?;
|
||||
Ok(NativeIndexAggregateResult {
|
||||
total: result.total as u32,
|
||||
buckets: result
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| NativeIndexBucket {
|
||||
key: string_values(vec![bucket.key]).pop().unwrap_or_default(),
|
||||
count: bucket.count as u32,
|
||||
score: bucket.max_score as f64,
|
||||
hits: bucket.hits.into_iter().map(|hit| table.hit(hit)).collect(),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn index_delete_by_query(&self, table: &str, query: NativeIndexQuery) -> Result<u32> {
|
||||
let table = self.indexes.table(table)?;
|
||||
let query = table.compile_query(query)?;
|
||||
Ok(table.index.write().await.delete_by_query(&query)? as u32)
|
||||
}
|
||||
}
|
||||
223
packages/frontend/native/nbstore/src/indexer/table.rs
Normal file
223
packages/frontend/native/nbstore/src/indexer/table.rs
Normal file
@@ -0,0 +1,223 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use memory_indexer::{
|
||||
FieldId, FieldOptions, FieldType, Highlight, MemoryIndex, PositionEncoding, Query, Schema, SearchMode, SearchOptions,
|
||||
TextOptions, Value,
|
||||
};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::{
|
||||
NativeIndexField, NativeIndexHighlight, NativeIndexHighlightValue, NativeIndexHit, NativeIndexQuery,
|
||||
NativeIndexSearchOptions, NativeIndexSpan,
|
||||
error::{Error, Result},
|
||||
};
|
||||
|
||||
pub(crate) struct TableIndex {
|
||||
pub(super) schema: Schema,
|
||||
fields: HashMap<String, FieldId>,
|
||||
pub(super) index: RwLock<MemoryIndex>,
|
||||
}
|
||||
|
||||
impl TableIndex {
|
||||
pub(super) fn doc() -> Self {
|
||||
let mut schema = Schema::builder().position_encoding(PositionEncoding::Utf16);
|
||||
let fields = HashMap::from([
|
||||
("docId".into(), schema.keyword("docId", FieldOptions::indexed_stored())),
|
||||
(
|
||||
"title".into(),
|
||||
schema.text("title", text_options(), FieldOptions::indexed_stored()),
|
||||
),
|
||||
(
|
||||
"summary".into(),
|
||||
schema.keyword("summary", FieldOptions::new().stored()),
|
||||
),
|
||||
]);
|
||||
Self::from_schema(schema, fields)
|
||||
}
|
||||
|
||||
pub(super) fn block() -> Self {
|
||||
let mut schema = Schema::builder().position_encoding(PositionEncoding::Utf16);
|
||||
let fields = HashMap::from([
|
||||
("docId".into(), schema.keyword("docId", keyword_options())),
|
||||
("blockId".into(), schema.keyword("blockId", keyword_options())),
|
||||
(
|
||||
"content".into(),
|
||||
schema.text("content", text_options(), FieldOptions::indexed_stored().multi_value()),
|
||||
),
|
||||
("flavour".into(), schema.keyword("flavour", keyword_options())),
|
||||
("blob".into(), schema.keyword("blob", keyword_options())),
|
||||
("refDocId".into(), schema.keyword("refDocId", keyword_options())),
|
||||
(
|
||||
"ref".into(),
|
||||
schema.keyword("ref", FieldOptions::new().stored().multi_value()),
|
||||
),
|
||||
(
|
||||
"parentFlavour".into(),
|
||||
schema.keyword("parentFlavour", keyword_options()),
|
||||
),
|
||||
(
|
||||
"parentBlockId".into(),
|
||||
schema.keyword("parentBlockId", keyword_options()),
|
||||
),
|
||||
(
|
||||
"additional".into(),
|
||||
schema.keyword("additional", FieldOptions::new().stored().multi_value()),
|
||||
),
|
||||
(
|
||||
"markdownPreview".into(),
|
||||
schema.keyword("markdownPreview", FieldOptions::new().stored().multi_value()),
|
||||
),
|
||||
]);
|
||||
Self::from_schema(schema, fields)
|
||||
}
|
||||
|
||||
fn from_schema(builder: memory_indexer::SchemaBuilder, fields: HashMap<String, FieldId>) -> Self {
|
||||
let schema = builder.build().expect("static nbstore index schema must be valid");
|
||||
Self {
|
||||
index: RwLock::new(MemoryIndex::new(schema.clone())),
|
||||
schema,
|
||||
fields,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn field(&self, name: &str) -> Result<FieldId> {
|
||||
self
|
||||
.fields
|
||||
.get(name)
|
||||
.copied()
|
||||
.ok_or_else(|| Error::Serialization(format!("unknown index field {name}")))
|
||||
}
|
||||
|
||||
pub(super) fn compile_query(&self, query: NativeIndexQuery) -> Result<Query> {
|
||||
match query.kind.as_str() {
|
||||
"match" => {
|
||||
let field = self.field(query.field.as_deref().unwrap_or_default())?;
|
||||
let value = query.value.unwrap_or_default();
|
||||
match self.schema.field(field).map(|field| &field.field_type) {
|
||||
Some(FieldType::Text(_)) => Ok(Query::text(field, value, SearchMode::Auto)),
|
||||
Some(FieldType::Keyword) => Ok(Query::term(field, value)),
|
||||
_ => Err(Error::Serialization("match requires Text or Keyword field".into())),
|
||||
}
|
||||
}
|
||||
"exists" => Ok(Query::Exists(self.field(query.field.as_deref().unwrap_or_default())?)),
|
||||
"all" => Ok(Query::All),
|
||||
"boost" => {
|
||||
let clause = query
|
||||
.clauses
|
||||
.unwrap_or_default()
|
||||
.pop()
|
||||
.ok_or_else(|| Error::Serialization("boost requires one clause".into()))?;
|
||||
Ok(Query::Boost {
|
||||
query: Box::new(self.compile_query(clause)?),
|
||||
factor: query.boost.unwrap_or(1.0) as f32,
|
||||
})
|
||||
}
|
||||
"boolean" => {
|
||||
let clauses = query
|
||||
.clauses
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|query| self.compile_query(query))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let (must, should, must_not) = match query.occur.as_deref() {
|
||||
Some("must") => (clauses, vec![], vec![]),
|
||||
Some("should") => (vec![], clauses, vec![]),
|
||||
Some("must_not") => (vec![], vec![], clauses),
|
||||
_ => return Err(Error::Serialization("invalid boolean occurrence".into())),
|
||||
};
|
||||
Ok(Query::boolean(must, should, must_not))
|
||||
}
|
||||
kind => Err(Error::Serialization(format!("unknown query kind {kind}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn compile_options(&self, options: NativeIndexSearchOptions) -> Result<SearchOptions> {
|
||||
Ok(SearchOptions {
|
||||
limit: options.limit as usize,
|
||||
offset: options.offset as usize,
|
||||
after: None,
|
||||
sort: vec![],
|
||||
stored_fields: options
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| self.field(field))
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
highlight_fields: options
|
||||
.highlights
|
||||
.iter()
|
||||
.map(|field| self.field(field))
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn hit(&self, hit: memory_indexer::SearchHit) -> NativeIndexHit {
|
||||
NativeIndexHit {
|
||||
id: hit.id,
|
||||
score: hit.score as f64,
|
||||
fields: hit
|
||||
.fields
|
||||
.into_iter()
|
||||
.map(|(field, values)| NativeIndexField {
|
||||
field: self.field_name(field),
|
||||
values: string_values(values),
|
||||
})
|
||||
.collect(),
|
||||
highlights: group_highlights(self, hit.highlights),
|
||||
}
|
||||
}
|
||||
|
||||
fn field_name(&self, field: FieldId) -> String {
|
||||
self
|
||||
.schema
|
||||
.field(field)
|
||||
.expect("result field belongs to schema")
|
||||
.name
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn string_values(values: Vec<Value>) -> Vec<String> {
|
||||
values
|
||||
.into_iter()
|
||||
.filter_map(|value| match value {
|
||||
Value::String(value) => Some(value),
|
||||
Value::I64(_) | Value::Bool(_) => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn keyword_options() -> FieldOptions {
|
||||
FieldOptions::indexed_stored().multi_value()
|
||||
}
|
||||
|
||||
fn text_options() -> TextOptions {
|
||||
TextOptions::multilingual()
|
||||
.with_pinyin()
|
||||
.with_prefix()
|
||||
.with_fuzzy()
|
||||
.with_positions()
|
||||
}
|
||||
|
||||
fn group_highlights(table: &TableIndex, highlights: Vec<Highlight>) -> Vec<NativeIndexHighlight> {
|
||||
let mut grouped: Vec<NativeIndexHighlight> = Vec::new();
|
||||
for highlight in highlights {
|
||||
let field = table.field_name(highlight.field);
|
||||
let values = NativeIndexHighlightValue {
|
||||
value_index: highlight.value_index as u32,
|
||||
spans: highlight
|
||||
.spans
|
||||
.into_iter()
|
||||
.map(|(start, end)| NativeIndexSpan { start, end })
|
||||
.collect(),
|
||||
};
|
||||
if let Some(existing) = grouped.iter_mut().find(|item| item.field == field) {
|
||||
existing.values.push(values);
|
||||
} else {
|
||||
grouped.push(NativeIndexHighlight {
|
||||
field,
|
||||
values: vec![values],
|
||||
});
|
||||
}
|
||||
}
|
||||
grouped
|
||||
}
|
||||
318
packages/frontend/native/nbstore/src/indexer/tests.rs
Normal file
318
packages/frontend/native/nbstore/src/indexer/tests.rs
Normal file
@@ -0,0 +1,318 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use affine_doc_loader::ParseError;
|
||||
use assert_json_diff::assert_json_eq;
|
||||
use chrono::Utc;
|
||||
use serde_json::Value;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
super::{DocIndexedClock, error::Error, storage::SqliteDocStorage},
|
||||
NativeIndexDocument, NativeIndexField, NativeIndexQuery, NativeIndexSearchOptions,
|
||||
};
|
||||
|
||||
const DEMO_BIN: &[u8] = include_bytes!("../../../../../common/native/fixtures/demo.ydoc");
|
||||
const DEMO_JSON: &[u8] = include_bytes!("../../../../../common/native/fixtures/demo.ydoc.json");
|
||||
|
||||
fn temp_workspace_dir() -> PathBuf {
|
||||
std::env::temp_dir().join(format!("affine-native-{}", Uuid::new_v4()))
|
||||
}
|
||||
|
||||
async fn init_db(path: &Path) -> SqliteDocStorage {
|
||||
fs::create_dir_all(path.parent().unwrap()).await.unwrap();
|
||||
let storage = SqliteDocStorage::new(path.to_string_lossy().into_owned());
|
||||
storage.connect().await.unwrap();
|
||||
storage
|
||||
}
|
||||
|
||||
async fn cleanup(path: &Path) {
|
||||
let _ = fs::remove_dir_all(path.parent().unwrap()).await;
|
||||
}
|
||||
|
||||
fn query(kind: &str, field: Option<&str>, value: Option<&str>) -> NativeIndexQuery {
|
||||
NativeIndexQuery {
|
||||
kind: kind.into(),
|
||||
field: field.map(Into::into),
|
||||
value: value.map(Into::into),
|
||||
occur: None,
|
||||
clauses: None,
|
||||
boost: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn options(fields: &[&str], highlights: &[&str]) -> NativeIndexSearchOptions {
|
||||
NativeIndexSearchOptions {
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
fields: fields.iter().map(|value| (*value).into()).collect(),
|
||||
highlights: highlights.iter().map(|value| (*value).into()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn document(id: &str, fields: &[(&str, &[&str])]) -> NativeIndexDocument {
|
||||
NativeIndexDocument {
|
||||
id: id.into(),
|
||||
fields: fields
|
||||
.iter()
|
||||
.map(|(field, values)| NativeIndexField {
|
||||
field: (*field).into(),
|
||||
values: values.iter().map(|value| (*value).into()).collect(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parse_demo_snapshot_matches_fixture() {
|
||||
let base = temp_workspace_dir();
|
||||
fs::create_dir_all(&base).await.unwrap();
|
||||
let db_path = base.join("storage.db");
|
||||
let storage = init_db(&db_path).await;
|
||||
sqlx::query(r#"INSERT INTO snapshots (doc_id, data, updated_at) VALUES (?, ?, ?)"#)
|
||||
.bind("demo-doc")
|
||||
.bind(DEMO_BIN)
|
||||
.bind(Utc::now().naive_utc())
|
||||
.execute(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(r#"INSERT INTO updates (doc_id, data, created_at) VALUES (?, ?, ?)"#)
|
||||
.bind("demo-doc")
|
||||
.bind(&[0, 0][..])
|
||||
.bind(Utc::now().naive_utc())
|
||||
.execute(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = storage.crawl_doc_data("demo-doc").await.unwrap();
|
||||
let mut expected: Value = serde_json::from_slice(DEMO_JSON).unwrap();
|
||||
let mut actual = serde_json::to_value(&result).unwrap();
|
||||
for document in [&mut expected, &mut actual] {
|
||||
for block in document["blocks"].as_array_mut().unwrap() {
|
||||
if let Some(additional) = block["additional"].as_str() {
|
||||
block["additional"] = serde_json::from_str(additional).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_json_eq!(expected, actual);
|
||||
storage.close().await;
|
||||
cleanup(&db_path).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_doc_returns_error() {
|
||||
let db_path = temp_workspace_dir().join("storage.db");
|
||||
let storage = init_db(&db_path).await;
|
||||
let error = storage.crawl_doc_data("absent-doc").await.unwrap_err();
|
||||
assert!(matches!(error, Error::Parse(ParseError::DocNotFound)));
|
||||
storage.close().await;
|
||||
cleanup(&db_path).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn index_tables_support_terminal_queries_and_restart() {
|
||||
let db_path = temp_workspace_dir().join("storage.db");
|
||||
let storage = init_db(&db_path).await;
|
||||
storage
|
||||
.index_upsert(
|
||||
"doc",
|
||||
document(
|
||||
"doc-1",
|
||||
&[
|
||||
("docId", &["doc-1"]),
|
||||
("title", &["Rust 搜索"]),
|
||||
("summary", &["stored summary"]),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.index_upsert(
|
||||
"block",
|
||||
document(
|
||||
"block-1",
|
||||
&[
|
||||
("docId", &["doc-1"]),
|
||||
("blockId", &["block-1"]),
|
||||
("content", &["hello world", "你好搜索"]),
|
||||
("flavour", &["affine:paragraph"]),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.index_upsert(
|
||||
"block",
|
||||
document(
|
||||
"block-2",
|
||||
&[
|
||||
("docId", &["doc-10"]),
|
||||
("blockId", &["block-2"]),
|
||||
("content", &["hello unrelated"]),
|
||||
("flavour", &["affine:code"]),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let text = storage
|
||||
.index_search(
|
||||
"block",
|
||||
query("match", Some("content"), Some("搜索")),
|
||||
options(&["content"], &["content"]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(text.total, 1);
|
||||
assert_eq!(text.hits[0].id, "block-1");
|
||||
assert!(!text.hits[0].highlights[0].values[0].spans.is_empty());
|
||||
|
||||
let exact = storage
|
||||
.index_search(
|
||||
"block",
|
||||
query("match", Some("docId"), Some("doc-1")),
|
||||
options(&["docId"], &[]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(exact.total, 1);
|
||||
assert_eq!(exact.hits[0].fields[0].values, ["doc-1"]);
|
||||
|
||||
let aggregate = storage
|
||||
.index_aggregate(
|
||||
"block",
|
||||
query("all", None, None),
|
||||
"flavour",
|
||||
10,
|
||||
0,
|
||||
Some(options(&["blockId"], &[])),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(aggregate.total, 2);
|
||||
assert_eq!(aggregate.buckets.len(), 2);
|
||||
|
||||
let clock = DocIndexedClock {
|
||||
doc_id: "doc-1".into(),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
indexer_version: SqliteDocStorage::index_version() as i64,
|
||||
};
|
||||
storage.commit_indexed_clocks(&[clock]).await.unwrap();
|
||||
storage.close().await;
|
||||
|
||||
let restored = init_db(&db_path).await;
|
||||
let result = restored
|
||||
.index_search(
|
||||
"block",
|
||||
query("match", Some("content"), Some("hello")),
|
||||
options(&[], &[]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.total, 2);
|
||||
assert_eq!(
|
||||
restored
|
||||
.index_delete_by_query("block", query("match", Some("docId"), Some("doc-1")))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
restored.close().await;
|
||||
cleanup(&db_path).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_checkpoint_requires_rebuild_and_clears_clocks() {
|
||||
let db_path = temp_workspace_dir().join("storage.db");
|
||||
let storage = init_db(&db_path).await;
|
||||
storage
|
||||
.index_upsert(
|
||||
"doc",
|
||||
document("doc-1", &[("docId", &["doc-1"]), ("title", &["title"])]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let clock = DocIndexedClock {
|
||||
doc_id: "doc-1".into(),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
indexer_version: SqliteDocStorage::index_version() as i64,
|
||||
};
|
||||
storage.commit_indexed_clocks(&[clock]).await.unwrap();
|
||||
sqlx::query("UPDATE idx_snapshots SET data = x'00' WHERE index_name = 'doc'")
|
||||
.execute(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
storage.close().await;
|
||||
|
||||
let restored = init_db(&db_path).await;
|
||||
assert!(matches!(
|
||||
restored
|
||||
.index_search("doc", query("all", None, None), options(&[], &[]))
|
||||
.await,
|
||||
Err(Error::IndexNotReady)
|
||||
));
|
||||
assert!(restored.get_doc_indexed_clock("doc-1".into()).await.unwrap().is_none());
|
||||
restored
|
||||
.index_upsert(
|
||||
"doc",
|
||||
document("doc-1", &[("docId", &["doc-1"]), ("title", &["rebuilt"])]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
restored
|
||||
.commit_indexed_clocks(&[DocIndexedClock {
|
||||
doc_id: "doc-1".into(),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
indexer_version: SqliteDocStorage::index_version() as i64,
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
restored
|
||||
.index_search("doc", query("all", None, None), options(&[], &[]))
|
||||
.await
|
||||
.unwrap()
|
||||
.total,
|
||||
1
|
||||
);
|
||||
restored.close().await;
|
||||
cleanup(&db_path).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_atomic_commit_keeps_index_dirty_and_clock_unadvanced() {
|
||||
let db_path = temp_workspace_dir().join("storage.db");
|
||||
let storage = init_db(&db_path).await;
|
||||
storage
|
||||
.index_upsert(
|
||||
"doc",
|
||||
document("doc-1", &[("docId", &["doc-1"]), ("title", &["title"])]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("DROP TABLE idx_snapshots")
|
||||
.execute(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let clock = DocIndexedClock {
|
||||
doc_id: "doc-1".into(),
|
||||
timestamp: Utc::now().naive_utc(),
|
||||
indexer_version: SqliteDocStorage::index_version() as i64,
|
||||
};
|
||||
assert!(storage.commit_indexed_clocks(&[clock]).await.is_err());
|
||||
assert!(
|
||||
storage
|
||||
.indexes
|
||||
.table("doc")
|
||||
.unwrap()
|
||||
.index
|
||||
.read()
|
||||
.await
|
||||
.has_unpersisted_changes()
|
||||
);
|
||||
assert!(storage.get_doc_indexed_clock("doc-1".into()).await.unwrap().is_none());
|
||||
storage.close().await;
|
||||
cleanup(&db_path).await;
|
||||
}
|
||||
99
packages/frontend/native/nbstore/src/indexer_sync.rs
Normal file
99
packages/frontend/native/nbstore/src/indexer_sync.rs
Normal file
@@ -0,0 +1,99 @@
|
||||
use chrono::NaiveDateTime;
|
||||
|
||||
use super::{DocIndexedClock, error::Result, storage::SqliteDocStorage};
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub async fn get_doc_indexed_clock(&self, doc_id: String) -> Result<Option<DocIndexedClock>> {
|
||||
let record = sqlx::query!(
|
||||
r#"SELECT doc_id, indexed_clock as "indexed_clock: NaiveDateTime", indexer_version
|
||||
FROM indexer_sync WHERE doc_id = ?"#,
|
||||
doc_id
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(record.map(|rec| DocIndexedClock {
|
||||
doc_id: rec.doc_id,
|
||||
timestamp: rec.indexed_clock,
|
||||
indexer_version: rec.indexer_version,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn set_doc_indexed_clock(
|
||||
&self,
|
||||
doc_id: String,
|
||||
indexed_clock: NaiveDateTime,
|
||||
indexer_version: i64,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO indexer_sync (doc_id, indexed_clock, indexer_version)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(doc_id)
|
||||
DO UPDATE SET indexed_clock=$2, indexer_version=$3;
|
||||
"#,
|
||||
)
|
||||
.bind(doc_id)
|
||||
.bind(indexed_clock)
|
||||
.bind(indexer_version)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn clear_doc_indexed_clock(&self, doc_id: String) -> Result<()> {
|
||||
sqlx::query("DELETE FROM indexer_sync WHERE doc_id = ?;")
|
||||
.bind(doc_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::Utc;
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn get_storage() -> SqliteDocStorage {
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_and_get_indexed_clock() {
|
||||
let storage = get_storage().await;
|
||||
let ts = Utc::now().naive_utc();
|
||||
|
||||
storage.set_doc_indexed_clock("doc1".to_string(), ts, 1).await.unwrap();
|
||||
|
||||
let clock = storage
|
||||
.get_doc_indexed_clock("doc1".to_string())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(clock.doc_id, "doc1");
|
||||
assert_eq!(clock.timestamp, ts);
|
||||
assert_eq!(clock.indexer_version, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clear_indexed_clock() {
|
||||
let storage = get_storage().await;
|
||||
let ts = Utc::now().naive_utc();
|
||||
|
||||
storage.set_doc_indexed_clock("doc1".to_string(), ts, 1).await.unwrap();
|
||||
|
||||
storage.clear_doc_indexed_clock("doc1".to_string()).await.unwrap();
|
||||
|
||||
let clock = storage.get_doc_indexed_clock("doc1".to_string()).await.unwrap();
|
||||
|
||||
assert!(clock.is_none());
|
||||
}
|
||||
}
|
||||
544
packages/frontend/native/nbstore/src/lib.rs
Normal file
544
packages/frontend/native/nbstore/src/lib.rs
Normal file
@@ -0,0 +1,544 @@
|
||||
pub mod blob;
|
||||
pub mod blob_sync;
|
||||
pub mod doc;
|
||||
pub mod doc_sync;
|
||||
pub mod error;
|
||||
pub mod indexer;
|
||||
pub mod indexer_sync;
|
||||
pub mod pool;
|
||||
pub mod storage;
|
||||
|
||||
#[cfg(not(feature = "use-as-lib"))]
|
||||
use affine_common::napi_utils::to_napi_error;
|
||||
use chrono::NaiveDateTime;
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
use pool::{Ref, SqliteDocStoragePool};
|
||||
use storage::SqliteDocStorage;
|
||||
|
||||
#[cfg(feature = "use-as-lib")]
|
||||
type Result<T> = anyhow::Result<T>;
|
||||
|
||||
#[cfg(not(feature = "use-as-lib"))]
|
||||
type Result<T> = napi::Result<T>;
|
||||
|
||||
#[cfg(not(feature = "use-as-lib"))]
|
||||
impl From<error::Error> for napi::Error {
|
||||
fn from(err: error::Error) -> Self {
|
||||
to_napi_error(err, napi::Status::GenericFailure)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "use-as-lib")]
|
||||
pub type Data = Vec<u8>;
|
||||
|
||||
#[cfg(not(feature = "use-as-lib"))]
|
||||
pub type Data = Uint8Array;
|
||||
|
||||
#[napi(object)]
|
||||
pub struct DocUpdate {
|
||||
pub doc_id: String,
|
||||
pub timestamp: NaiveDateTime,
|
||||
#[napi(ts_type = "Uint8Array")]
|
||||
pub bin: Data,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct DocRecord {
|
||||
pub doc_id: String,
|
||||
#[napi(ts_type = "Uint8Array")]
|
||||
pub bin: Data,
|
||||
pub timestamp: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[napi(object)]
|
||||
pub struct DocClock {
|
||||
pub doc_id: String,
|
||||
pub timestamp: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[napi(object)]
|
||||
pub struct DocIndexedClock {
|
||||
pub doc_id: String,
|
||||
pub timestamp: NaiveDateTime,
|
||||
pub indexer_version: i64,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct SetBlob {
|
||||
pub key: String,
|
||||
#[napi(ts_type = "Uint8Array")]
|
||||
pub data: Data,
|
||||
pub mime: String,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct Blob {
|
||||
pub key: String,
|
||||
#[napi(ts_type = "Uint8Array")]
|
||||
pub data: Data,
|
||||
pub mime: String,
|
||||
pub size: i64,
|
||||
pub created_at: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct ListedBlob {
|
||||
pub key: String,
|
||||
pub size: i64,
|
||||
pub mime: String,
|
||||
pub created_at: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct DocStoragePool {
|
||||
pool: SqliteDocStoragePool,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl DocStoragePool {
|
||||
#[napi(constructor)]
|
||||
pub fn new() -> Result<Self> {
|
||||
Ok(Self {
|
||||
pool: SqliteDocStoragePool::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn get(&self, universal_id: String) -> Result<Ref<SqliteDocStorage>> {
|
||||
Ok(self.pool.get(universal_id).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
/// Initialize the database and run migrations.
|
||||
pub async fn connect(&self, universal_id: String, path: String) -> Result<()> {
|
||||
self.pool.connect(universal_id, path).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn disconnect(&self, universal_id: String) -> Result<()> {
|
||||
self.pool.disconnect(universal_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn checkpoint(&self, universal_id: String) -> Result<()> {
|
||||
self.pool.get(universal_id).await?.checkpoint().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn vacuum_into(&self, universal_id: String, path: String) -> Result<()> {
|
||||
self.pool.get(universal_id).await?.vacuum_into(path).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn crawl_doc_data(&self, universal_id: String, doc_id: String) -> Result<indexer::NativeCrawlResult> {
|
||||
let result = self.get(universal_id).await?.crawl_doc_data(&doc_id).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_space_id(&self, universal_id: String, space_id: String) -> Result<()> {
|
||||
self.get(universal_id).await?.set_space_id(space_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn push_update(&self, universal_id: String, doc_id: String, update: Uint8Array) -> Result<NaiveDateTime> {
|
||||
Ok(self.get(universal_id).await?.push_update(doc_id, update).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_doc_snapshot(&self, universal_id: String, doc_id: String) -> Result<Option<DocRecord>> {
|
||||
Ok(self.get(universal_id).await?.get_doc_snapshot(doc_id).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_doc_snapshot(&self, universal_id: String, snapshot: DocRecord) -> Result<bool> {
|
||||
Ok(self.get(universal_id).await?.set_doc_snapshot(snapshot).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_doc_updates(&self, universal_id: String, doc_id: String) -> Result<Vec<DocUpdate>> {
|
||||
Ok(self.get(universal_id).await?.get_doc_updates(doc_id).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn mark_updates_merged(
|
||||
&self,
|
||||
universal_id: String,
|
||||
doc_id: String,
|
||||
updates: Vec<NaiveDateTime>,
|
||||
) -> Result<u32> {
|
||||
Ok(
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.mark_updates_merged(doc_id, updates)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn delete_doc(&self, universal_id: String, doc_id: String) -> Result<()> {
|
||||
self.get(universal_id).await?.delete_doc(doc_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_doc_clocks(&self, universal_id: String, after: Option<NaiveDateTime>) -> Result<Vec<DocClock>> {
|
||||
Ok(self.get(universal_id).await?.get_doc_clocks(after).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_doc_clock(&self, universal_id: String, doc_id: String) -> Result<Option<DocClock>> {
|
||||
Ok(self.get(universal_id).await?.get_doc_clock(doc_id).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<Option<DocIndexedClock>> {
|
||||
Ok(self.get(universal_id).await?.get_doc_indexed_clock(doc_id).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_doc_indexed_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
doc_id: String,
|
||||
indexed_clock: NaiveDateTime,
|
||||
indexer_version: i64,
|
||||
) -> Result<()> {
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.set_doc_indexed_clock(doc_id, indexed_clock, indexer_version)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_doc_indexed_clocks(&self, universal_id: String, clocks: Vec<DocIndexedClock>) -> Result<()> {
|
||||
self.get(universal_id).await?.commit_indexed_clocks(&clocks).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn clear_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<()> {
|
||||
self.get(universal_id).await?.clear_doc_indexed_clock(doc_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi(async_runtime)]
|
||||
pub async fn get_blob(&self, universal_id: String, key: String) -> Result<Option<Blob>> {
|
||||
Ok(self.get(universal_id).await?.get_blob(key).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_blob(&self, universal_id: String, blob: SetBlob) -> Result<()> {
|
||||
self.get(universal_id).await?.set_blob(blob).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn delete_blob(&self, universal_id: String, key: String, permanently: bool) -> Result<()> {
|
||||
self.get(universal_id).await?.delete_blob(key, permanently).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn release_blobs(&self, universal_id: String) -> Result<()> {
|
||||
self.get(universal_id).await?.release_blobs().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn list_blobs(&self, universal_id: String) -> Result<Vec<ListedBlob>> {
|
||||
Ok(self.get(universal_id).await?.list_blobs().await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_peer_remote_clocks(&self, universal_id: String, peer: String) -> Result<Vec<DocClock>> {
|
||||
Ok(self.get(universal_id).await?.get_peer_remote_clocks(peer).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_peer_remote_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
doc_id: String,
|
||||
) -> Result<Option<DocClock>> {
|
||||
Ok(
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.get_peer_remote_clock(peer, doc_id)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_peer_remote_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
doc_id: String,
|
||||
clock: NaiveDateTime,
|
||||
) -> Result<()> {
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.set_peer_remote_clock(peer, doc_id, clock)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_peer_pulled_remote_clocks(&self, universal_id: String, peer: String) -> Result<Vec<DocClock>> {
|
||||
Ok(
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.get_peer_pulled_remote_clocks(peer)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_peer_pulled_remote_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
doc_id: String,
|
||||
) -> Result<Option<DocClock>> {
|
||||
Ok(
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.get_peer_pulled_remote_clock(peer, doc_id)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_peer_pulled_remote_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
doc_id: String,
|
||||
clock: NaiveDateTime,
|
||||
) -> Result<()> {
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.set_peer_pulled_remote_clock(peer, doc_id, clock)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_peer_pushed_clocks(&self, universal_id: String, peer: String) -> Result<Vec<DocClock>> {
|
||||
Ok(self.get(universal_id).await?.get_peer_pushed_clocks(peer).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_peer_pushed_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
doc_id: String,
|
||||
) -> Result<Option<DocClock>> {
|
||||
Ok(
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.get_peer_pushed_clock(peer, doc_id)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_peer_pushed_clock(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
doc_id: String,
|
||||
clock: NaiveDateTime,
|
||||
) -> Result<()> {
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.set_peer_pushed_clock(peer, doc_id, clock)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn clear_clocks(&self, universal_id: String) -> Result<()> {
|
||||
self.get(universal_id).await?.clear_clocks().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_blob_uploaded_at(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
blob_id: String,
|
||||
uploaded_at: Option<NaiveDateTime>,
|
||||
) -> Result<()> {
|
||||
self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.set_blob_uploaded_at(peer, blob_id, uploaded_at)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_blob_uploaded_at(
|
||||
&self,
|
||||
universal_id: String,
|
||||
peer: String,
|
||||
blob_id: String,
|
||||
) -> Result<Option<NaiveDateTime>> {
|
||||
let result = self
|
||||
.get(universal_id)
|
||||
.await?
|
||||
.get_blob_uploaded_at(peer, blob_id)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn index_upsert(&self, id: String, table: String, document: indexer::NativeIndexDocument) -> Result<()> {
|
||||
let storage = self.pool.get(id).await?;
|
||||
storage.index_upsert(&table, document).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn index_flush(&self, id: String) -> Result<()> {
|
||||
let storage = self.pool.get(id).await?;
|
||||
storage.flush_index().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn index_version(&self) -> Result<u32> {
|
||||
Ok(SqliteDocStorage::index_version())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn index_delete(&self, id: String, table: String, doc_id: String) -> Result<()> {
|
||||
let storage = self.pool.get(id).await?;
|
||||
storage.index_delete(&table, &doc_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn index_search(
|
||||
&self,
|
||||
id: String,
|
||||
table: String,
|
||||
query: indexer::NativeIndexQuery,
|
||||
options: indexer::NativeIndexSearchOptions,
|
||||
) -> Result<indexer::NativeIndexSearchResult> {
|
||||
let storage = self.pool.get(id).await?;
|
||||
Ok(storage.index_search(&table, query, options).await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn index_aggregate(
|
||||
&self,
|
||||
id: String,
|
||||
table: String,
|
||||
query: indexer::NativeIndexQuery,
|
||||
field: String,
|
||||
limit: u32,
|
||||
offset: u32,
|
||||
hits: Option<indexer::NativeIndexSearchOptions>,
|
||||
) -> Result<indexer::NativeIndexAggregateResult> {
|
||||
let storage = self.pool.get(id).await?;
|
||||
Ok(
|
||||
storage
|
||||
.index_aggregate(&table, query, &field, limit, offset, hits)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn index_delete_by_query(
|
||||
&self,
|
||||
id: String,
|
||||
table: String,
|
||||
query: indexer::NativeIndexQuery,
|
||||
) -> Result<u32> {
|
||||
let storage = self.pool.get(id).await?;
|
||||
Ok(storage.index_delete_by_query(&table, query).await?)
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct DocStorage {
|
||||
storage: SqliteDocStorage,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl DocStorage {
|
||||
#[napi(constructor, async_runtime)]
|
||||
pub fn new(path: String) -> Self {
|
||||
Self {
|
||||
storage: SqliteDocStorage::new(path),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn validate(&self) -> Result<bool> {
|
||||
Ok(self.storage.validate().await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn validate_import_schema(&self) -> Result<bool> {
|
||||
Ok(self.storage.validate_import_schema().await?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn vacuum_into(&self, path: String) -> Result<()> {
|
||||
self.storage.vacuum_into(path).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_space_id(&self, space_id: String) -> Result<()> {
|
||||
self.storage.connect().await?;
|
||||
self.storage.set_space_id(space_id).await?;
|
||||
self.storage.close().await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(feature = "use-as-lib")))]
|
||||
mod tests {
|
||||
use super::error;
|
||||
|
||||
#[test]
|
||||
fn napi_error_mapping_preserves_reason() {
|
||||
let err: napi::Error = error::Error::InvalidOperation.into();
|
||||
assert_eq!(err.status, napi::Status::GenericFailure);
|
||||
assert!(err.reason.contains("Invalid operation"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn napi_error_mapping_connection_in_progress() {
|
||||
let err: napi::Error = error::Error::ConnectionInProgress.into();
|
||||
assert_eq!(err.status, napi::Status::GenericFailure);
|
||||
assert!(err.reason.contains("Connection in progress"));
|
||||
}
|
||||
}
|
||||
130
packages/frontend/native/nbstore/src/pool.rs
Normal file
130
packages/frontend/native/nbstore/src/pool.rs
Normal file
@@ -0,0 +1,130 @@
|
||||
use core::ops::Deref;
|
||||
use std::{
|
||||
collections::hash_map::{Entry, HashMap},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::{
|
||||
error::{Error, Result},
|
||||
storage::SqliteDocStorage,
|
||||
};
|
||||
|
||||
pub struct Ref<V> {
|
||||
inner: Arc<V>,
|
||||
}
|
||||
|
||||
impl<V> Deref for Ref<V> {
|
||||
type Target = V;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
self.inner.deref()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct SqliteDocStoragePool {
|
||||
inner: RwLock<HashMap<String, StorageState>>,
|
||||
}
|
||||
|
||||
enum StorageState {
|
||||
Connecting(Arc<SqliteDocStorage>),
|
||||
Connected(Arc<SqliteDocStorage>),
|
||||
}
|
||||
|
||||
impl SqliteDocStoragePool {
|
||||
pub async fn get(&self, universal_id: String) -> Result<Ref<SqliteDocStorage>> {
|
||||
let lock = self.inner.read().await;
|
||||
let Some(state) = lock.get(&universal_id) else {
|
||||
return Err(Error::InvalidOperation);
|
||||
};
|
||||
let StorageState::Connected(storage) = state else {
|
||||
return Err(Error::InvalidOperation);
|
||||
};
|
||||
Ok(Ref {
|
||||
inner: Arc::clone(storage),
|
||||
})
|
||||
}
|
||||
|
||||
/// Initialize the database and run migrations.
|
||||
pub async fn connect(&self, universal_id: String, path: String) -> Result<()> {
|
||||
let storage = {
|
||||
let mut lock = self.inner.write().await;
|
||||
match lock.entry(universal_id.clone()) {
|
||||
Entry::Occupied(entry) => match entry.get() {
|
||||
StorageState::Connected(_) => return Ok(()),
|
||||
StorageState::Connecting(_) => return Err(Error::ConnectionInProgress),
|
||||
},
|
||||
Entry::Vacant(entry) => {
|
||||
let storage = Arc::new(SqliteDocStorage::new(path));
|
||||
entry.insert(StorageState::Connecting(Arc::clone(&storage)));
|
||||
storage
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) = storage.connect().await {
|
||||
let mut lock = self.inner.write().await;
|
||||
if matches!(
|
||||
lock.get(&universal_id),
|
||||
Some(StorageState::Connecting(existing)) if Arc::ptr_eq(existing, &storage)
|
||||
) {
|
||||
lock.remove(&universal_id);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut transitioned = false;
|
||||
{
|
||||
let mut lock = self.inner.write().await;
|
||||
if matches!(
|
||||
lock.get(&universal_id),
|
||||
Some(StorageState::Connecting(existing)) if Arc::ptr_eq(existing, &storage)
|
||||
) {
|
||||
lock.insert(universal_id.clone(), StorageState::Connected(Arc::clone(&storage)));
|
||||
transitioned = true;
|
||||
}
|
||||
}
|
||||
|
||||
if !transitioned {
|
||||
let mut lock = self.inner.write().await;
|
||||
if matches!(
|
||||
lock.get(&universal_id),
|
||||
Some(StorageState::Connecting(existing)) if Arc::ptr_eq(existing, &storage)
|
||||
) {
|
||||
lock.remove(&universal_id);
|
||||
}
|
||||
drop(lock);
|
||||
storage.close().await;
|
||||
return Err(Error::InvalidOperation);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn disconnect(&self, universal_id: String) -> Result<()> {
|
||||
let storage = {
|
||||
let mut lock = self.inner.write().await;
|
||||
match lock.get(&universal_id) {
|
||||
None => return Ok(()),
|
||||
Some(StorageState::Connecting(_)) => return Err(Error::ConnectionInProgress),
|
||||
Some(StorageState::Connected(storage)) => {
|
||||
// Prevent shutting down the shared storage while requests still hold
|
||||
// refs.
|
||||
if Arc::strong_count(storage) > 1 {
|
||||
return Err(Error::InvalidOperation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(StorageState::Connected(storage)) = lock.remove(&universal_id) else {
|
||||
return Err(Error::InvalidOperation);
|
||||
};
|
||||
storage
|
||||
};
|
||||
|
||||
storage.close().await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
398
packages/frontend/native/nbstore/src/storage.rs
Normal file
398
packages/frontend/native/nbstore/src/storage.rs
Normal file
@@ -0,0 +1,398 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use affine_schema::{
|
||||
get_migrator,
|
||||
import_validation::{V2_IMPORT_SCHEMA_RULES, validate_import_schema, validate_required_schema},
|
||||
};
|
||||
use sqlx::{
|
||||
Pool, Row,
|
||||
migrate::{MigrateDatabase, Migration, Migrator},
|
||||
sqlite::{Sqlite, SqliteConnectOptions, SqlitePoolOptions},
|
||||
};
|
||||
|
||||
use super::{error::Result, indexer::IndexManager};
|
||||
|
||||
pub struct SqliteDocStorage {
|
||||
pub pool: Pool<Sqlite>,
|
||||
path: String,
|
||||
pub indexes: Arc<IndexManager>,
|
||||
}
|
||||
|
||||
impl SqliteDocStorage {
|
||||
pub fn new(path: String) -> Self {
|
||||
let sqlite_options = SqliteConnectOptions::new().filename(&path).foreign_keys(false);
|
||||
|
||||
let mut pool_options = SqlitePoolOptions::new();
|
||||
|
||||
let indexes = Arc::new(IndexManager::new());
|
||||
|
||||
if path == ":memory:" {
|
||||
pool_options = pool_options
|
||||
.min_connections(1)
|
||||
.max_connections(1)
|
||||
.idle_timeout(None)
|
||||
.max_lifetime(None);
|
||||
|
||||
Self {
|
||||
pool: pool_options.connect_lazy_with(sqlite_options),
|
||||
path,
|
||||
indexes,
|
||||
}
|
||||
} else {
|
||||
Self {
|
||||
pool: pool_options
|
||||
.max_connections(4)
|
||||
.connect_lazy_with(sqlite_options.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)),
|
||||
path,
|
||||
indexes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn validate(&self) -> Result<bool> {
|
||||
if self.path == ":memory:" {
|
||||
return Ok(validate_required_schema(&self.pool, &V2_IMPORT_SCHEMA_RULES).await?);
|
||||
}
|
||||
|
||||
let Ok(pool) = self.open_readonly_pool().await else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
Ok(validate_required_schema(&pool, &V2_IMPORT_SCHEMA_RULES).await?)
|
||||
}
|
||||
|
||||
pub async fn validate_import_schema(&self) -> Result<bool> {
|
||||
if self.path == ":memory:" {
|
||||
return Ok(validate_import_schema(&self.pool, &V2_IMPORT_SCHEMA_RULES).await?);
|
||||
}
|
||||
|
||||
let Ok(pool) = self.open_readonly_pool().await else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
Ok(validate_import_schema(&pool, &V2_IMPORT_SCHEMA_RULES).await?)
|
||||
}
|
||||
|
||||
pub async fn connect(&self) -> Result<()> {
|
||||
if !Sqlite::database_exists(&self.path).await? {
|
||||
Sqlite::create_database(&self.path).await?;
|
||||
};
|
||||
|
||||
self.migrate().await?;
|
||||
self.init_index().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn migrate(&self) -> Result<()> {
|
||||
let migrator = get_migrator();
|
||||
if let Err(err) = migrator.run(&self.pool).await {
|
||||
// Compatibility: migration 3 (`add_idx_snapshots`) had a whitespace-only
|
||||
// SQL change (trailing space) between releases, which causes sqlx
|
||||
// to reject existing DBs with: `VersionMismatch(3)`. It's safe to
|
||||
// fix by updating the stored checksum.
|
||||
if matches!(err, sqlx::migrate::MigrateError::VersionMismatch(3))
|
||||
&& self.try_repair_migration_3_checksum(&migrator).await?
|
||||
{
|
||||
migrator.run(&self.pool).await?;
|
||||
} else {
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn try_repair_migration_3_checksum(&self, migrator: &Migrator) -> Result<bool> {
|
||||
let Some(migration) = migrator.iter().find(|m| m.version == 3) else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
// We're only prepared to repair the known `add_idx_snapshots`
|
||||
// whitespace-only mismatch.
|
||||
if migration.description.as_ref() != "add_idx_snapshots" {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let row = sqlx::query("SELECT description, checksum FROM _sqlx_migrations WHERE version = 3")
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
let Some(row) = row else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
let applied_description: String = row.try_get("description")?;
|
||||
if applied_description != migration.description.as_ref() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let applied_checksum: Vec<u8> = row.try_get("checksum")?;
|
||||
let expected_checksum = migration.checksum.as_ref();
|
||||
|
||||
// sqlx computes the checksum as SHA-384 of the raw SQL bytes. The legacy
|
||||
// variant had an extra trailing space at the end of the SQL string (after
|
||||
// the final newline).
|
||||
let legacy_sql = format!("{} ", migration.sql);
|
||||
let legacy_migration = Migration::new(
|
||||
migration.version,
|
||||
migration.description.clone(),
|
||||
migration.migration_type,
|
||||
std::borrow::Cow::Owned(legacy_sql),
|
||||
migration.no_tx,
|
||||
);
|
||||
|
||||
if applied_checksum.as_slice() != legacy_migration.checksum.as_ref() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
sqlx::query("UPDATE _sqlx_migrations SET checksum = ? WHERE version = 3")
|
||||
.bind(expected_checksum)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn close(&self) {
|
||||
self.pool.close().await
|
||||
}
|
||||
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.pool.is_closed()
|
||||
}
|
||||
|
||||
///
|
||||
/// Flush the WAL file to the database file.
|
||||
/// See https://www.sqlite.org/pragma.html#pragma_wal_checkpoint:~:text=PRAGMA%20schema.wal_checkpoint%3B
|
||||
pub async fn checkpoint(&self) -> Result<()> {
|
||||
sqlx::query("PRAGMA wal_checkpoint(FULL);").execute(&self.pool).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn vacuum_into(&self, path: String) -> Result<()> {
|
||||
if self.path == ":memory:" {
|
||||
sqlx::query("VACUUM INTO ?;").bind(path).execute(&self.pool).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let pool = self.open_readonly_pool().await?;
|
||||
sqlx::query("VACUUM INTO ?;").bind(path).execute(&pool).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn open_readonly_pool(&self) -> Result<Pool<Sqlite>> {
|
||||
let sqlite_options = SqliteConnectOptions::new()
|
||||
.filename(&self.path)
|
||||
.foreign_keys(false)
|
||||
.read_only(true);
|
||||
|
||||
Ok(
|
||||
SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(sqlite_options)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{borrow::Cow, fs, path::Path};
|
||||
|
||||
use affine_schema::get_migrator;
|
||||
use sqlx::migrate::{Migration, Migrator};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn get_storage() -> SqliteDocStorage {
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_tables() {
|
||||
let storage = get_storage().await;
|
||||
|
||||
sqlx::query("INSERT INTO meta (space_id) VALUES ($1);")
|
||||
.bind("test")
|
||||
.execute(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let record = sqlx::query!("SELECT space_id FROM meta;")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(record.space_id, "test");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_db() {
|
||||
let storage = get_storage().await;
|
||||
assert!(storage.validate().await.unwrap());
|
||||
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
assert!(!storage.validate().await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_repairs_whitespace_only_migration_checksum_mismatch() {
|
||||
// Simulate a DB migrated with an older `add_idx_snapshots` SQL that had a
|
||||
// trailing space.
|
||||
let storage = SqliteDocStorage::new(":memory:".to_string());
|
||||
|
||||
let new_migrator = get_migrator();
|
||||
let mut migrations = new_migrator.migrations.to_vec();
|
||||
assert!(migrations.len() >= 3);
|
||||
|
||||
let mig3 = migrations[2].clone();
|
||||
assert_eq!(mig3.version, 3);
|
||||
assert_eq!(mig3.description.as_ref(), "add_idx_snapshots");
|
||||
|
||||
let legacy_sql = format!("{} ", mig3.sql);
|
||||
migrations[2] = Migration::new(
|
||||
mig3.version,
|
||||
mig3.description.clone(),
|
||||
mig3.migration_type,
|
||||
Cow::Owned(legacy_sql),
|
||||
mig3.no_tx,
|
||||
);
|
||||
|
||||
// The legacy DB didn't have newer migrations.
|
||||
migrations.truncate(3);
|
||||
let legacy_migrator = Migrator {
|
||||
migrations: Cow::Owned(migrations),
|
||||
..Migrator::DEFAULT
|
||||
};
|
||||
|
||||
legacy_migrator.run(&storage.pool).await.unwrap();
|
||||
|
||||
// Now connecting with the current code should auto-repair the checksum and
|
||||
// succeed.
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
let expected_checksum = get_migrator()
|
||||
.iter()
|
||||
.find(|m| m.version == 3)
|
||||
.unwrap()
|
||||
.checksum
|
||||
.as_ref()
|
||||
.to_vec();
|
||||
|
||||
let row = sqlx::query("SELECT checksum FROM _sqlx_migrations WHERE version = 3")
|
||||
.fetch_one(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let checksum: Vec<u8> = row.get("checksum");
|
||||
|
||||
assert_eq!(checksum, expected_checksum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vacuum_into_exports_a_compacted_database() {
|
||||
let base = std::env::temp_dir().join(format!("nbstore-vacuum-{}", Uuid::new_v4()));
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
|
||||
let source = base.join("storage.db");
|
||||
let export = base.join("backup.affine");
|
||||
|
||||
let storage = SqliteDocStorage::new(path_string(&source));
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
storage
|
||||
.set_blob(crate::SetBlob {
|
||||
key: "large-blob".to_string(),
|
||||
data: Into::<crate::Data>::into(vec![7; 1024 * 1024]),
|
||||
mime: "application/octet-stream".to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
storage.delete_blob("large-blob".to_string(), true).await.unwrap();
|
||||
storage.checkpoint().await.unwrap();
|
||||
|
||||
let source_len = fs::metadata(&source).unwrap().len();
|
||||
assert!(source_len > 0);
|
||||
|
||||
storage.vacuum_into(path_string(&export)).await.unwrap();
|
||||
|
||||
let export_len = fs::metadata(&export).unwrap().len();
|
||||
assert!(export_len < source_len);
|
||||
|
||||
let exported = SqliteDocStorage::new(path_string(&export));
|
||||
exported.connect().await.unwrap();
|
||||
assert!(exported.list_blobs().await.unwrap().is_empty());
|
||||
exported.close().await;
|
||||
storage.close().await;
|
||||
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_import_schema_rejects_unexpected_schema_objects() {
|
||||
let base = std::env::temp_dir().join(format!("nbstore-schema-{}", Uuid::new_v4()));
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
|
||||
let source = base.join("storage.db");
|
||||
fs::File::create(&source).unwrap();
|
||||
let storage = SqliteDocStorage::new(path_string(&source));
|
||||
storage.connect().await.unwrap();
|
||||
|
||||
sqlx::query("CREATE VIEW rogue_view AS SELECT space_id FROM meta")
|
||||
.execute(&storage.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!storage.validate_import_schema().await.unwrap());
|
||||
|
||||
storage.close().await;
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_import_schema_accepts_initial_v2_schema() {
|
||||
let base = std::env::temp_dir().join(format!("nbstore-v2-schema-{}", Uuid::new_v4()));
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
|
||||
let source = base.join("storage.db");
|
||||
let source_path = path_string(&source);
|
||||
let setup_pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(
|
||||
SqliteConnectOptions::new()
|
||||
.filename(&source_path)
|
||||
.create_if_missing(true)
|
||||
.foreign_keys(false),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut migrations = get_migrator().migrations.to_vec();
|
||||
migrations.truncate(1);
|
||||
let migrator = Migrator {
|
||||
migrations: Cow::Owned(migrations),
|
||||
..Migrator::DEFAULT
|
||||
};
|
||||
|
||||
migrator.run(&setup_pool).await.unwrap();
|
||||
setup_pool.close().await;
|
||||
|
||||
let storage = SqliteDocStorage::new(source_path);
|
||||
|
||||
assert!(storage.validate().await.unwrap());
|
||||
assert!(storage.validate_import_schema().await.unwrap());
|
||||
|
||||
storage.close().await;
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
|
||||
fn path_string(path: &Path) -> String {
|
||||
path.to_string_lossy().into_owned()
|
||||
}
|
||||
}
|
||||
47
packages/frontend/native/package.json
Normal file
47
packages/frontend/native/package.json
Normal file
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"name": "@affine/native",
|
||||
"private": true,
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
"napi": {
|
||||
"binaryName": "affine",
|
||||
"targets": [
|
||||
"x86_64-apple-darwin",
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"x86_64-pc-windows-msvc",
|
||||
"aarch64-apple-darwin",
|
||||
"aarch64-unknown-linux-gnu",
|
||||
"aarch64-pc-windows-msvc"
|
||||
],
|
||||
"constEnum": false
|
||||
},
|
||||
"license": "MIT",
|
||||
"ava": {
|
||||
"extensions": {
|
||||
"mts": "module"
|
||||
},
|
||||
"files": [
|
||||
"__tests__/*.spec.mts"
|
||||
]
|
||||
},
|
||||
"devDependencies": {
|
||||
"@napi-rs/cli": "3.5.0",
|
||||
"@napi-rs/whisper": "^0.0.4",
|
||||
"@types/node": "^22.0.0",
|
||||
"ava": "^7.0.0",
|
||||
"rxjs": "^7.8.2",
|
||||
"typescript": "npm:@typescript/typescript6@6.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"scripts": {
|
||||
"artifacts": "napi artifacts",
|
||||
"build": "napi build -p affine_native --platform --release --strip",
|
||||
"build:debug": "napi build -p affine_native --platform",
|
||||
"universal": "napi universal",
|
||||
"test": "ava --no-worker-threads --concurrency=2",
|
||||
"version": "napi version"
|
||||
},
|
||||
"version": "0.27.5"
|
||||
}
|
||||
8
packages/frontend/native/schema/Cargo.toml
Normal file
8
packages/frontend/native/schema/Cargo.toml
Normal file
@@ -0,0 +1,8 @@
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "affine_schema"
|
||||
publish = false
|
||||
version = "0.0.0"
|
||||
|
||||
[dependencies]
|
||||
sqlx = { workspace = true, default-features = false, features = ["migrate"] }
|
||||
1
packages/frontend/native/schema/README.md
Normal file
1
packages/frontend/native/schema/README.md
Normal file
@@ -0,0 +1 @@
|
||||
A temporary crate to share the schema between AFFiNE native and `build.rs` in the AFFiNE native.
|
||||
261
packages/frontend/native/schema/src/import_validation.rs
Normal file
261
packages/frontend/native/schema/src/import_validation.rs
Normal file
@@ -0,0 +1,261 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use sqlx::{Pool, Row, sqlite::Sqlite};
|
||||
|
||||
pub struct ImportSchemaRules {
|
||||
pub tables: &'static [ImportTableRule],
|
||||
pub indexes: &'static [ImportIndexRule],
|
||||
}
|
||||
|
||||
pub struct ImportTableRule {
|
||||
pub name: &'static str,
|
||||
pub columns: &'static [&'static str],
|
||||
pub enforce_columns: bool,
|
||||
pub required: bool,
|
||||
}
|
||||
|
||||
pub struct ImportIndexRule {
|
||||
pub name: &'static str,
|
||||
pub table: &'static str,
|
||||
pub columns: &'static [&'static str],
|
||||
pub required: bool,
|
||||
}
|
||||
|
||||
pub const V2_IMPORT_SCHEMA_RULES: ImportSchemaRules = ImportSchemaRules {
|
||||
tables: &[
|
||||
ImportTableRule {
|
||||
name: "meta",
|
||||
columns: &["space_id"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "snapshots",
|
||||
columns: &["doc_id", "data", "created_at", "updated_at"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "updates",
|
||||
columns: &["doc_id", "created_at", "data"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "clocks",
|
||||
columns: &["doc_id", "timestamp"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "blobs",
|
||||
columns: &["key", "data", "mime", "size", "created_at", "deleted_at"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "peer_clocks",
|
||||
columns: &["peer", "doc_id", "remote_clock", "pulled_remote_clock", "pushed_clock"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "peer_blob_sync",
|
||||
columns: &["peer", "blob_id", "uploaded_at"],
|
||||
enforce_columns: true,
|
||||
required: false,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "idx_snapshots",
|
||||
columns: &["index_name", "data", "created_at"],
|
||||
enforce_columns: true,
|
||||
required: false,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "indexer_sync",
|
||||
columns: &["doc_id", "indexed_clock", "indexer_version"],
|
||||
enforce_columns: true,
|
||||
required: false,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "_sqlx_migrations",
|
||||
columns: &[],
|
||||
enforce_columns: false,
|
||||
required: false,
|
||||
},
|
||||
],
|
||||
indexes: &[
|
||||
ImportIndexRule {
|
||||
name: "peer_clocks_doc_id",
|
||||
table: "peer_clocks",
|
||||
columns: &["doc_id"],
|
||||
required: true,
|
||||
},
|
||||
ImportIndexRule {
|
||||
name: "peer_blob_sync_peer",
|
||||
table: "peer_blob_sync",
|
||||
columns: &["peer"],
|
||||
required: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
pub const V1_IMPORT_SCHEMA_RULES: ImportSchemaRules = ImportSchemaRules {
|
||||
tables: &[
|
||||
ImportTableRule {
|
||||
name: "updates",
|
||||
columns: &["id", "timestamp", "data", "doc_id"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "blobs",
|
||||
columns: &["key", "data", "timestamp"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "version_info",
|
||||
columns: &["version", "timestamp"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "server_clock",
|
||||
columns: &["key", "data", "timestamp"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
ImportTableRule {
|
||||
name: "sync_metadata",
|
||||
columns: &["key", "data", "timestamp"],
|
||||
enforce_columns: true,
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
indexes: &[ImportIndexRule {
|
||||
name: "idx_doc_id",
|
||||
table: "updates",
|
||||
columns: &["doc_id"],
|
||||
required: false,
|
||||
}],
|
||||
};
|
||||
|
||||
pub async fn validate_import_schema(pool: &Pool<Sqlite>, rules: &ImportSchemaRules) -> sqlx::Result<bool> {
|
||||
validate_schema(pool, rules, ValidationMode::Strict).await
|
||||
}
|
||||
|
||||
pub async fn validate_required_schema(pool: &Pool<Sqlite>, rules: &ImportSchemaRules) -> sqlx::Result<bool> {
|
||||
validate_schema(pool, rules, ValidationMode::RequiredOnly).await
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ValidationMode {
|
||||
Strict,
|
||||
RequiredOnly,
|
||||
}
|
||||
|
||||
async fn validate_schema(pool: &Pool<Sqlite>, rules: &ImportSchemaRules, mode: ValidationMode) -> sqlx::Result<bool> {
|
||||
let query = match mode {
|
||||
ValidationMode::Strict => {
|
||||
"SELECT type, name, tbl_name FROM sqlite_schema WHERE type IN ('table', 'index', 'view', 'trigger')"
|
||||
}
|
||||
ValidationMode::RequiredOnly => "SELECT type, name, tbl_name FROM sqlite_schema WHERE type IN ('table', 'index')",
|
||||
};
|
||||
let rows = sqlx::query(query).fetch_all(pool).await?;
|
||||
|
||||
let mut seen_tables = BTreeSet::new();
|
||||
let mut seen_indexes = BTreeSet::new();
|
||||
|
||||
for row in rows {
|
||||
let object_type: String = row.try_get("type")?;
|
||||
let name: String = row.try_get("name")?;
|
||||
let table_name: String = row.try_get("tbl_name")?;
|
||||
|
||||
if name.starts_with("sqlite_") {
|
||||
continue;
|
||||
}
|
||||
|
||||
match object_type.as_str() {
|
||||
"table" => {
|
||||
let Some(rule) = rules.tables.iter().find(|rule| rule.name == name) else {
|
||||
if matches!(mode, ValidationMode::Strict) {
|
||||
return Ok(false);
|
||||
}
|
||||
continue;
|
||||
};
|
||||
if rule.enforce_columns && !table_columns_match(pool, rule).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
seen_tables.insert(name);
|
||||
}
|
||||
"index" => {
|
||||
let Some(rule) = rules
|
||||
.indexes
|
||||
.iter()
|
||||
.find(|rule| rule.name == name && rule.table == table_name)
|
||||
else {
|
||||
if matches!(mode, ValidationMode::Strict) {
|
||||
return Ok(false);
|
||||
}
|
||||
continue;
|
||||
};
|
||||
if !index_columns_match(pool, rule).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
seen_indexes.insert(name);
|
||||
}
|
||||
"view" | "trigger" => return Ok(false),
|
||||
_ => return Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
if rules
|
||||
.tables
|
||||
.iter()
|
||||
.filter(|rule| rule.required)
|
||||
.any(|rule| !seen_tables.contains(rule.name))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if rules
|
||||
.indexes
|
||||
.iter()
|
||||
.filter(|rule| rule.required)
|
||||
.any(|rule| !seen_indexes.contains(rule.name))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn table_columns_match(pool: &Pool<Sqlite>, rule: &ImportTableRule) -> sqlx::Result<bool> {
|
||||
let pragma = format!("PRAGMA table_info(\"{}\")", rule.name);
|
||||
let rows = sqlx::query(&pragma).fetch_all(pool).await?;
|
||||
let columns = rows
|
||||
.into_iter()
|
||||
.map(|row| row.try_get::<String, _>("name"))
|
||||
.collect::<std::result::Result<BTreeSet<_>, _>>()?;
|
||||
|
||||
Ok(columns == rule.columns.iter().map(|column| (*column).to_string()).collect())
|
||||
}
|
||||
|
||||
async fn index_columns_match(pool: &Pool<Sqlite>, rule: &ImportIndexRule) -> sqlx::Result<bool> {
|
||||
let pragma = format!("PRAGMA index_info(\"{}\")", rule.name);
|
||||
let rows = sqlx::query(&pragma).fetch_all(pool).await?;
|
||||
let columns = rows
|
||||
.into_iter()
|
||||
.map(|row| row.try_get::<String, _>("name"))
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok(
|
||||
columns
|
||||
== rule
|
||||
.columns
|
||||
.iter()
|
||||
.map(|column| (*column).to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
135
packages/frontend/native/schema/src/lib.rs
Normal file
135
packages/frontend/native/schema/src/lib.rs
Normal file
@@ -0,0 +1,135 @@
|
||||
use std::borrow::Cow;
|
||||
|
||||
use sqlx::migrate::{Migration, MigrationType, Migrator};
|
||||
|
||||
pub mod import_validation;
|
||||
pub mod v1;
|
||||
|
||||
type SimpleMigration = (
|
||||
/* name */ &'static str,
|
||||
/* up */ &'static str,
|
||||
/* down */ Option<&'static str>,
|
||||
);
|
||||
|
||||
// ORDER MATTERS
|
||||
const MIGRATIONS: &[SimpleMigration] = &[
|
||||
// v2 db init
|
||||
(
|
||||
"init_v2",
|
||||
r#"
|
||||
CREATE TABLE "meta" (
|
||||
space_id VARCHAR PRIMARY KEY NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE "snapshots" (
|
||||
doc_id VARCHAR PRIMARY KEY NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE "updates" (
|
||||
doc_id VARCHAR NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
PRIMARY KEY (doc_id, created_at)
|
||||
);
|
||||
|
||||
CREATE TABLE "clocks" (
|
||||
doc_id VARCHAR PRIMARY KEY NOT NULL,
|
||||
timestamp TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE "blobs" (
|
||||
key VARCHAR PRIMARY KEY NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
mime VARCHAR NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
deleted_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE "peer_clocks" (
|
||||
peer VARCHAR NOT NULL,
|
||||
doc_id VARCHAR NOT NULL,
|
||||
remote_clock TIMESTAMP NOT NULL DEFAULT 0,
|
||||
pulled_remote_clock TIMESTAMP NOT NULL DEFAULT 0,
|
||||
pushed_clock TIMESTAMP NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (peer, doc_id)
|
||||
);
|
||||
CREATE INDEX peer_clocks_doc_id ON peer_clocks (doc_id);
|
||||
"#,
|
||||
None,
|
||||
),
|
||||
// add blob_sync table
|
||||
(
|
||||
"add_blob_sync",
|
||||
r#"
|
||||
CREATE TABLE "peer_blob_sync" (
|
||||
peer VARCHAR NOT NULL,
|
||||
blob_id VARCHAR NOT NULL,
|
||||
uploaded_at TIMESTAMP,
|
||||
PRIMARY KEY (peer, blob_id)
|
||||
);
|
||||
CREATE INDEX peer_blob_sync_peer ON peer_blob_sync (peer);
|
||||
"#,
|
||||
None,
|
||||
),
|
||||
// add idx snapshots
|
||||
(
|
||||
"add_idx_snapshots",
|
||||
r#"
|
||||
CREATE TABLE idx_snapshots (
|
||||
index_name TEXT PRIMARY KEY NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
);
|
||||
"#,
|
||||
None,
|
||||
),
|
||||
// add indexer sync table
|
||||
(
|
||||
"add_indexer_sync",
|
||||
r#"
|
||||
CREATE TABLE "indexer_sync" (
|
||||
doc_id VARCHAR PRIMARY KEY NOT NULL,
|
||||
indexed_clock TIMESTAMP NOT NULL DEFAULT 0,
|
||||
indexer_version INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
"#,
|
||||
None,
|
||||
),
|
||||
];
|
||||
|
||||
pub fn get_migrator() -> Migrator {
|
||||
let mut migrations = vec![];
|
||||
|
||||
MIGRATIONS.iter().for_each(|&(name, up, down)| {
|
||||
migrations.push(Migration::new(
|
||||
migrations.len() as i64 + 1,
|
||||
Cow::from(name),
|
||||
if down.is_some() {
|
||||
MigrationType::ReversibleUp
|
||||
} else {
|
||||
MigrationType::Simple
|
||||
},
|
||||
Cow::from(up),
|
||||
false,
|
||||
));
|
||||
|
||||
if let Some(down) = down {
|
||||
migrations.push(Migration::new(
|
||||
migrations.len() as i64 + 1,
|
||||
Cow::from(name),
|
||||
MigrationType::ReversibleDown,
|
||||
Cow::from(down),
|
||||
false,
|
||||
));
|
||||
}
|
||||
});
|
||||
|
||||
Migrator {
|
||||
migrations: Cow::Owned(migrations),
|
||||
..Migrator::DEFAULT
|
||||
}
|
||||
}
|
||||
26
packages/frontend/native/schema/src/v1.rs
Normal file
26
packages/frontend/native/schema/src/v1.rs
Normal file
@@ -0,0 +1,26 @@
|
||||
pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
data BLOB NOT NULL,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
doc_id TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS "blobs" (
|
||||
key TEXT PRIMARY KEY NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS "version_info" (
|
||||
version NUMBER NOT NULL,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS "server_clock" (
|
||||
key TEXT PRIMARY KEY NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS "sync_metadata" (
|
||||
key TEXT PRIMARY KEY NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
)
|
||||
"#;
|
||||
36
packages/frontend/native/sqlite_v1/Cargo.toml
Normal file
36
packages/frontend/native/sqlite_v1/Cargo.toml
Normal file
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "affine_sqlite_v1"
|
||||
publish = false
|
||||
version = "0.0.0"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[dependencies]
|
||||
affine_schema = { path = "../schema" }
|
||||
anyhow = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
napi = { workspace = true }
|
||||
napi-derive = { workspace = true }
|
||||
sqlx = { workspace = true, default-features = false, features = [
|
||||
"chrono",
|
||||
"macros",
|
||||
"migrate",
|
||||
"runtime-tokio",
|
||||
"sqlite",
|
||||
] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
|
||||
[build-dependencies]
|
||||
affine_schema = { path = "../schema" }
|
||||
dotenvy = { workspace = true }
|
||||
napi-build = { workspace = true }
|
||||
sqlx = { workspace = true, default-features = false, features = [
|
||||
"chrono",
|
||||
"macros",
|
||||
"migrate",
|
||||
"runtime-tokio",
|
||||
"sqlite",
|
||||
] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
33
packages/frontend/native/sqlite_v1/build.rs
Normal file
33
packages/frontend/native/sqlite_v1/build.rs
Normal file
@@ -0,0 +1,33 @@
|
||||
use std::fs;
|
||||
|
||||
use sqlx::sqlite::SqliteConnectOptions;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), std::io::Error> {
|
||||
napi_build::setup();
|
||||
|
||||
// always start with a fresh database to have latest db schema
|
||||
let cwd = std::env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
let db_path = format!("{cwd}/affine.db");
|
||||
|
||||
if fs::metadata(&db_path).is_ok() {
|
||||
fs::remove_file(&db_path)?;
|
||||
}
|
||||
|
||||
let options = SqliteConnectOptions::new()
|
||||
.filename(&db_path)
|
||||
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
|
||||
.locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
|
||||
.create_if_missing(true);
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
sqlx::query(affine_schema::v1::SCHEMA).execute(&pool).await.unwrap();
|
||||
|
||||
println!("cargo::rustc-env=DATABASE_URL=sqlite://{db_path}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
602
packages/frontend/native/sqlite_v1/src/lib.rs
Normal file
602
packages/frontend/native/sqlite_v1/src/lib.rs
Normal file
@@ -0,0 +1,602 @@
|
||||
use affine_schema::import_validation::{V1_IMPORT_SCHEMA_RULES, validate_import_schema};
|
||||
use chrono::NaiveDateTime;
|
||||
use napi::bindgen_prelude::{Buffer, Uint8Array};
|
||||
use napi_derive::napi;
|
||||
use sqlx::{
|
||||
Pool, Row,
|
||||
migrate::MigrateDatabase,
|
||||
sqlite::{Sqlite, SqliteConnectOptions, SqlitePoolOptions},
|
||||
};
|
||||
|
||||
// latest version
|
||||
const LATEST_VERSION: i32 = 4;
|
||||
|
||||
#[napi(object)]
|
||||
pub struct BlobRow {
|
||||
pub key: String,
|
||||
pub data: Buffer,
|
||||
pub timestamp: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct UpdateRow {
|
||||
pub id: i64,
|
||||
pub timestamp: NaiveDateTime,
|
||||
pub data: Buffer,
|
||||
pub doc_id: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct DocTimestampRow {
|
||||
pub doc_id: Option<String>,
|
||||
pub timestamp: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct InsertRow {
|
||||
pub doc_id: Option<String>,
|
||||
pub data: Uint8Array,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct SqliteConnection {
|
||||
pool: Pool<Sqlite>,
|
||||
path: String,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub enum ValidationResult {
|
||||
MissingTables,
|
||||
MissingDocIdColumn,
|
||||
MissingVersionColumn,
|
||||
GeneralError,
|
||||
Valid,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl SqliteConnection {
|
||||
#[napi(constructor, async_runtime)]
|
||||
pub fn new(path: String) -> napi::Result<Self> {
|
||||
let sqlite_options = SqliteConnectOptions::new()
|
||||
.filename(&path)
|
||||
.foreign_keys(false)
|
||||
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal);
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(4)
|
||||
.connect_lazy_with(sqlite_options);
|
||||
Ok(Self { pool, path })
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn connect(&self) -> napi::Result<()> {
|
||||
if !Sqlite::database_exists(&self.path).await.unwrap_or(false) {
|
||||
Sqlite::create_database(&self.path).await.map_err(anyhow::Error::from)?;
|
||||
};
|
||||
let mut connection = self.pool.acquire().await.map_err(anyhow::Error::from)?;
|
||||
sqlx::query(affine_schema::v1::SCHEMA)
|
||||
.execute(connection.as_mut())
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
self.migrate_add_doc_id().await?;
|
||||
self.migrate_add_doc_id_index().await?;
|
||||
connection.detach();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn add_blob(&self, key: String, blob: Uint8Array) -> napi::Result<()> {
|
||||
let blob = blob.as_ref();
|
||||
sqlx::query_as!(
|
||||
BlobRow,
|
||||
"INSERT INTO blobs (key, data) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET data = excluded.data",
|
||||
key,
|
||||
blob,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
|
||||
sqlx::query_as!(BlobRow, "SELECT key, data, timestamp FROM blobs WHERE key = ?", key)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn delete_blob(&self, key: String) -> napi::Result<()> {
|
||||
sqlx::query!("DELETE FROM blobs WHERE key = ?", key)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_blob_keys(&self) -> napi::Result<Vec<String>> {
|
||||
let keys = sqlx::query!("SELECT key FROM blobs")
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map(|rows| rows.into_iter().map(|row| row.key).collect())
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_updates(&self, doc_id: Option<String>) -> napi::Result<Vec<UpdateRow>> {
|
||||
let updates = match doc_id {
|
||||
Some(doc_id) => sqlx::query_as!(
|
||||
UpdateRow,
|
||||
"SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id = ?",
|
||||
doc_id
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?,
|
||||
None => sqlx::query_as!(
|
||||
UpdateRow,
|
||||
"SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id is NULL",
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?,
|
||||
};
|
||||
Ok(updates)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_doc_timestamps(&self) -> napi::Result<Vec<DocTimestampRow>> {
|
||||
// get the greatest timestamp of each doc_id
|
||||
let updates = sqlx::query_as!(
|
||||
DocTimestampRow,
|
||||
"SELECT doc_id, MAX(timestamp) as timestamp FROM updates GROUP BY doc_id"
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
|
||||
Ok(updates)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn delete_updates(&self, doc_id: Option<String>) -> napi::Result<()> {
|
||||
match doc_id {
|
||||
Some(doc_id) => {
|
||||
sqlx::query!("DELETE FROM updates WHERE doc_id = ?", doc_id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
}
|
||||
None => {
|
||||
sqlx::query!("DELETE FROM updates WHERE doc_id is NULL")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_updates_count(&self, doc_id: Option<String>) -> napi::Result<i64> {
|
||||
let count = match doc_id {
|
||||
Some(doc_id) => {
|
||||
sqlx::query!("SELECT COUNT(*) as count FROM updates WHERE doc_id = ?", doc_id)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?
|
||||
.count
|
||||
}
|
||||
None => {
|
||||
sqlx::query!("SELECT COUNT(*) as count FROM updates WHERE doc_id is NULL")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?
|
||||
.count
|
||||
}
|
||||
};
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_all_updates(&self) -> napi::Result<Vec<UpdateRow>> {
|
||||
let updates = sqlx::query_as!(UpdateRow, "SELECT id, timestamp, data, doc_id FROM updates")
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(updates)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn insert_updates(&self, updates: Vec<InsertRow>) -> napi::Result<()> {
|
||||
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
|
||||
for InsertRow { data, doc_id } in updates {
|
||||
let update = data.as_ref();
|
||||
sqlx::query_as!(
|
||||
UpdateRow,
|
||||
"INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
|
||||
update,
|
||||
doc_id
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
}
|
||||
transaction.commit().await.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn replace_updates(&self, doc_id: Option<String>, updates: Vec<InsertRow>) -> napi::Result<()> {
|
||||
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
|
||||
|
||||
match doc_id {
|
||||
Some(doc_id) => sqlx::query!("DELETE FROM updates where doc_id = ?", doc_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?,
|
||||
None => sqlx::query!("DELETE FROM updates where doc_id is NULL",)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?,
|
||||
};
|
||||
|
||||
for InsertRow { data, doc_id } in updates {
|
||||
let update = data.as_ref();
|
||||
sqlx::query_as!(
|
||||
UpdateRow,
|
||||
"INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
|
||||
update,
|
||||
doc_id
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
}
|
||||
transaction.commit().await.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_server_clock(&self, key: String) -> Option<BlobRow> {
|
||||
sqlx::query_as!(
|
||||
BlobRow,
|
||||
"SELECT key, data, timestamp FROM server_clock WHERE key = ?",
|
||||
key
|
||||
)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_server_clock(&self, key: String, data: Uint8Array) -> napi::Result<()> {
|
||||
let data = data.as_ref();
|
||||
sqlx::query!(
|
||||
"INSERT INTO server_clock (key, data) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET data = excluded.data",
|
||||
key,
|
||||
data,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_server_clock_keys(&self) -> napi::Result<Vec<String>> {
|
||||
let keys = sqlx::query!("SELECT key FROM server_clock")
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map(|rows| rows.into_iter().map(|row| row.key).collect())
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn clear_server_clock(&self) -> napi::Result<()> {
|
||||
sqlx::query!("DELETE FROM server_clock")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn del_server_clock(&self, key: String) -> napi::Result<()> {
|
||||
sqlx::query!("DELETE FROM server_clock WHERE key = ?", key)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_sync_metadata(&self, key: String) -> Option<BlobRow> {
|
||||
sqlx::query_as!(
|
||||
BlobRow,
|
||||
"SELECT key, data, timestamp FROM sync_metadata WHERE key = ?",
|
||||
key
|
||||
)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_sync_metadata(&self, key: String, data: Uint8Array) -> napi::Result<()> {
|
||||
let data = data.as_ref();
|
||||
sqlx::query!(
|
||||
"INSERT INTO sync_metadata (key, data) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET data = excluded.data",
|
||||
key,
|
||||
data,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_sync_metadata_keys(&self) -> napi::Result<Vec<String>> {
|
||||
let keys = sqlx::query!("SELECT key FROM sync_metadata")
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map(|rows| rows.into_iter().map(|row| row.key).collect())
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn clear_sync_metadata(&self) -> napi::Result<()> {
|
||||
sqlx::query!("DELETE FROM sync_metadata")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn del_sync_metadata(&self, key: String) -> napi::Result<()> {
|
||||
sqlx::query!("DELETE FROM sync_metadata WHERE key = ?", key)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn init_version(&self) -> napi::Result<()> {
|
||||
// create version_info table
|
||||
sqlx::query!(
|
||||
"CREATE TABLE IF NOT EXISTS version_info (
|
||||
version NUMBER NOT NULL,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
)"
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
// `3` is the first version that has version_info table,
|
||||
// do not modify the version number.
|
||||
sqlx::query!("INSERT INTO version_info (version) VALUES (3)")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn set_version(&self, version: i32) -> napi::Result<()> {
|
||||
if version > LATEST_VERSION {
|
||||
return Err(anyhow::Error::msg("Version is too new").into());
|
||||
}
|
||||
sqlx::query!("UPDATE version_info SET version = ?", version)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn get_max_version(&self) -> napi::Result<i64> {
|
||||
// 4 is the current version
|
||||
let version = sqlx::query!("SELECT COALESCE(MAX(version), 4) AS max_version FROM version_info")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?
|
||||
.max_version;
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn close(&self) {
|
||||
self.pool.close().await;
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn is_close(&self) -> bool {
|
||||
self.pool.is_closed()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn validate(path: String) -> ValidationResult {
|
||||
let pool = match open_readonly_pool(&path).await {
|
||||
Ok(pool) => pool,
|
||||
Err(_) => return ValidationResult::GeneralError,
|
||||
};
|
||||
|
||||
let tables_res = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
.fetch_all(&pool)
|
||||
.await;
|
||||
|
||||
let tables_exist = match tables_res {
|
||||
Ok(res) => {
|
||||
let names: Vec<String> = res.iter().map(|row| row.get(0)).collect();
|
||||
names.contains(&"updates".to_string()) && names.contains(&"blobs".to_string())
|
||||
}
|
||||
Err(_) => return ValidationResult::GeneralError,
|
||||
};
|
||||
|
||||
let tables_res = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
.fetch_all(&pool)
|
||||
.await;
|
||||
|
||||
let version_exist = match tables_res {
|
||||
Ok(res) => {
|
||||
let names: Vec<String> = res.iter().map(|row| row.get(0)).collect();
|
||||
names.contains(&"version_info".to_string())
|
||||
}
|
||||
Err(_) => return ValidationResult::GeneralError,
|
||||
};
|
||||
|
||||
let columns_res = sqlx::query("PRAGMA table_info(updates)").fetch_all(&pool).await;
|
||||
|
||||
let doc_id_exist = match columns_res {
|
||||
Ok(res) => {
|
||||
let names: Vec<String> = res.iter().map(|row| row.get(1)).collect();
|
||||
names.contains(&"doc_id".to_string())
|
||||
}
|
||||
Err(_) => return ValidationResult::GeneralError,
|
||||
};
|
||||
|
||||
if !tables_exist {
|
||||
ValidationResult::MissingTables
|
||||
} else if !doc_id_exist {
|
||||
ValidationResult::MissingDocIdColumn
|
||||
} else if !version_exist {
|
||||
ValidationResult::MissingVersionColumn
|
||||
} else {
|
||||
ValidationResult::Valid
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn validate_import_schema(&self) -> napi::Result<bool> {
|
||||
let pool = open_readonly_pool(&self.path).await?;
|
||||
Ok(
|
||||
validate_import_schema(&pool, &V1_IMPORT_SCHEMA_RULES)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?,
|
||||
)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn migrate_add_doc_id(&self) -> napi::Result<()> {
|
||||
// ignore errors
|
||||
match sqlx::query("ALTER TABLE updates ADD COLUMN doc_id TEXT")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err.to_string().contains("duplicate column name") {
|
||||
Ok(()) // Ignore error if it's due to duplicate column
|
||||
} else {
|
||||
Err(anyhow::Error::from(err).into()) // Propagate other errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush the WAL file to the database file.
|
||||
* See https://www.sqlite.org/pragma.html#pragma_wal_checkpoint:~:text=PRAGMA%20schema.wal_checkpoint%3B
|
||||
*/
|
||||
#[napi]
|
||||
pub async fn checkpoint(&self) -> napi::Result<()> {
|
||||
sqlx::query("PRAGMA wal_checkpoint(FULL);")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn vacuum_into(&self, path: String) -> napi::Result<()> {
|
||||
let pool = open_readonly_pool(&self.path).await?;
|
||||
sqlx::query("VACUUM INTO ?;")
|
||||
.bind(path)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn migrate_add_doc_id_index(&self) -> napi::Result<()> {
|
||||
// ignore errors
|
||||
match sqlx::query("CREATE INDEX IF NOT EXISTS idx_doc_id ON updates(doc_id);")
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
Err(anyhow::Error::from(err).into()) // Propagate other errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_readonly_pool(path: &str) -> anyhow::Result<Pool<Sqlite>> {
|
||||
let options = SqliteConnectOptions::new()
|
||||
.filename(path)
|
||||
.foreign_keys(false)
|
||||
.read_only(true);
|
||||
|
||||
Ok(
|
||||
SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(options)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
fs,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_import_schema_accepts_current_v1_schema() {
|
||||
let unique = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
|
||||
let base = std::env::temp_dir().join(format!("sqlite-v1-schema-valid-{unique}"));
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
|
||||
let source = base.join("storage.db");
|
||||
let connection = SqliteConnection::new(source.to_string_lossy().into_owned()).unwrap();
|
||||
connection.connect().await.unwrap();
|
||||
|
||||
assert!(connection.validate_import_schema().await.unwrap());
|
||||
|
||||
connection.close().await;
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_import_schema_rejects_unexpected_schema_objects() {
|
||||
let unique = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
|
||||
let base = std::env::temp_dir().join(format!("sqlite-v1-schema-{unique}"));
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
|
||||
let source = base.join("storage.db");
|
||||
let connection = SqliteConnection::new(source.to_string_lossy().into_owned()).unwrap();
|
||||
connection.connect().await.unwrap();
|
||||
|
||||
sqlx::query("CREATE TRIGGER rogue_trigger AFTER INSERT ON updates BEGIN SELECT 1; END;")
|
||||
.execute(&connection.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!connection.validate_import_schema().await.unwrap());
|
||||
|
||||
connection.close().await;
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
}
|
||||
90
packages/frontend/native/src/hashcash.rs
Normal file
90
packages/frontend/native/src/hashcash.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use affine_common::hashcash::Stamp;
|
||||
use napi::{Env, Result, Task, bindgen_prelude::AsyncTask};
|
||||
use napi_derive::napi;
|
||||
|
||||
pub struct AsyncVerifyChallengeResponse {
|
||||
response: String,
|
||||
bits: u32,
|
||||
resource: String,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Task for AsyncVerifyChallengeResponse {
|
||||
type Output = bool;
|
||||
type JsValue = bool;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
Ok(if let Ok(stamp) = Stamp::try_from(self.response.as_str()) {
|
||||
stamp.check(self.bits, &self.resource)
|
||||
} else {
|
||||
false
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _: Env, output: bool) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn verify_challenge_response(
|
||||
response: String,
|
||||
bits: u32,
|
||||
resource: String,
|
||||
) -> AsyncTask<AsyncVerifyChallengeResponse> {
|
||||
AsyncTask::new(AsyncVerifyChallengeResponse {
|
||||
response,
|
||||
bits,
|
||||
resource,
|
||||
})
|
||||
}
|
||||
|
||||
pub struct AsyncMintChallengeResponse {
|
||||
bits: Option<u32>,
|
||||
resource: String,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Task for AsyncMintChallengeResponse {
|
||||
type Output = String;
|
||||
type JsValue = String;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
Ok(Stamp::mint(self.resource.clone(), self.bits).format())
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _: Env, output: String) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn mint_challenge_response(resource: String, bits: Option<u32>) -> AsyncTask<AsyncMintChallengeResponse> {
|
||||
AsyncTask::new(AsyncMintChallengeResponse { bits, resource })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use napi::Task;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hashcash_roundtrip() {
|
||||
let resource = "test-resource".to_string();
|
||||
let mut mint = AsyncMintChallengeResponse {
|
||||
bits: Some(8),
|
||||
resource: resource.clone(),
|
||||
};
|
||||
let stamp = mint.compute().unwrap();
|
||||
|
||||
let mut verify = AsyncVerifyChallengeResponse {
|
||||
response: stamp,
|
||||
bits: 8,
|
||||
resource,
|
||||
};
|
||||
assert!(verify.compute().unwrap());
|
||||
}
|
||||
}
|
||||
5
packages/frontend/native/src/import/mod.rs
Normal file
5
packages/frontend/native/src/import/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
mod onenote;
|
||||
mod session;
|
||||
|
||||
use onenote::OneNoteImportProvider;
|
||||
pub use session::*;
|
||||
534
packages/frontend/native/src/import/onenote/content/mod.rs
Normal file
534
packages/frontend/native/src/import/onenote/content/mod.rs
Normal file
@@ -0,0 +1,534 @@
|
||||
use std::{collections::HashSet, io::Read};
|
||||
|
||||
mod rich_text;
|
||||
|
||||
use affine_importer::{ImportWarning, ImportedAsset};
|
||||
use onenote_parser::{
|
||||
contents::{Content, EmbeddedFile, Image, Outline, OutlineElement, OutlineGroup, OutlineItem, Table},
|
||||
page::Page,
|
||||
};
|
||||
use rich_text::rich_text_markdown;
|
||||
use serde_json::Value as JsonValue;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(super) fn page_markdown(
|
||||
page: &Page,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
let mut lines = Vec::new();
|
||||
for content in page.contents() {
|
||||
if let Some(outline) = content.outline() {
|
||||
lines.push(outline_markdown(outline, assets, warnings, source_path));
|
||||
} else if let Some(image) = content.image() {
|
||||
lines.push(image_markdown(image, assets, warnings, source_path));
|
||||
} else if let Some(file) = content.embedded_file() {
|
||||
lines.push(embedded_file_markdown(file, assets, warnings, source_path));
|
||||
} else if content.ink().is_some() {
|
||||
push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content");
|
||||
}
|
||||
}
|
||||
let markdown = lines
|
||||
.into_iter()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
if markdown.trim().is_empty() {
|
||||
" ".to_string()
|
||||
} else {
|
||||
markdown
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn rewrite_non_image_assets_to_attachments(value: &mut JsonValue, assets: &[ImportedAsset]) {
|
||||
match value {
|
||||
JsonValue::Object(map) => {
|
||||
let attachment = map
|
||||
.get("flavour")
|
||||
.and_then(JsonValue::as_str)
|
||||
.filter(|flavour| *flavour == "affine:image")
|
||||
.and_then(|_| map.get("props"))
|
||||
.and_then(JsonValue::as_object)
|
||||
.and_then(|props| props.get("sourceId"))
|
||||
.and_then(JsonValue::as_str)
|
||||
.and_then(|source_id| assets.iter().find(|asset| asset.blob_id == source_id))
|
||||
.filter(|asset| !asset.mime.starts_with("image/"));
|
||||
|
||||
if let Some(asset) = attachment {
|
||||
map.insert(
|
||||
"flavour".to_string(),
|
||||
JsonValue::String("affine:attachment".to_string()),
|
||||
);
|
||||
if let Some(props) = map.get_mut("props").and_then(JsonValue::as_object_mut) {
|
||||
props.insert("name".to_string(), JsonValue::String(asset.file_name.clone()));
|
||||
props.insert("size".to_string(), JsonValue::Number(asset.bytes.len().into()));
|
||||
props.insert("type".to_string(), JsonValue::String(asset.mime.clone()));
|
||||
props.insert("embed".to_string(), JsonValue::Bool(false));
|
||||
props.insert("style".to_string(), JsonValue::String("horizontalThin".to_string()));
|
||||
props.insert("footnoteIdentifier".to_string(), JsonValue::Null);
|
||||
}
|
||||
}
|
||||
for value in map.values_mut() {
|
||||
rewrite_non_image_assets_to_attachments(value, assets);
|
||||
}
|
||||
}
|
||||
JsonValue::Array(values) => {
|
||||
for value in values {
|
||||
rewrite_non_image_assets_to_attachments(value, assets);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn remove_blocks_with_source_ids(value: &mut JsonValue, source_ids: &HashSet<String>) {
|
||||
if source_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
match value {
|
||||
JsonValue::Object(map) => {
|
||||
for value in map.values_mut() {
|
||||
remove_blocks_with_source_ids(value, source_ids);
|
||||
}
|
||||
}
|
||||
JsonValue::Array(values) => {
|
||||
for value in values.iter_mut() {
|
||||
remove_blocks_with_source_ids(value, source_ids);
|
||||
}
|
||||
values.retain(|value| {
|
||||
block_source_id(value)
|
||||
.map(|source_id| !source_ids.contains(source_id))
|
||||
.unwrap_or(true)
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn block_source_id(value: &JsonValue) -> Option<&str> {
|
||||
value
|
||||
.as_object()
|
||||
.and_then(|map| map.get("props"))
|
||||
.and_then(JsonValue::as_object)
|
||||
.and_then(|props| props.get("sourceId"))
|
||||
.and_then(JsonValue::as_str)
|
||||
}
|
||||
|
||||
fn outline_markdown(
|
||||
outline: &Outline,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
outline
|
||||
.items()
|
||||
.iter()
|
||||
.map(|item| outline_item_markdown(item, 0, assets, warnings, source_path))
|
||||
.filter(|item| !item.trim().is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n")
|
||||
}
|
||||
|
||||
fn outline_item_markdown(
|
||||
item: &OutlineItem,
|
||||
depth: usize,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
match item {
|
||||
OutlineItem::Element(element) => outline_element_markdown(element, depth, assets, warnings, source_path),
|
||||
OutlineItem::Group(group) => outline_group_markdown(group, depth, assets, warnings, source_path),
|
||||
}
|
||||
}
|
||||
|
||||
fn outline_group_markdown(
|
||||
group: &OutlineGroup,
|
||||
depth: usize,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
let depth = depth + group.child_level() as usize;
|
||||
group
|
||||
.outlines()
|
||||
.iter()
|
||||
.map(|item| outline_item_markdown(item, depth, assets, warnings, source_path))
|
||||
.filter(|item| !item.trim().is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn outline_element_markdown(
|
||||
element: &OutlineElement,
|
||||
depth: usize,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
let mut content = element
|
||||
.contents()
|
||||
.iter()
|
||||
.map(|content| content_markdown(content, assets, warnings, source_path))
|
||||
.filter(|content| !content.trim().is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
|
||||
if !content.trim().is_empty() && element_is_list(element) {
|
||||
content = list_item_markdown(element, depth, &content);
|
||||
}
|
||||
|
||||
let mut lines = Vec::new();
|
||||
if !content.trim().is_empty() {
|
||||
lines.push(content);
|
||||
}
|
||||
for child in element.children() {
|
||||
let child = outline_item_markdown(child, depth + 1, assets, warnings, source_path);
|
||||
if !child.trim().is_empty() {
|
||||
lines.push(child);
|
||||
}
|
||||
}
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
fn content_markdown(
|
||||
content: &Content,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
if let Some(text) = content.rich_text() {
|
||||
return rich_text_markdown(text);
|
||||
}
|
||||
if let Some(table) = content.table() {
|
||||
return table_markdown(table, assets, warnings, source_path);
|
||||
}
|
||||
if let Some(image) = content.image() {
|
||||
return image_markdown(image, assets, warnings, source_path);
|
||||
} else if let Some(file) = content.embedded_file() {
|
||||
return embedded_file_markdown(file, assets, warnings, source_path);
|
||||
} else if content.ink().is_some() {
|
||||
push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content");
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn table_markdown(
|
||||
table: &Table,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
let rows = table
|
||||
.contents()
|
||||
.iter()
|
||||
.map(|row| {
|
||||
row
|
||||
.contents()
|
||||
.iter()
|
||||
.map(|cell| {
|
||||
let markdown = cell
|
||||
.contents()
|
||||
.iter()
|
||||
.map(|element| outline_element_markdown(element, 0, assets, warnings, source_path))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
markdown_table_cell(&markdown)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if rows.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let cols = rows.iter().map(Vec::len).max().unwrap_or(0).max(1);
|
||||
let mut markdown = String::new();
|
||||
let header = rows.first().cloned().unwrap_or_default();
|
||||
markdown.push_str(&markdown_row(&header, cols));
|
||||
markdown.push('\n');
|
||||
markdown.push_str(&markdown_row(&vec!["---".to_string(); cols], cols));
|
||||
for row in rows.iter().skip(1) {
|
||||
markdown.push('\n');
|
||||
markdown.push_str(&markdown_row(row, cols));
|
||||
}
|
||||
markdown
|
||||
}
|
||||
|
||||
fn markdown_row(row: &[String], cols: usize) -> String {
|
||||
let mut cells = row.to_vec();
|
||||
cells.resize(cols, String::new());
|
||||
format!("| {} |", cells.join(" | "))
|
||||
}
|
||||
|
||||
fn markdown_table_cell(value: &str) -> String {
|
||||
value
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join("<br />")
|
||||
.replace('|', "\\|")
|
||||
}
|
||||
|
||||
fn escape_markdown_label(value: &str) -> String {
|
||||
let mut output = String::with_capacity(value.len());
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'\\' | '[' | ']' | '*' | '_' | '`' => {
|
||||
output.push('\\');
|
||||
output.push(ch);
|
||||
}
|
||||
'\n' | '\r' => output.push(' '),
|
||||
_ => output.push(ch),
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn image_markdown(
|
||||
image: &Image,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
let Some(bytes) = read_optional_blob(image.read(), warnings, source_path, "onenote-image") else {
|
||||
return image
|
||||
.alt_text()
|
||||
.or_else(|| image.text())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
};
|
||||
let detected = infer::get(&bytes);
|
||||
let file_name = image_file_name(image, assets.len() + 1, detected.map(|kind| kind.extension()));
|
||||
let blob_id = blob_id(&bytes);
|
||||
let mime = image_mime(image, &file_name, detected.map(|kind| kind.mime_type()));
|
||||
assets.push(ImportedAsset {
|
||||
blob_id: blob_id.clone(),
|
||||
source_path: format!("{source_path}/{file_name}"),
|
||||
file_name,
|
||||
mime,
|
||||
bytes,
|
||||
});
|
||||
let caption = image.alt_text().or_else(|| image.text()).unwrap_or("");
|
||||
format!("", escape_markdown_label(caption), blob_id)
|
||||
}
|
||||
|
||||
fn element_is_list(element: &OutlineElement) -> bool {
|
||||
!element.list_contents().is_empty()
|
||||
}
|
||||
|
||||
fn list_item_markdown(element: &OutlineElement, depth: usize, content: &str) -> String {
|
||||
let marker = list_marker(element);
|
||||
let indent = " ".repeat(depth);
|
||||
content
|
||||
.lines()
|
||||
.enumerate()
|
||||
.map(|(index, line)| {
|
||||
if index == 0 {
|
||||
format!("{indent}{marker} {line}")
|
||||
} else if line.trim().is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{indent} {line}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn list_marker(element: &OutlineElement) -> &'static str {
|
||||
let ordered = element.list_contents().iter().any(|list| {
|
||||
list.list_restart().is_some()
|
||||
|| list
|
||||
.list_format()
|
||||
.iter()
|
||||
.any(|ch| ch.is_ascii_digit() || matches!(ch, '%' | '#'))
|
||||
});
|
||||
if ordered { "1." } else { "-" }
|
||||
}
|
||||
|
||||
fn image_file_name(image: &Image, index: usize, detected_ext: Option<&'static str>) -> String {
|
||||
let raw_name = image
|
||||
.image_filename()
|
||||
.filter(|name| !name.trim().is_empty())
|
||||
.map(|name| sanitize_file_name(name, "image"))
|
||||
.unwrap_or_else(|| {
|
||||
let ext = image.extension().or(detected_ext).unwrap_or("png");
|
||||
format!("image-{index}.{ext}")
|
||||
});
|
||||
|
||||
if mime_from_path(&raw_name).starts_with("image/") {
|
||||
return raw_name;
|
||||
}
|
||||
|
||||
image
|
||||
.extension()
|
||||
.or(detected_ext)
|
||||
.map(|ext| format!("{raw_name}.{ext}"))
|
||||
.unwrap_or(raw_name)
|
||||
}
|
||||
|
||||
fn embedded_file_markdown(
|
||||
file: &EmbeddedFile,
|
||||
assets: &mut Vec<ImportedAsset>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
) -> String {
|
||||
let mut reader = file.read();
|
||||
let mut bytes = Vec::new();
|
||||
if let Err(error) = reader.read_to_end(&mut bytes) {
|
||||
push_warning(
|
||||
warnings,
|
||||
source_path,
|
||||
"onenote-attachment",
|
||||
&format!("Skipped OneNote embedded file content: {error}"),
|
||||
);
|
||||
return file.filename().to_string();
|
||||
}
|
||||
let file_name = sanitize_file_name(file.filename(), "attachment");
|
||||
let blob_id = blob_id(&bytes);
|
||||
assets.push(ImportedAsset {
|
||||
blob_id: blob_id.clone(),
|
||||
source_path: format!("{source_path}/{file_name}"),
|
||||
file_name: file_name.clone(),
|
||||
mime: mime_from_path(&file_name).to_string(),
|
||||
bytes,
|
||||
});
|
||||
format!("", escape_markdown_label(&file_name), blob_id)
|
||||
}
|
||||
|
||||
fn read_optional_blob(
|
||||
reader: Option<Box<dyn Read>>,
|
||||
warnings: &mut Vec<ImportWarning>,
|
||||
source_path: &str,
|
||||
code: &str,
|
||||
) -> Option<Vec<u8>> {
|
||||
let Some(mut reader) = reader else {
|
||||
push_warning(
|
||||
warnings,
|
||||
source_path,
|
||||
code,
|
||||
"Skipped OneNote blob content: missing data",
|
||||
);
|
||||
return None;
|
||||
};
|
||||
let mut bytes = Vec::new();
|
||||
if let Err(error) = reader.read_to_end(&mut bytes) {
|
||||
push_warning(
|
||||
warnings,
|
||||
source_path,
|
||||
code,
|
||||
&format!("Skipped OneNote blob content: {error}"),
|
||||
);
|
||||
return None;
|
||||
}
|
||||
Some(bytes)
|
||||
}
|
||||
|
||||
fn image_mime(image: &Image, file_name: &str, detected_mime: Option<&'static str>) -> String {
|
||||
image
|
||||
.extension()
|
||||
.map(mime_from_extension)
|
||||
.or(detected_mime)
|
||||
.unwrap_or_else(|| mime_from_path(file_name))
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn blob_id(bytes: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(bytes);
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn sanitize_file_name(value: &str, fallback: &str) -> String {
|
||||
let name = value
|
||||
.split(['/', '\\', '\0'])
|
||||
.filter(|part| !part.is_empty() && *part != "." && *part != "..")
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.trim()
|
||||
.to_string();
|
||||
if name.is_empty() { fallback.to_string() } else { name }
|
||||
}
|
||||
|
||||
fn mime_from_path(path: &str) -> &'static str {
|
||||
match path.rsplit('.').next().unwrap_or("").to_lowercase().as_str() {
|
||||
"png" => "image/png",
|
||||
"jpg" | "jpeg" => "image/jpeg",
|
||||
"gif" => "image/gif",
|
||||
"webp" => "image/webp",
|
||||
"bmp" => "image/bmp",
|
||||
"svg" => "image/svg+xml",
|
||||
"pdf" => "application/pdf",
|
||||
"txt" => "text/plain",
|
||||
"md" | "markdown" => "text/markdown",
|
||||
"csv" => "text/csv",
|
||||
"zip" => "application/zip",
|
||||
"doc" => "application/msword",
|
||||
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xls" => "application/vnd.ms-excel",
|
||||
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"ppt" => "application/vnd.ms-powerpoint",
|
||||
"pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
fn mime_from_extension(extension: &str) -> &'static str {
|
||||
mime_from_path(extension)
|
||||
}
|
||||
|
||||
fn push_warning(warnings: &mut Vec<ImportWarning>, source_path: &str, code: &str, message: &str) {
|
||||
warnings.push(ImportWarning {
|
||||
code: code.to_string(),
|
||||
source_path: Some(source_path.to_string()),
|
||||
message: message.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn markdown_table_cell_collapses_multiline_content() {
|
||||
assert_eq!(
|
||||
markdown_table_cell("first line\nsecond | line\n\n third line "),
|
||||
"first line<br />second \\| line<br />third line"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_blocks_with_source_ids_removes_matching_nested_blocks() {
|
||||
let mut snapshot = json!({
|
||||
"children": [
|
||||
{
|
||||
"flavour": "affine:paragraph",
|
||||
"props": {},
|
||||
"children": [
|
||||
{
|
||||
"flavour": "affine:image",
|
||||
"props": { "sourceId": "skipped" },
|
||||
"children": []
|
||||
},
|
||||
{
|
||||
"flavour": "affine:image",
|
||||
"props": { "sourceId": "kept" },
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
});
|
||||
let source_ids = HashSet::from(["skipped".to_string()]);
|
||||
|
||||
remove_blocks_with_source_ids(&mut snapshot, &source_ids);
|
||||
|
||||
assert_eq!(snapshot["children"][0]["children"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(snapshot["children"][0]["children"][0]["props"]["sourceId"], "kept");
|
||||
}
|
||||
}
|
||||
171
packages/frontend/native/src/import/onenote/content/rich_text.rs
Normal file
171
packages/frontend/native/src/import/onenote/content/rich_text.rs
Normal file
@@ -0,0 +1,171 @@
|
||||
use onenote_parser::{
|
||||
contents::{ParagraphStyling, RichText},
|
||||
property::common::ColorRef,
|
||||
};
|
||||
|
||||
pub(super) fn rich_text_markdown(text: &RichText) -> String {
|
||||
let mut markdown = styled_text_markdown(text);
|
||||
let trimmed = markdown.trim();
|
||||
if trimmed.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
if let Some(prefix) = heading_prefix(text, trimmed) {
|
||||
markdown = format!("{prefix} {}", trimmed.trim_start_matches('#').trim_start());
|
||||
}
|
||||
markdown
|
||||
}
|
||||
|
||||
fn styled_text_markdown(text: &RichText) -> String {
|
||||
let chars = text.text().chars().collect::<Vec<_>>();
|
||||
let styles = text.text_run_formatting();
|
||||
if chars.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
if styles.is_empty() {
|
||||
return chars.into_iter().collect();
|
||||
}
|
||||
|
||||
let mut output = String::new();
|
||||
let mut start = 0usize;
|
||||
let mut pending_link: Option<String> = None;
|
||||
for (index, style) in styles.iter().enumerate() {
|
||||
let end = text
|
||||
.text_run_indices()
|
||||
.get(index)
|
||||
.map(|end| (*end as usize).min(chars.len()))
|
||||
.unwrap_or(chars.len());
|
||||
if end < start {
|
||||
continue;
|
||||
}
|
||||
let segment = chars[start..end].iter().collect::<String>();
|
||||
start = end;
|
||||
if segment.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if style.hidden() {
|
||||
if let Some(url) = extract_hyperlink_url(&segment) {
|
||||
pending_link = Some(url);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
output.push_str(&format_segment(&segment, style, &mut pending_link));
|
||||
}
|
||||
if start < chars.len() {
|
||||
output.push_str(&chars[start..].iter().collect::<String>());
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn heading_prefix(text: &RichText, value: &str) -> Option<&'static str> {
|
||||
if value.chars().count() > 120 || value.lines().count() > 1 {
|
||||
return None;
|
||||
}
|
||||
let font_size = text
|
||||
.text_run_formatting()
|
||||
.iter()
|
||||
.filter(|style| !style.hidden())
|
||||
.filter_map(ParagraphStyling::font_size)
|
||||
.max()
|
||||
.or_else(|| text.paragraph_style().font_size());
|
||||
|
||||
match font_size {
|
||||
Some(size) if size >= 28 => Some("##"),
|
||||
Some(size) if size >= 24 => Some("###"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn format_segment(segment: &str, style: &ParagraphStyling, pending_link: &mut Option<String>) -> String {
|
||||
let mut value = escape_html(segment);
|
||||
if segment.trim().is_empty() {
|
||||
return value;
|
||||
}
|
||||
|
||||
if style.bold() && should_keep_inline_style(segment) {
|
||||
value = wrap_inline(&value, "<strong>", "</strong>");
|
||||
}
|
||||
if style.italic() && should_keep_inline_style(segment) {
|
||||
value = wrap_inline(&value, "<em>", "</em>");
|
||||
}
|
||||
if style.strikethrough() {
|
||||
value = wrap_inline(&value, "<del>", "</del>");
|
||||
}
|
||||
if style.underline() {
|
||||
value = wrap_inline(&value, "<u>", "</u>");
|
||||
}
|
||||
if let Some(color) = style.font_color().and_then(hex_color) {
|
||||
value = wrap_inline(&value, &format!("<span style=\"color: {color}\">"), "</span>");
|
||||
}
|
||||
if let Some(color) = style.highlight().and_then(hex_color) {
|
||||
value = wrap_inline(&value, &format!("<span style=\"color: {color}\">"), "</span>");
|
||||
}
|
||||
|
||||
if style.hyperlink_protected()
|
||||
&& let Some(url) = pending_link.take()
|
||||
{
|
||||
return wrap_inline(&value, &format!("<a href=\"{}\">", escape_html_attr(&url)), "</a>");
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
fn should_keep_inline_style(value: &str) -> bool {
|
||||
value.chars().any(|ch| ch.is_alphanumeric())
|
||||
}
|
||||
|
||||
fn wrap_inline(value: &str, open: &str, close: &str) -> String {
|
||||
let leading_len = value.len() - value.trim_start().len();
|
||||
let trailing_len = value.len() - value.trim_end().len();
|
||||
let (leading, rest) = value.split_at(leading_len);
|
||||
let (middle, trailing) = rest.split_at(rest.len().saturating_sub(trailing_len));
|
||||
if middle.is_empty() {
|
||||
value.to_string()
|
||||
} else {
|
||||
format!("{leading}{open}{middle}{close}{trailing}")
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_color(color: ColorRef) -> Option<String> {
|
||||
match color {
|
||||
ColorRef::Auto => None,
|
||||
ColorRef::Manual { r, g, b } => Some(format!("#{r:02x}{g:02x}{b:02x}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_html(value: &str) -> String {
|
||||
let mut output = String::with_capacity(value.len());
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'&' => output.push_str("&"),
|
||||
'<' => output.push_str("<"),
|
||||
'>' => output.push_str(">"),
|
||||
_ => output.push(ch),
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn escape_html_attr(value: &str) -> String {
|
||||
let mut output = String::with_capacity(value.len());
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'&' => output.push_str("&"),
|
||||
'<' => output.push_str("<"),
|
||||
'>' => output.push_str(">"),
|
||||
'"' => output.push_str("""),
|
||||
_ => output.push(ch),
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn extract_hyperlink_url(value: &str) -> Option<String> {
|
||||
let marker = "HYPERLINK";
|
||||
let start = value.find(marker)? + marker.len();
|
||||
let rest = value[start..].trim();
|
||||
if let Some(quoted) = rest.strip_prefix('"') {
|
||||
let end = quoted.find('"')?;
|
||||
return Some(quoted[..end].to_string());
|
||||
}
|
||||
rest.split_whitespace().next().map(ToString::to_string)
|
||||
}
|
||||
458
packages/frontend/native/src/import/onenote/mod.rs
Normal file
458
packages/frontend/native/src/import/onenote/mod.rs
Normal file
@@ -0,0 +1,458 @@
|
||||
use std::{
|
||||
collections::{HashSet, hash_map::DefaultHasher},
|
||||
fs,
|
||||
hash::{Hash, Hasher},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
mod content;
|
||||
#[cfg(windows)]
|
||||
mod windows_fs;
|
||||
|
||||
use affine_importer::{
|
||||
FolderHierarchyDelta, ImportBatch, ImportBatchLimits, ImportCursor, ImportError, ImportOptions, ImportProgress,
|
||||
ImportProvider, ImportResult, ImportSource, ImportWarning, ImportedAsset, ImportedDocMeta, ImportedDocSnapshot,
|
||||
};
|
||||
use content::{page_markdown, remove_blocks_with_source_ids, rewrite_non_image_assets_to_attachments};
|
||||
use onenote_parser::{
|
||||
Parser,
|
||||
notebook::Notebook,
|
||||
page::Page,
|
||||
section::{Section, SectionEntry, SectionGroup},
|
||||
};
|
||||
use typed_path::TypedPath;
|
||||
#[cfg(windows)]
|
||||
use windows_fs::WindowsFs;
|
||||
|
||||
pub struct OneNoteImportProvider;
|
||||
|
||||
impl OneNoteImportProvider {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportProvider for OneNoteImportProvider {
|
||||
fn format(&self) -> &'static str {
|
||||
"oneNote"
|
||||
}
|
||||
|
||||
fn create_cursor(
|
||||
&self,
|
||||
source: ImportSource,
|
||||
_options: ImportOptions,
|
||||
limits: ImportBatchLimits,
|
||||
) -> ImportResult<Box<dyn ImportCursor>> {
|
||||
let mut planner = OneNotePlanner {
|
||||
docs: Vec::new(),
|
||||
folders: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
};
|
||||
planner.read_source(source)?;
|
||||
Ok(Box::new(OneNoteImportCursor {
|
||||
docs: planner.docs,
|
||||
folders: planner.folders,
|
||||
warnings: planner.warnings,
|
||||
next_asset_doc: 0,
|
||||
next_asset_index: 0,
|
||||
next_doc: 0,
|
||||
emitted_final: false,
|
||||
skipped_blob_ids: HashSet::new(),
|
||||
limits,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
struct OneNoteDoc {
|
||||
id: String,
|
||||
title: String,
|
||||
markdown: String,
|
||||
assets: Vec<ImportedAsset>,
|
||||
}
|
||||
|
||||
struct OneNotePlanner {
|
||||
docs: Vec<OneNoteDoc>,
|
||||
folders: Vec<FolderHierarchyDelta>,
|
||||
warnings: Vec<ImportWarning>,
|
||||
}
|
||||
|
||||
impl OneNotePlanner {
|
||||
fn read_source(&mut self, source: ImportSource) -> ImportResult<()> {
|
||||
match source {
|
||||
ImportSource::FilePath(path) => self.read_file(&path),
|
||||
ImportSource::DirectoryPath(path) => self.read_directory(&path),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_file(&mut self, path: &Path) -> ImportResult<()> {
|
||||
let parser = parser();
|
||||
match extension(path).as_deref() {
|
||||
Some("one") => {
|
||||
let source_path = path.to_string_lossy();
|
||||
let section = parser
|
||||
.parse_section(TypedPath::derive(source_path.as_ref()))
|
||||
.map_err(import_error)?;
|
||||
self.add_section(§ion, None);
|
||||
Ok(())
|
||||
}
|
||||
Some("onetoc2") => {
|
||||
let source_path = path.to_string_lossy();
|
||||
let notebook = parser
|
||||
.parse_notebook(TypedPath::derive(source_path.as_ref()))
|
||||
.map_err(import_error)?;
|
||||
self.add_notebook(¬ebook, notebook_name(path));
|
||||
Ok(())
|
||||
}
|
||||
Some("onepkg") => {
|
||||
let source_path = path.to_string_lossy();
|
||||
let notebook = parser
|
||||
.parse_package(TypedPath::derive(source_path.as_ref()))
|
||||
.map_err(import_error)?;
|
||||
self.add_notebook(¬ebook, notebook_name(path));
|
||||
Ok(())
|
||||
}
|
||||
_ => Err(ImportError::InvalidSource(format!(
|
||||
"unsupported OneNote source: {}",
|
||||
path.to_string_lossy()
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_directory(&mut self, path: &Path) -> ImportResult<()> {
|
||||
let toc = find_first_onetoc2(path)?.ok_or_else(|| {
|
||||
ImportError::InvalidSource(format!(
|
||||
"directory contains no .onetoc2 file: {}",
|
||||
path.to_string_lossy()
|
||||
))
|
||||
})?;
|
||||
let parser = parser();
|
||||
let toc_path = toc.to_string_lossy();
|
||||
let notebook = parser
|
||||
.parse_notebook(TypedPath::derive(toc_path.as_ref()))
|
||||
.map_err(import_error)?;
|
||||
self.add_notebook(¬ebook, notebook_name(path));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_notebook(&mut self, notebook: &Notebook, name: String) {
|
||||
let root = sanitize_path_part(&name);
|
||||
self.folders.push(FolderHierarchyDelta {
|
||||
path: root.clone(),
|
||||
name,
|
||||
parent_path: None,
|
||||
page_id: None,
|
||||
icon: None,
|
||||
});
|
||||
for entry in notebook.entries() {
|
||||
self.add_section_entry(entry, Some(root.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
fn add_section_entry(&mut self, entry: &SectionEntry, parent_path: Option<String>) {
|
||||
match entry {
|
||||
SectionEntry::Section(section) => self.add_section(section, parent_path),
|
||||
SectionEntry::SectionGroup(group) => self.add_section_group(group, parent_path),
|
||||
}
|
||||
}
|
||||
|
||||
fn add_section_group(&mut self, group: &SectionGroup, parent_path: Option<String>) {
|
||||
let path = join_import_path(parent_path.as_deref(), group.display_name());
|
||||
self.folders.push(FolderHierarchyDelta {
|
||||
path: path.clone(),
|
||||
name: group.display_name().to_string(),
|
||||
parent_path,
|
||||
page_id: None,
|
||||
icon: None,
|
||||
});
|
||||
for entry in group.entries() {
|
||||
self.add_section_entry(entry, Some(path.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
fn add_section(&mut self, section: &Section, parent_path: Option<String>) {
|
||||
let path = join_import_path(parent_path.as_deref(), section.display_name());
|
||||
self.folders.push(FolderHierarchyDelta {
|
||||
path: path.clone(),
|
||||
name: section.display_name().to_string(),
|
||||
parent_path,
|
||||
page_id: None,
|
||||
icon: None,
|
||||
});
|
||||
let mut page_index = 0;
|
||||
for series in section.page_series() {
|
||||
for page in series.pages() {
|
||||
self.add_page(page, &path, page_index);
|
||||
page_index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_page(&mut self, page: &Page, parent_path: &str, page_index: usize) {
|
||||
let title = page
|
||||
.title_text()
|
||||
.map(str::trim)
|
||||
.filter(|title| !title.is_empty())
|
||||
.unwrap_or("Untitled")
|
||||
.to_string();
|
||||
let source_path = join_import_path(Some(parent_path), &title);
|
||||
let id = doc_id(page.link_target_id(), &title, parent_path, page_index);
|
||||
let mut assets = Vec::new();
|
||||
let markdown = page_markdown(page, &mut assets, &mut self.warnings, &source_path);
|
||||
self.folders.push(FolderHierarchyDelta {
|
||||
path: source_path.clone(),
|
||||
name: title.clone(),
|
||||
parent_path: Some(parent_path.to_string()),
|
||||
page_id: Some(id.clone()),
|
||||
icon: None,
|
||||
});
|
||||
self.docs.push(OneNoteDoc {
|
||||
id,
|
||||
title,
|
||||
markdown,
|
||||
assets,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn parser() -> Parser {
|
||||
Parser::new()
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn parser() -> Parser<WindowsFs> {
|
||||
Parser::new_with_fs(WindowsFs)
|
||||
}
|
||||
|
||||
struct OneNoteImportCursor {
|
||||
docs: Vec<OneNoteDoc>,
|
||||
folders: Vec<FolderHierarchyDelta>,
|
||||
warnings: Vec<ImportWarning>,
|
||||
next_asset_doc: usize,
|
||||
next_asset_index: usize,
|
||||
next_doc: usize,
|
||||
emitted_final: bool,
|
||||
skipped_blob_ids: HashSet<String>,
|
||||
limits: ImportBatchLimits,
|
||||
}
|
||||
|
||||
impl ImportCursor for OneNoteImportCursor {
|
||||
fn total(&self) -> usize {
|
||||
self.docs.len()
|
||||
}
|
||||
|
||||
fn next_batch(&mut self) -> ImportResult<Option<ImportBatch>> {
|
||||
if self.next_doc >= self.docs.len() && self.emitted_final {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(batch) = self.next_asset_batch() {
|
||||
return Ok(Some(batch));
|
||||
}
|
||||
|
||||
let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.docs.len());
|
||||
let mut batch = empty_batch(self.docs.len());
|
||||
for doc in &self.docs[self.next_doc..end] {
|
||||
let mut snapshot = affine_doc_loader::build_doc_snapshot(&doc.title, &doc.markdown, &doc.id)?;
|
||||
rewrite_non_image_assets_to_attachments(&mut snapshot, &doc.assets);
|
||||
remove_blocks_with_source_ids(&mut snapshot, &self.skipped_blob_ids);
|
||||
batch.docs.push(ImportedDocSnapshot {
|
||||
id: doc.id.clone(),
|
||||
snapshot,
|
||||
meta: Some(ImportedDocMeta {
|
||||
title: Some(doc.title.clone()),
|
||||
create_date: None,
|
||||
updated_date: None,
|
||||
tags: None,
|
||||
favorite: None,
|
||||
trash: None,
|
||||
}),
|
||||
});
|
||||
}
|
||||
self.next_doc = end;
|
||||
if self.next_doc >= self.docs.len() {
|
||||
batch.folders = std::mem::take(&mut self.folders);
|
||||
batch.warnings = std::mem::take(&mut self.warnings);
|
||||
self.emitted_final = true;
|
||||
}
|
||||
batch.progress = ImportProgress {
|
||||
completed: self.next_doc,
|
||||
total: self.docs.len(),
|
||||
};
|
||||
batch.done = self.emitted_final;
|
||||
Ok(Some(batch))
|
||||
}
|
||||
}
|
||||
|
||||
impl OneNoteImportCursor {
|
||||
fn next_asset_batch(&mut self) -> Option<ImportBatch> {
|
||||
let max_blobs = self.limits.max_blobs.max(1);
|
||||
let max_blob_bytes = self.limits.max_blob_bytes;
|
||||
let mut batch = empty_batch(self.docs.len());
|
||||
let mut bytes_in_batch = 0u64;
|
||||
|
||||
while self.next_asset_doc < self.docs.len() && batch.blobs.len() < max_blobs {
|
||||
let doc = &self.docs[self.next_asset_doc];
|
||||
if self.next_asset_index >= doc.assets.len() {
|
||||
self.next_asset_doc += 1;
|
||||
self.next_asset_index = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
let asset = &doc.assets[self.next_asset_index];
|
||||
self.next_asset_index += 1;
|
||||
let asset_size = asset.bytes.len() as u64;
|
||||
if asset_size > max_blob_bytes {
|
||||
self.skipped_blob_ids.insert(asset.blob_id.clone());
|
||||
batch.warnings.push(ImportWarning {
|
||||
code: "blob_too_large".to_string(),
|
||||
source_path: Some(asset.source_path.clone()),
|
||||
message: format!("Skipped {}: blob is larger than batch limit", asset.source_path),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if !batch.blobs.is_empty() && bytes_in_batch + asset_size > max_blob_bytes {
|
||||
self.next_asset_index -= 1;
|
||||
break;
|
||||
}
|
||||
bytes_in_batch += asset_size;
|
||||
batch.blobs.push(asset.clone());
|
||||
}
|
||||
|
||||
if batch.blobs.is_empty() && batch.warnings.is_empty() {
|
||||
return None;
|
||||
}
|
||||
batch.progress = ImportProgress {
|
||||
completed: self.next_doc,
|
||||
total: self.docs.len(),
|
||||
};
|
||||
Some(batch)
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_batch(total: usize) -> ImportBatch {
|
||||
ImportBatch {
|
||||
docs: Vec::new(),
|
||||
blobs: Vec::new(),
|
||||
folders: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
icons: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
progress: ImportProgress { completed: 0, total },
|
||||
entry_id: None,
|
||||
is_workspace_file: false,
|
||||
done: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn find_first_onetoc2(root: &Path) -> ImportResult<Option<PathBuf>> {
|
||||
for entry in fs::read_dir(root)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
if let Some(found) = find_first_onetoc2(&path)? {
|
||||
return Ok(Some(found));
|
||||
}
|
||||
} else if extension(&path).as_deref() == Some("onetoc2") {
|
||||
return Ok(Some(path));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn import_error(error: onenote_parser::errors::Error) -> ImportError {
|
||||
ImportError::InvalidSource(error.to_string())
|
||||
}
|
||||
|
||||
fn notebook_name(path: &Path) -> String {
|
||||
path
|
||||
.file_stem()
|
||||
.or_else(|| path.file_name())
|
||||
.map(|name| name.to_string_lossy().to_string())
|
||||
.filter(|name| !name.is_empty())
|
||||
.unwrap_or_else(|| "OneNote".to_string())
|
||||
}
|
||||
|
||||
fn extension(path: &Path) -> Option<String> {
|
||||
path.extension().map(|ext| ext.to_string_lossy().to_lowercase())
|
||||
}
|
||||
|
||||
fn join_import_path(parent: Option<&str>, name: &str) -> String {
|
||||
let name = sanitize_path_part(name);
|
||||
parent
|
||||
.filter(|parent| !parent.is_empty())
|
||||
.map(|parent| format!("{parent}/{name}"))
|
||||
.unwrap_or(name)
|
||||
}
|
||||
|
||||
fn sanitize_path_part(value: &str) -> String {
|
||||
let name = value
|
||||
.split(['/', '\\'])
|
||||
.filter(|part| !part.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.trim()
|
||||
.to_string();
|
||||
if name.is_empty() { "Untitled".to_string() } else { name }
|
||||
}
|
||||
|
||||
fn doc_id(raw_id: &str, title: &str, parent_path: &str, page_index: usize) -> String {
|
||||
let mut id = raw_id
|
||||
.chars()
|
||||
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_')
|
||||
.collect::<String>();
|
||||
if id.is_empty() {
|
||||
let slug = title
|
||||
.chars()
|
||||
.filter(|ch| ch.is_ascii_alphanumeric())
|
||||
.take(32)
|
||||
.collect::<String>();
|
||||
let mut hasher = DefaultHasher::new();
|
||||
title.hash(&mut hasher);
|
||||
parent_path.hash(&mut hasher);
|
||||
page_index.hash(&mut hasher);
|
||||
let hash = format!("{:x}", hasher.finish());
|
||||
id = if slug.is_empty() {
|
||||
hash
|
||||
} else {
|
||||
format!("{slug}-{hash}")
|
||||
};
|
||||
}
|
||||
format!("onenote-{id}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sanitize_path_part_removes_path_separators() {
|
||||
assert_eq!(sanitize_path_part("Group/Section\\Page"), "Group Section Page");
|
||||
assert_eq!(sanitize_path_part("///"), "Untitled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_import_path_sanitizes_path_parts() {
|
||||
assert_eq!(
|
||||
join_import_path(Some("Notebook/Section"), "Page/Subpage\\Draft"),
|
||||
"Notebook/Section/Page Subpage Draft"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn doc_id_keeps_ascii_ids_and_hashes_non_ascii_titles() {
|
||||
assert_eq!(doc_id("{abc-123}", "Ignored", "Notebook/Section", 0), "onenote-abc-123");
|
||||
|
||||
let id = doc_id("", "中文标题", "Notebook/Section", 0);
|
||||
assert!(id.starts_with("onenote-"));
|
||||
assert!(id.len() > "onenote-".len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn doc_id_fallback_distinguishes_duplicate_titles() {
|
||||
let first = doc_id("", "Untitled", "Notebook/Section", 0);
|
||||
let second = doc_id("", "Untitled", "Notebook/Section", 1);
|
||||
|
||||
assert_ne!(first, second);
|
||||
}
|
||||
}
|
||||
152
packages/frontend/native/src/import/onenote/windows_fs.rs
Normal file
152
packages/frontend/native/src/import/onenote/windows_fs.rs
Normal file
@@ -0,0 +1,152 @@
|
||||
use std::{
|
||||
ffi::OsString,
|
||||
fs::{self, File},
|
||||
io::{Error, Read},
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use bytes::Bytes;
|
||||
use onenote_parser::{
|
||||
FileSystem,
|
||||
fs::{FileSource, file_source::CachedFileSource},
|
||||
};
|
||||
use typed_path::{TypedPath, TypedPathBuf};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct WindowsFs;
|
||||
|
||||
impl FileSystem for WindowsFs {
|
||||
fn is_directory(&self, path: TypedPath) -> Result<bool, Error> {
|
||||
match fs::metadata(resolve_path(path)?) {
|
||||
Ok(meta) => Ok(meta.is_dir()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_dir(&self, path: TypedPath) -> Result<Vec<TypedPathBuf>, Error> {
|
||||
fs::read_dir(resolve_path(path)?)?
|
||||
.map(|entry| entry.and_then(|entry| as_typed_path(&entry.path())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn read_file(&self, path: TypedPath) -> Result<Vec<u8>, Error> {
|
||||
fs::read(resolve_path(path)?)
|
||||
}
|
||||
|
||||
fn write_file(&self, path: TypedPath, data: &[u8]) -> Result<(), Error> {
|
||||
fs::write(resolve_path(path)?, data)
|
||||
}
|
||||
|
||||
fn stream_to_file(&self, path: TypedPath, reader: &mut dyn Read) -> Result<(), Error> {
|
||||
let mut file = File::create(resolve_path(path)?)?;
|
||||
std::io::copy(reader, &mut file)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn make_dir(&self, path: TypedPath) -> Result<(), Error> {
|
||||
fs::create_dir_all(resolve_path(path)?)
|
||||
}
|
||||
|
||||
fn canonicalize(&self, path: TypedPath) -> Result<TypedPathBuf, Error> {
|
||||
as_typed_path(&fs::canonicalize(resolve_path(path)?)?)
|
||||
}
|
||||
|
||||
fn exists(&self, path: TypedPath) -> Result<bool, Error> {
|
||||
fs::exists(resolve_path(path)?)
|
||||
}
|
||||
|
||||
fn open_file(&self, path: TypedPath) -> Result<Arc<dyn FileSource>, Error> {
|
||||
let file = File::open(resolve_path(path)?)?;
|
||||
let byte_length = file.metadata()?.len();
|
||||
Ok(Arc::new(CachedFileSource::new(WindowsFileSource { file, byte_length })))
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_path(path: TypedPath) -> Result<PathBuf, Error> {
|
||||
let path = if path.is_windows() {
|
||||
path.to_path_buf()
|
||||
} else {
|
||||
path
|
||||
.with_windows_encoding_checked()
|
||||
.map_err(|error| Error::new(std::io::ErrorKind::InvalidInput, error))?
|
||||
};
|
||||
let path = path
|
||||
.to_str()
|
||||
.ok_or_else(|| Error::new(std::io::ErrorKind::InvalidData, "path bytes are not valid UTF-8"))?;
|
||||
to_verbatim(PathBuf::from(path))
|
||||
}
|
||||
|
||||
fn as_typed_path(path: &Path) -> Result<TypedPathBuf, Error> {
|
||||
let path = path
|
||||
.to_str()
|
||||
.ok_or_else(|| Error::new(std::io::ErrorKind::InvalidData, "path is not valid UTF-8"))?;
|
||||
Ok(TypedPath::windows(path.as_bytes()).to_path_buf())
|
||||
}
|
||||
|
||||
fn to_verbatim(path: PathBuf) -> Result<PathBuf, Error> {
|
||||
let path = std::path::absolute(path)?.to_string_lossy().replace('/', "\\");
|
||||
if path.starts_with(r"\\?\") {
|
||||
return Ok(PathBuf::from(path));
|
||||
}
|
||||
let mut verbatim = OsString::new();
|
||||
if let Some(path) = path.strip_prefix(r"\\") {
|
||||
verbatim.push(r"\\?\UNC\");
|
||||
verbatim.push(path);
|
||||
} else {
|
||||
verbatim.push(r"\\?\");
|
||||
verbatim.push(path);
|
||||
}
|
||||
Ok(PathBuf::from(verbatim))
|
||||
}
|
||||
|
||||
struct WindowsFileSource {
|
||||
file: File,
|
||||
byte_length: u64,
|
||||
}
|
||||
|
||||
impl FileSource for WindowsFileSource {
|
||||
fn byte_length(&self) -> u64 {
|
||||
self.byte_length
|
||||
}
|
||||
|
||||
fn read_at(&self, offset: u64, len: usize) -> Result<Bytes, Error> {
|
||||
use std::os::windows::fs::FileExt;
|
||||
|
||||
let mut bytes = vec![0; len];
|
||||
let mut read = 0;
|
||||
while read < len {
|
||||
let count = self.file.seek_read(&mut bytes[read..], offset + read as u64)?;
|
||||
if count == 0 {
|
||||
return Err(Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"unexpected end of OneNote file",
|
||||
));
|
||||
}
|
||||
read += count;
|
||||
}
|
||||
Ok(Bytes::from(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn preserves_windows_absolute_path_prefixes() {
|
||||
assert_eq!(
|
||||
resolve_path(TypedPath::derive(r"C:\Users\Alice\Notebook.one")).unwrap(),
|
||||
PathBuf::from(r"\\?\C:\Users\Alice\Notebook.one")
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_path(TypedPath::derive(r"\\server\share\Notebook.one")).unwrap(),
|
||||
PathBuf::from(r"\\?\UNC\server\share\Notebook.one")
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_path(TypedPath::derive(r"\\?\D:\Notebook.one")).unwrap(),
|
||||
PathBuf::from(r"\\?\D:\Notebook.one")
|
||||
);
|
||||
}
|
||||
}
|
||||
279
packages/frontend/native/src/import/session.rs
Normal file
279
packages/frontend/native/src/import/session.rs
Normal file
@@ -0,0 +1,279 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
use affine_importer::{
|
||||
ImportBatchLimits, ImportOptions, ImportRegistry, ImportSession, ImportSessionOptions, ImportSource,
|
||||
};
|
||||
use napi::{Status, bindgen_prelude::*};
|
||||
use napi_derive::napi;
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
static IMPORT_SESSIONS: Lazy<Mutex<HashMap<String, ImportSession>>> = Lazy::new(|| Mutex::new(HashMap::new()));
|
||||
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
|
||||
static IMPORT_REGISTRY: Lazy<Arc<ImportRegistry>> = Lazy::new(|| {
|
||||
let mut registry = ImportRegistry::with_builtin();
|
||||
registry.register(super::OneNoteImportProvider::new());
|
||||
Arc::new(registry)
|
||||
});
|
||||
|
||||
#[napi(object)]
|
||||
pub struct CreateImportSessionOptions {
|
||||
pub format: String,
|
||||
pub source: CreateImportSessionSource,
|
||||
pub batch_limits: Option<CreateImportBatchLimits>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct CreateImportSessionSource {
|
||||
pub kind: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct CreateImportBatchLimits {
|
||||
pub max_docs: Option<u32>,
|
||||
pub max_blobs: Option<u32>,
|
||||
pub max_blob_bytes: Option<i64>,
|
||||
}
|
||||
|
||||
fn map_import_error(error: affine_importer::ImportError) -> Error {
|
||||
let status = match error {
|
||||
affine_importer::ImportError::Cancelled => Status::Cancelled,
|
||||
affine_importer::ImportError::UnsupportedFormat(_) | affine_importer::ImportError::InvalidSource(_) => {
|
||||
Status::InvalidArg
|
||||
}
|
||||
affine_importer::ImportError::Zip(_)
|
||||
| affine_importer::ImportError::Io(_)
|
||||
| affine_importer::ImportError::Document(_) => Status::GenericFailure,
|
||||
};
|
||||
Error::new(status, error.to_string())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn create_import_session(options: CreateImportSessionOptions) -> Result<String> {
|
||||
let source = match options.source.kind.as_str() {
|
||||
"filePath" => ImportSource::FilePath(PathBuf::from(options.source.path)),
|
||||
"directoryPath" => ImportSource::DirectoryPath(PathBuf::from(options.source.path)),
|
||||
kind => {
|
||||
return Err(Error::new(
|
||||
Status::InvalidArg,
|
||||
format!("unsupported import source kind: {kind}"),
|
||||
));
|
||||
}
|
||||
};
|
||||
let mut batch_limits = ImportBatchLimits::default();
|
||||
if let Some(limits) = options.batch_limits {
|
||||
if let Some(max_docs) = limits.max_docs {
|
||||
batch_limits.max_docs = max_docs as usize;
|
||||
}
|
||||
if let Some(max_blobs) = limits.max_blobs {
|
||||
batch_limits.max_blobs = max_blobs as usize;
|
||||
}
|
||||
if let Some(max_blob_bytes) = limits.max_blob_bytes {
|
||||
if !(0..=100 * 1024 * 1024).contains(&max_blob_bytes) {
|
||||
return Err(Error::new(Status::InvalidArg, "maxBlobBytes not valid"));
|
||||
}
|
||||
batch_limits.max_blob_bytes = max_blob_bytes as u64;
|
||||
}
|
||||
}
|
||||
let session = ImportSession::create(
|
||||
ImportSessionOptions {
|
||||
format: options.format,
|
||||
source,
|
||||
import_options: ImportOptions::default(),
|
||||
batch_limits,
|
||||
},
|
||||
IMPORT_REGISTRY.clone(),
|
||||
)
|
||||
.map_err(map_import_error)?;
|
||||
let id = format!("import-session-{}", NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed));
|
||||
IMPORT_SESSIONS
|
||||
.lock()
|
||||
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?
|
||||
.insert(id.clone(), session);
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn next_import_batch(session_id: String) -> Result<Option<String>> {
|
||||
let mut sessions = IMPORT_SESSIONS
|
||||
.lock()
|
||||
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?;
|
||||
let session = sessions
|
||||
.get_mut(&session_id)
|
||||
.ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?;
|
||||
session
|
||||
.next_batch()
|
||||
.map_err(map_import_error)?
|
||||
.map(|batch| serde_json::to_string(&batch).map_err(|err| Error::new(Status::GenericFailure, err.to_string())))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn cancel_import_session(session_id: String) -> Result<()> {
|
||||
let mut sessions = IMPORT_SESSIONS
|
||||
.lock()
|
||||
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?;
|
||||
let session = sessions
|
||||
.get_mut(&session_id)
|
||||
.ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?;
|
||||
session.cancel();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn dispose_import_session(session_id: String) -> Result<()> {
|
||||
IMPORT_SESSIONS
|
||||
.lock()
|
||||
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?
|
||||
.remove(&session_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
fs,
|
||||
io::Write,
|
||||
path::PathBuf,
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
|
||||
use zip::{ZipWriter, write::SimpleFileOptions};
|
||||
|
||||
use super::*;
|
||||
|
||||
static NEXT_TEST_ARCHIVE_ID: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
fn archive_path(entries: &[(&str, &[u8])]) -> PathBuf {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"affine-native-import-{}-{}.zip",
|
||||
std::process::id(),
|
||||
NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let file = std::fs::File::create(&path).unwrap();
|
||||
let mut writer = ZipWriter::new(file);
|
||||
for (path, bytes) in entries {
|
||||
writer.start_file(path, SimpleFileOptions::default()).unwrap();
|
||||
writer.write_all(bytes).unwrap();
|
||||
}
|
||||
writer.finish().unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn directory_path(entries: &[(&str, &[u8])]) -> PathBuf {
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"affine-native-import-dir-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
for (path, bytes) in entries {
|
||||
let path = root.join(path);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, bytes).unwrap();
|
||||
}
|
||||
root
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_session_returns_one_serialized_batch() {
|
||||
let path = archive_path(&[("entry.md", b"entry")]);
|
||||
let id = create_import_session(CreateImportSessionOptions {
|
||||
format: "markdownZip".to_string(),
|
||||
source: CreateImportSessionSource {
|
||||
kind: "filePath".to_string(),
|
||||
path: path.to_string_lossy().to_string(),
|
||||
},
|
||||
batch_limits: None,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let batch = next_import_batch(id.clone()).unwrap().unwrap();
|
||||
assert!(batch.contains("\"docs\""));
|
||||
assert!(next_import_batch(id.clone()).unwrap().is_none());
|
||||
dispose_import_session(id).unwrap();
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_session_cancel_blocks_next_batch() {
|
||||
let path = archive_path(&[("entry.md", b"entry")]);
|
||||
let id = create_import_session(CreateImportSessionOptions {
|
||||
format: "markdownZip".to_string(),
|
||||
source: CreateImportSessionSource {
|
||||
kind: "filePath".to_string(),
|
||||
path: path.to_string_lossy().to_string(),
|
||||
},
|
||||
batch_limits: None,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
cancel_import_session(id.clone()).unwrap();
|
||||
assert!(next_import_batch(id.clone()).is_err());
|
||||
dispose_import_session(id).unwrap();
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_session_accepts_directory_source() {
|
||||
let path = directory_path(&[("entry.md", b"entry")]);
|
||||
let id = create_import_session(CreateImportSessionOptions {
|
||||
format: "obsidian".to_string(),
|
||||
source: CreateImportSessionSource {
|
||||
kind: "directoryPath".to_string(),
|
||||
path: path.to_string_lossy().to_string(),
|
||||
},
|
||||
batch_limits: None,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let batch = next_import_batch(id.clone()).unwrap().unwrap();
|
||||
assert!(batch.contains("\"docs\""));
|
||||
assert!(next_import_batch(id.clone()).unwrap().is_none());
|
||||
dispose_import_session(id).unwrap();
|
||||
let _ = fs::remove_dir_all(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_session_rejects_negative_blob_byte_limit() {
|
||||
let path = archive_path(&[("entry.md", b"entry")]);
|
||||
let result = create_import_session(CreateImportSessionOptions {
|
||||
format: "markdownZip".to_string(),
|
||||
source: CreateImportSessionSource {
|
||||
kind: "filePath".to_string(),
|
||||
path: path.to_string_lossy().to_string(),
|
||||
},
|
||||
batch_limits: Some(CreateImportBatchLimits {
|
||||
max_docs: None,
|
||||
max_blobs: None,
|
||||
max_blob_bytes: Some(-1),
|
||||
}),
|
||||
});
|
||||
|
||||
assert!(result.is_err());
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_session_routes_onenote_format_to_provider() {
|
||||
let path = archive_path(&[("entry.md", b"entry")]);
|
||||
let result = create_import_session(CreateImportSessionOptions {
|
||||
format: "oneNote".to_string(),
|
||||
source: CreateImportSessionSource {
|
||||
kind: "filePath".to_string(),
|
||||
path: path.to_string_lossy().to_string(),
|
||||
},
|
||||
batch_limits: None,
|
||||
});
|
||||
|
||||
let error = result.unwrap_err().to_string();
|
||||
assert!(error.contains("unsupported OneNote source"));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
14
packages/frontend/native/src/lib.rs
Normal file
14
packages/frontend/native/src/lib.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
pub mod hashcash;
|
||||
mod import;
|
||||
#[cfg(not(any(target_os = "android", target_os = "ios")))]
|
||||
pub mod preview;
|
||||
|
||||
#[cfg(not(target_arch = "arm"))]
|
||||
#[global_allocator]
|
||||
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub use affine_media_capture::*;
|
||||
pub use affine_nbstore::*;
|
||||
pub use affine_sqlite_v1::*;
|
||||
pub use import::*;
|
||||
112
packages/frontend/native/src/preview.rs
Normal file
112
packages/frontend/native/src/preview.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use affine_preview::{
|
||||
MermaidRenderOptions as PreviewMermaidRenderOptions, TypstRenderOptions as PreviewTypstRenderOptions,
|
||||
};
|
||||
use napi::{Error, Result};
|
||||
use napi_derive::napi;
|
||||
|
||||
#[napi(object)]
|
||||
pub struct MermaidRenderOptions {
|
||||
pub theme: Option<String>,
|
||||
pub font_family: Option<String>,
|
||||
pub font_size: Option<f64>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct MermaidRenderRequest {
|
||||
pub code: String,
|
||||
pub options: Option<MermaidRenderOptions>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct MermaidRenderResult {
|
||||
pub svg: String,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn render_mermaid_svg(request: MermaidRenderRequest) -> Result<MermaidRenderResult> {
|
||||
let options = request.options.unwrap_or(MermaidRenderOptions {
|
||||
theme: None,
|
||||
font_family: None,
|
||||
font_size: None,
|
||||
});
|
||||
let svg = affine_preview::render_mermaid_svg(
|
||||
&request.code,
|
||||
PreviewMermaidRenderOptions {
|
||||
theme: options.theme,
|
||||
font_family: options.font_family,
|
||||
font_size: options.font_size,
|
||||
},
|
||||
)
|
||||
.map_err(|error| Error::from_reason(error.to_string()))?;
|
||||
|
||||
Ok(MermaidRenderResult { svg })
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct TypstRenderOptions {
|
||||
pub font_urls: Option<Vec<String>>,
|
||||
pub font_dirs: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct TypstRenderRequest {
|
||||
pub code: String,
|
||||
pub options: Option<TypstRenderOptions>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct TypstRenderResult {
|
||||
pub svg: String,
|
||||
}
|
||||
|
||||
fn resolve_local_font_dir(value: &str) -> Option<PathBuf> {
|
||||
let path = if let Some(stripped) = value.strip_prefix("file://") {
|
||||
PathBuf::from(stripped)
|
||||
} else {
|
||||
let path = PathBuf::from(value);
|
||||
if !path.is_absolute() {
|
||||
return None;
|
||||
}
|
||||
path
|
||||
};
|
||||
|
||||
if path.is_dir() {
|
||||
return Some(path);
|
||||
}
|
||||
|
||||
path.parent().map(|parent| parent.to_path_buf())
|
||||
}
|
||||
|
||||
fn resolve_typst_font_dirs(options: &Option<TypstRenderOptions>) -> Vec<PathBuf> {
|
||||
let Some(options) = options.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut font_dirs = options
|
||||
.font_dirs
|
||||
.as_ref()
|
||||
.map(|dirs| dirs.iter().map(PathBuf::from).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Some(font_urls) = options.font_urls.as_ref() {
|
||||
font_dirs.extend(font_urls.iter().filter_map(|url| resolve_local_font_dir(url)));
|
||||
}
|
||||
|
||||
font_dirs
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn render_typst_svg(request: TypstRenderRequest) -> Result<TypstRenderResult> {
|
||||
let font_dirs = resolve_typst_font_dirs(&request.options);
|
||||
let svg = affine_preview::render_typst_svg(
|
||||
&request.code,
|
||||
PreviewTypstRenderOptions {
|
||||
font_dirs,
|
||||
cache_dir: None,
|
||||
},
|
||||
)
|
||||
.map_err(|error| Error::from_reason(error.to_string()))?;
|
||||
Ok(TypstRenderResult { svg })
|
||||
}
|
||||
8
packages/frontend/native/tsconfig.json
Normal file
8
packages/frontend/native/tsconfig.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../../tsconfig.node.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist"
|
||||
},
|
||||
"include": ["index.d.ts", "media-capture-example.ts"],
|
||||
"references": []
|
||||
}
|
||||
Reference in New Issue
Block a user