snapshot of https://github.com/ggml-org/whisper.cpp.git @ da54572229bcf64ba367d96c7ef15770376c4280 (2026-09-15)

examples/CMakeLists.txt

@@ -0,0 +1,127 @@
+# dependencies
+
+find_package(Threads REQUIRED)
+
+# third-party
+
+if (WHISPER_SDL2)
+    # SDL2
+    find_package(SDL2 REQUIRED)
+
+    string(STRIP "${SDL2_LIBRARIES}" SDL2_LIBRARIES)
+
+    message(STATUS "SDL2_INCLUDE_DIRS = ${SDL2_INCLUDE_DIRS}")
+    message(STATUS "SDL2_LIBRARIES    = ${SDL2_LIBRARIES}")
+endif()
+
+# common
+
+set(TARGET common)
+
+unset(COMMON_EXTRA_LIBS)
+
+if (WHISPER_COMMON_FFMPEG)
+    # As of cmake 3.27, there is no official cmake support for FindFFmpeg.
+    # Consequnelty we added a FindFFmpeg.cmake script the cmake subfolder:
+    # whisper.cpp does not need the full ffmpeg libs, just AVFORMAT AVCODEC AVUTIL SWRESAMPLE
+    # libswresample  performs highly optimized audio resampling, rematrixing and sample format conversion operations
+    # libavcodec provides a generic encoding/decoding framework and contains multiple decoders and encoders for audio, video and subtitle streams, and several bitstream filters.
+    # libavformat provides a generic framework for multiplexing and demultiplexing (muxing and demuxing) audio, video and subtitle streams.
+    find_package(FFmpeg REQUIRED)
+
+    if (NOT ${FFMPEG_FOUND})
+        message(FATAL_ERROR "Cannot find ffmpeg libs/headers")
+    endif()
+
+    message(STATUS "Found ffmpeg libs:       ${FFMPEG_LIBRARIES}")
+    message(STATUS "Found ffmpeg headers in: ${FFMPEG_INCLUDE_DIRS}")
+    message(STATUS "ffmpeg definitions:      ${FFMPEG_DEFINITIONS}")
+    message(STATUS "Found avformat           ${AVFORMAT_VERSION}")
+
+    include_directories(${FFMPEG_INCLUDE_DIRS})
+    add_compile_definitions(WHISPER_COMMON_FFMPEG)
+
+    list(APPEND COMMON_EXTRA_LIBS ${FFMPEG_LIBRARIES})
+
+    set(COMMON_SOURCES_FFMPEG ffmpeg-transcode.cpp)
+endif()
+
+
+add_library(${TARGET} STATIC
+    common.h
+    common.cpp
+    common-ggml.h
+    common-ggml.cpp
+    common-whisper.h
+    common-whisper.cpp
+    grammar-parser.h
+    grammar-parser.cpp
+    ${COMMON_SOURCES_FFMPEG}
+    )
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE whisper ${COMMON_EXTRA_LIBS} ${CMAKE_DL_LIBS})
+
+set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
+
+if (WHISPER_SDL2)
+    # common-sdl
+
+    set(TARGET common-sdl)
+
+    add_library(${TARGET} STATIC
+        common-sdl.h
+        common-sdl.cpp
+        )
+
+    include(DefaultTargetOptions)
+
+    target_include_directories(${TARGET} PUBLIC  ${SDL2_INCLUDE_DIRS})
+    target_link_libraries     (${TARGET} PRIVATE ${SDL2_LIBRARIES})
+
+    set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
+endif()
+
+# add json lib
+add_library(json_cpp INTERFACE)
+target_include_directories(json_cpp INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
+
+# examples
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+if (EMSCRIPTEN)
+    add_subdirectory(whisper.wasm)
+    add_subdirectory(stream.wasm)
+    add_subdirectory(command.wasm)
+    add_subdirectory(bench.wasm)
+    add_subdirectory(wchess)
+elseif(CMAKE_JS_VERSION)
+    add_subdirectory(addon.node)
+else()
+    add_subdirectory(cli)
+    add_subdirectory(bench)
+    add_subdirectory(server)
+    add_subdirectory(quantize)
+    add_subdirectory(vad-speech-segments)
+    add_subdirectory(parakeet-cli)
+    add_subdirectory(parakeet-quantize)
+    if (WHISPER_SDL2)
+        add_subdirectory(stream)
+        add_subdirectory(command)
+        add_subdirectory(talk-llama)
+        add_subdirectory(lsp)
+        if (GGML_SYCL)
+            add_subdirectory(sycl)
+        endif()
+    endif (WHISPER_SDL2)
+
+    add_subdirectory(deprecation-warning)
+endif()
+
+if (WHISPER_SDL2)
+    add_subdirectory(wchess)
+endif (WHISPER_SDL2)

examples/addon.node/.gitignore

@@ -0,0 +1,3 @@
+.idea
+node_modules
+build

examples/addon.node/CMakeLists.txt

@@ -0,0 +1,34 @@
+set(TARGET addon.node)
+
+# Base settings
+#==================================================================
+# env var supported by cmake-js
+add_definitions(-DNAPI_VERSION=4)
+include_directories(${CMAKE_JS_INC})
+#==================================================================
+
+add_library(${TARGET} SHARED ${CMAKE_JS_SRC} addon.cpp)
+set_target_properties(${TARGET} PROPERTIES
+    PREFIX ""
+    OUTPUT_NAME "addon"
+    SUFFIX ".node"
+    LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/Release"
+)
+
+# Include N-API wrappers
+#==================================================================
+execute_process(COMMAND node -p "require('node-addon-api').include"
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+        OUTPUT_VARIABLE NODE_ADDON_API_DIR
+        )
+string(REPLACE "\n" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR})
+string(REPLACE "\"" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR})
+target_include_directories(${TARGET} PRIVATE ${NODE_ADDON_API_DIR})
+#==================================================================
+
+target_link_libraries(${TARGET} ${CMAKE_JS_LIB} common whisper ${CMAKE_THREAD_LIBS_INIT})
+
+if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
+    # Generate node.lib
+    execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
+endif()

examples/addon.node/README.md

@@ -0,0 +1,115 @@
+# whisper.cpp Node.js addon
+
+This is an addon demo that can **perform whisper model reasoning in `node` and `electron` environments**, based on [cmake-js](https://github.com/cmake-js/cmake-js).
+It can be used as a reference for using the whisper.cpp project in other node projects.
+
+This addon now supports **Voice Activity Detection (VAD)** for improved transcription performance.
+
+## Install
+
+```shell
+npm install
+```
+
+## Compile
+
+Make sure it is in the project root directory and compiled with make-js.
+
+```shell
+npx cmake-js compile -T addon.node -B Release
+```
+
+For Electron addon and cmake-js options, you can see [cmake-js](https://github.com/cmake-js/cmake-js) and make very few configuration changes.
+
+> Such as appointing special cmake path:
+> ```shell
+> npx cmake-js compile -c 'xxx/cmake' -T addon.node -B Release
+> ```
+
+## Run
+
+### Basic Usage
+
+```shell
+cd examples/addon.node
+
+node index.js --language='language' --model='model-path' --fname_inp='file-path'
+```
+
+### VAD (Voice Activity Detection) Usage
+
+Run the VAD example with performance comparison:
+
+```shell
+node vad-example.js
+```
+
+## Voice Activity Detection (VAD) Support
+
+VAD can significantly improve transcription performance by only processing speech segments, which is especially beneficial for audio files with long periods of silence.
+
+### VAD Model Setup
+
+Before using VAD, download a VAD model:
+
+```shell
+# From the whisper.cpp root directory
+./models/download-vad-model.sh silero-v6.2.0
+```
+
+### VAD Parameters
+
+All VAD parameters are optional and have sensible defaults:
+
+- `vad`: Enable VAD (default: false)
+- `vad_model`: Path to VAD model file (required when VAD enabled)
+- `vad_threshold`: Speech detection threshold 0.0-1.0 (default: 0.5)
+- `vad_min_speech_duration_ms`: Min speech duration in ms (default: 250)
+- `vad_min_silence_duration_ms`: Min silence duration in ms (default: 100)
+- `vad_max_speech_duration_s`: Max speech duration in seconds (default: FLT_MAX)
+- `vad_speech_pad_ms`: Speech padding in ms (default: 30)
+- `vad_samples_overlap`: Sample overlap 0.0-1.0 (default: 0.1)
+
+### JavaScript API Example
+
+```javascript
+const path = require("path");
+const { whisper } = require(path.join(__dirname, "../../build/Release/addon.node"));
+const { promisify } = require("util");
+
+const whisperAsync = promisify(whisper);
+
+// With VAD enabled
+const vadParams = {
+  language: "en",
+  model: path.join(__dirname, "../../models/ggml-base.en.bin"),
+  fname_inp: path.join(__dirname, "../../samples/jfk.wav"),
+  vad: true,
+  vad_model: path.join(__dirname, "../../models/ggml-silero-v6.2.0.bin"),
+  vad_threshold: 0.5,
+  progress_callback: (progress) => console.log(`Progress: ${progress}%`)
+};
+
+whisperAsync(vadParams).then(result => console.log(result));
+```
+
+## Supported Parameters
+
+Both traditional whisper.cpp parameters and new VAD parameters are supported:
+
+- `language`: Language code (e.g., "en", "es", "fr")
+- `model`: Path to whisper model file
+- `fname_inp`: Path to input audio file
+- `use_gpu`: Enable GPU acceleration (default: true)
+- `flash_attn`: Enable flash attention (default: false)
+- `no_prints`: Disable console output (default: false)
+- `no_timestamps`: Disable timestamps (default: false)
+- `detect_language`: Auto-detect language (default: false)
+- `audio_ctx`: Audio context size (default: 0)
+- `max_len`: Maximum segment length (default: 0)
+- `max_context`: Maximum context size (default: -1)
+- `prompt`: Initial prompt for decoder
+- `comma_in_time`: Use comma in timestamps (default: true)
+- `print_progress`: Print progress info (default: false)
+- `progress_callback`: Progress callback function
+- VAD parameters (see above section)

examples/addon.node/__test__/whisper.spec.js

@@ -0,0 +1,133 @@
+const { join } = require('path');
+const { whisper } = require('../../../build/Release/addon.node');
+const { promisify } = require('util');
+
+const whisperAsync = promisify(whisper);
+
+const commonParams = {
+  language: 'en',
+  model: join(__dirname, '../../../models/ggml-base.en.bin'),
+  fname_inp: join(__dirname, '../../../samples/jfk.wav'),
+  use_gpu: true,
+  flash_attn: false,
+  no_prints: true,
+  no_timestamps: false,
+  detect_language: false,
+  audio_ctx: 0,
+  max_len: 0
+};
+
+describe('Whisper.cpp Node.js addon with VAD support', () => {
+  test('Basic whisper transcription without VAD', async () => {
+    const params = {
+      ...commonParams,
+      vad: false
+    };
+
+    const result = await whisperAsync(params);
+    
+    expect(typeof result).toBe('object');
+    expect(Array.isArray(result.transcription)).toBe(true);
+    expect(result.transcription.length).toBeGreaterThan(0);
+    
+    // Check that we got some transcription text
+    const text = result.transcription.map(segment => segment[2]).join(' ');
+    expect(text.length).toBeGreaterThan(0);
+    expect(text.toLowerCase()).toContain('ask not');
+  }, 30000);
+
+  test('VAD parameters validation', async () => {
+    // Test with invalid VAD model - should return empty transcription
+    const invalidParams = {
+      ...commonParams,
+      vad: true,
+      vad_model: 'non-existent-model.bin',
+      vad_threshold: 0.5
+    };
+
+    // This should handle the error gracefully and return empty transcription
+    const result = await whisperAsync(invalidParams);
+    expect(typeof result).toBe('object');
+    expect(Array.isArray(result.transcription)).toBe(true);
+    // When VAD model doesn't exist, it should return empty transcription
+    expect(result.transcription.length).toBe(0);
+  }, 10000);
+
+  test('VAD parameter parsing', async () => {
+    // Test that VAD parameters are properly parsed (even if VAD model doesn't exist)
+    const vadParams = {
+      ...commonParams,
+      vad: false, // Disabled so no model required
+      vad_threshold: 0.7,
+      vad_min_speech_duration_ms: 300,
+      vad_min_silence_duration_ms: 150,
+      vad_max_speech_duration_s: 45.0,
+      vad_speech_pad_ms: 50,
+      vad_samples_overlap: 0.15
+    };
+
+    const result = await whisperAsync(vadParams);
+    
+    expect(typeof result).toBe('object');
+    expect(Array.isArray(result.transcription)).toBe(true);
+  }, 30000);
+
+  test('Progress callback with VAD disabled', async () => {
+    let progressCalled = false;
+    let lastProgress = 0;
+
+    const params = {
+      ...commonParams,
+      vad: false,
+      progress_callback: (progress) => {
+        progressCalled = true;
+        lastProgress = progress;
+        expect(progress).toBeGreaterThanOrEqual(0);
+        expect(progress).toBeLessThanOrEqual(100);
+      }
+    };
+
+    const result = await whisperAsync(params);
+    
+    expect(progressCalled).toBe(true);
+    expect(lastProgress).toBe(100);
+    expect(typeof result).toBe('object');
+  }, 30000);
+
+  test('Language detection without VAD', async () => {
+    const params = {
+      ...commonParams,
+      vad: false,
+      detect_language: true,
+      language: 'auto'
+    };
+
+    const result = await whisperAsync(params);
+    
+    expect(typeof result).toBe('object');
+    expect(typeof result.language).toBe('string');
+    expect(result.language.length).toBeGreaterThan(0);
+  }, 30000);
+
+  test('Basic transcription with all VAD parameters set', async () => {
+    // Test with VAD disabled but all parameters set to ensure no crashes
+    const params = {
+      ...commonParams,
+      vad: false, // Disabled so it works without VAD model
+      vad_model: '', // Empty model path
+      vad_threshold: 0.6,
+      vad_min_speech_duration_ms: 200,
+      vad_min_silence_duration_ms: 80,
+      vad_max_speech_duration_s: 25.0,
+      vad_speech_pad_ms: 40,
+      vad_samples_overlap: 0.08
+    };
+
+    const result = await whisperAsync(params);
+    
+    expect(typeof result).toBe('object');
+    expect(Array.isArray(result.transcription)).toBe(true);
+    expect(result.transcription.length).toBeGreaterThan(0);
+  }, 30000);
+});
+

examples/addon.node/addon.cpp

@@ -0,0 +1,557 @@
+#include "napi.h"
+#include "common.h"
+#include "common-whisper.h"
+
+#include "whisper.h"
+
+#include <string>
+#include <thread>
+#include <vector>
+#include <cmath>
+#include <cstdint>
+#include <cfloat>
+
+struct whisper_params {
+    int32_t n_threads    = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t n_processors = 1;
+    int32_t offset_t_ms  = 0;
+    int32_t offset_n     = 0;
+    int32_t duration_ms  = 0;
+    int32_t max_context  = -1;
+    int32_t max_len      = 0;
+    int32_t best_of      = 5;
+    int32_t beam_size    = -1;
+    int32_t audio_ctx    = 0;
+
+    float word_thold    = 0.01f;
+    float entropy_thold = 2.4f;
+    float logprob_thold = -1.0f;
+
+    bool translate      = false;
+    bool diarize        = false;
+    bool output_txt     = false;
+    bool output_vtt     = false;
+    bool output_srt     = false;
+    bool output_wts     = false;
+    bool output_csv     = false;
+    bool print_special  = false;
+    bool print_colors   = false;
+    bool print_progress = false;
+    bool no_timestamps  = false;
+    bool no_prints      = false;
+    bool detect_language= false;
+    bool use_gpu        = true;
+    bool flash_attn     = false;
+    bool comma_in_time  = true;
+
+    std::string language = "en";
+    std::string prompt;
+    std::string model    = "../../ggml-large.bin";
+
+    std::vector<std::string> fname_inp = {};
+    std::vector<std::string> fname_out = {};
+
+    std::vector<float> pcmf32 = {}; // mono-channel F32 PCM
+
+    // Voice Activity Detection (VAD) parameters
+    bool        vad           = false;
+    std::string vad_model     = "";
+    float       vad_threshold = 0.5f;
+    int         vad_min_speech_duration_ms = 250;
+    int         vad_min_silence_duration_ms = 100;
+    float       vad_max_speech_duration_s = FLT_MAX;
+    int         vad_speech_pad_ms = 30;
+    float       vad_samples_overlap = 0.1f;
+};
+
+struct whisper_print_user_data {
+    const whisper_params * params;
+
+    const std::vector<std::vector<float>> * pcmf32s;
+};
+
+void whisper_print_segment_callback(struct whisper_context * ctx, struct whisper_state * state, int n_new, void * user_data) {
+    const auto & params  = *((whisper_print_user_data *) user_data)->params;
+    const auto & pcmf32s = *((whisper_print_user_data *) user_data)->pcmf32s;
+
+    const int n_segments = whisper_full_n_segments(ctx);
+
+    std::string speaker = "";
+
+    int64_t t0;
+    int64_t t1;
+
+    // print the last n_new segments
+    const int s0 = n_segments - n_new;
+
+    if (s0 == 0) {
+        printf("\n");
+    }
+
+    for (int i = s0; i < n_segments; i++) {
+        if (!params.no_timestamps || params.diarize) {
+            t0 = whisper_full_get_segment_t0(ctx, i);
+            t1 = whisper_full_get_segment_t1(ctx, i);
+        }
+
+        if (!params.no_timestamps && !params.no_prints) {
+            printf("[%s --> %s]  ", to_timestamp(t0).c_str(), to_timestamp(t1).c_str());
+        }
+
+        if (params.diarize && pcmf32s.size() == 2) {
+            const int64_t n_samples = pcmf32s[0].size();
+
+            const int64_t is0 = timestamp_to_sample(t0, n_samples, WHISPER_SAMPLE_RATE);
+            const int64_t is1 = timestamp_to_sample(t1, n_samples, WHISPER_SAMPLE_RATE);
+
+            double energy0 = 0.0f;
+            double energy1 = 0.0f;
+
+            for (int64_t j = is0; j < is1; j++) {
+                energy0 += fabs(pcmf32s[0][j]);
+                energy1 += fabs(pcmf32s[1][j]);
+            }
+
+            if (energy0 > 1.1*energy1) {
+                speaker = "(speaker 0)";
+            } else if (energy1 > 1.1*energy0) {
+                speaker = "(speaker 1)";
+            } else {
+                speaker = "(speaker ?)";
+            }
+
+            //printf("is0 = %lld, is1 = %lld, energy0 = %f, energy1 = %f, %s\n", is0, is1, energy0, energy1, speaker.c_str());
+        }
+
+        // colorful print bug
+        //
+        if (!params.no_prints) {
+            const char * text = whisper_full_get_segment_text(ctx, i);
+            printf("%s%s", speaker.c_str(), text);
+        }
+
+
+        // with timestamps or speakers: each segment on new line
+        if ((!params.no_timestamps || params.diarize) && !params.no_prints) {
+            printf("\n");
+        }
+
+        fflush(stdout);
+    }
+}
+
+void cb_log_disable(enum ggml_log_level, const char *, void *) {}
+
+struct whisper_result {
+    std::vector<std::vector<std::string>> segments;
+    std::string language;
+};
+
+class ProgressWorker : public Napi::AsyncWorker {
+ public:
+    ProgressWorker(Napi::Function& callback, whisper_params params, Napi::Function progress_callback, Napi::Env env)
+        : Napi::AsyncWorker(callback), params(params), env(env) {
+        // Create thread-safe function
+        if (!progress_callback.IsEmpty()) {
+            tsfn = Napi::ThreadSafeFunction::New(
+                env,
+                progress_callback,
+                "Progress Callback",
+                0,
+                1
+            );
+        }
+    }
+
+    ~ProgressWorker() {
+        if (tsfn) {
+            // Make sure to release the thread-safe function on destruction
+            tsfn.Release();
+        }
+    }
+
+    void Execute() override {
+        // Use custom run function with progress callback support
+        run_with_progress(params, result);
+    }
+
+    void OnOK() override {
+        Napi::HandleScope scope(Env());
+
+        if (params.detect_language) {
+            Napi::Object resultObj = Napi::Object::New(Env());
+            resultObj.Set("language", Napi::String::New(Env(), result.language));
+            Callback().Call({Env().Null(), resultObj});
+        }
+
+        Napi::Object returnObj = Napi::Object::New(Env());
+        if (!result.language.empty()) {
+            returnObj.Set("language", Napi::String::New(Env(), result.language));
+        }
+        Napi::Array transcriptionArray = Napi::Array::New(Env(), result.segments.size());
+        for (uint64_t i = 0; i < result.segments.size(); ++i) {
+            Napi::Object tmp = Napi::Array::New(Env(), 3);
+            for (uint64_t j = 0; j < 3; ++j) {
+                tmp[j] = Napi::String::New(Env(), result.segments[i][j]);
+            }
+            transcriptionArray[i] = tmp;
+         }
+         returnObj.Set("transcription", transcriptionArray);
+         Callback().Call({Env().Null(), returnObj});
+    }
+
+    // Progress callback function - using thread-safe function
+    void OnProgress(int progress) {
+        if (tsfn) {
+            // Use thread-safe function to call JavaScript callback
+            auto callback = [progress](Napi::Env env, Napi::Function jsCallback) {
+                jsCallback.Call({Napi::Number::New(env, progress)});
+            };
+
+            tsfn.BlockingCall(callback);
+        }
+    }
+
+ private:
+    whisper_params params;
+    whisper_result result;
+    Napi::Env env;
+    Napi::ThreadSafeFunction tsfn;
+
+    // Custom run function with progress callback support
+    int run_with_progress(whisper_params &params, whisper_result & result) {
+        if (params.no_prints) {
+            whisper_log_set(cb_log_disable, NULL);
+        }
+
+        if (params.fname_inp.empty() && params.pcmf32.empty()) {
+            fprintf(stderr, "error: no input files or audio buffer specified\n");
+            return 2;
+        }
+
+        if (params.language != "auto" && whisper_lang_id(params.language.c_str()) == -1) {
+            fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+            exit(0);
+        }
+
+        // whisper init
+        struct whisper_context_params cparams = whisper_context_default_params();
+        cparams.use_gpu = params.use_gpu;
+        cparams.flash_attn = params.flash_attn;
+        struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
+
+        if (ctx == nullptr) {
+            fprintf(stderr, "error: failed to initialize whisper context\n");
+            return 3;
+        }
+
+        // If params.pcmf32 provides, set params.fname_inp as "buffer"
+        if (!params.pcmf32.empty()) {
+            fprintf(stderr, "info: using audio buffer as input\n");
+            params.fname_inp.clear();
+            params.fname_inp.emplace_back("buffer");
+        }
+
+        for (int f = 0; f < (int) params.fname_inp.size(); ++f) {
+            const auto fname_inp = params.fname_inp[f];
+            const auto fname_out = f < (int)params.fname_out.size() && !params.fname_out[f].empty() ? params.fname_out[f] : params.fname_inp[f];
+
+            std::vector<float> pcmf32; // mono-channel F32 PCM
+            std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM
+
+            // If params.pcmf32 is empty, read input audio file
+            if (params.pcmf32.empty()) {
+                if (!::read_audio_data(fname_inp, pcmf32, pcmf32s, params.diarize)) {
+                    fprintf(stderr, "error: failed to read audio file '%s'\n", fname_inp.c_str());
+                    continue;
+                }
+            } else {
+                pcmf32 = params.pcmf32;
+            }
+
+            // Print system info
+            if (!params.no_prints) {
+                fprintf(stderr, "\n");
+                fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
+                        params.n_threads*params.n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());
+            }
+
+            // Print processing info
+            if (!params.no_prints) {
+                fprintf(stderr, "\n");
+                if (!whisper_is_multilingual(ctx)) {
+                    if (params.language != "en" || params.translate) {
+                        params.language = "en";
+                        params.translate = false;
+                        fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
+                    }
+                }
+                fprintf(stderr, "%s: processing '%s' (%d samples, %.1f sec), %d threads, %d processors, lang = %s, task = %s, timestamps = %d, audio_ctx = %d ...\n",
+                        __func__, fname_inp.c_str(), int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,
+                        params.n_threads, params.n_processors,
+                        params.language.c_str(),
+                        params.translate ? "translate" : "transcribe",
+                        params.no_timestamps ? 0 : 1,
+                        params.audio_ctx);
+
+                fprintf(stderr, "\n");
+            }
+
+            // Run inference
+            {
+                whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+
+                wparams.strategy = params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;
+
+                wparams.print_realtime   = false;
+                wparams.print_progress   = params.print_progress;
+                wparams.print_timestamps = !params.no_timestamps;
+                wparams.print_special    = params.print_special;
+                wparams.translate        = params.translate;
+                wparams.language         = params.detect_language ? "auto" : params.language.c_str();
+                wparams.detect_language  = params.detect_language;
+                wparams.n_threads        = params.n_threads;
+                wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;
+                wparams.offset_ms        = params.offset_t_ms;
+                wparams.duration_ms      = params.duration_ms;
+
+                wparams.token_timestamps = params.output_wts || params.max_len > 0;
+                wparams.thold_pt         = params.word_thold;
+                wparams.entropy_thold    = params.entropy_thold;
+                wparams.logprob_thold    = params.logprob_thold;
+                wparams.max_len          = params.output_wts && params.max_len == 0 ? 60 : params.max_len;
+                wparams.audio_ctx        = params.audio_ctx;
+
+                wparams.greedy.best_of        = params.best_of;
+                wparams.beam_search.beam_size = params.beam_size;
+
+                wparams.initial_prompt   = params.prompt.c_str();
+
+                wparams.no_timestamps    = params.no_timestamps;
+
+                whisper_print_user_data user_data = { &params, &pcmf32s };
+
+                // This callback is called for each new segment
+                if (!wparams.print_realtime) {
+                    wparams.new_segment_callback           = whisper_print_segment_callback;
+                    wparams.new_segment_callback_user_data = &user_data;
+                }
+
+                // Set progress callback
+                wparams.progress_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) {
+                    ProgressWorker* worker = static_cast<ProgressWorker*>(user_data);
+                    worker->OnProgress(progress);
+                };
+                wparams.progress_callback_user_data = this;
+
+                // Set VAD parameters
+                wparams.vad            = params.vad;
+                wparams.vad_model_path = params.vad_model.c_str();
+
+                wparams.vad_params.threshold               = params.vad_threshold;
+                wparams.vad_params.min_speech_duration_ms  = params.vad_min_speech_duration_ms;
+                wparams.vad_params.min_silence_duration_ms = params.vad_min_silence_duration_ms;
+                wparams.vad_params.max_speech_duration_s   = params.vad_max_speech_duration_s;
+                wparams.vad_params.speech_pad_ms           = params.vad_speech_pad_ms;
+                wparams.vad_params.samples_overlap         = params.vad_samples_overlap;
+
+                if (whisper_full_parallel(ctx, wparams, pcmf32.data(), pcmf32.size(), params.n_processors) != 0) {
+                    fprintf(stderr, "failed to process audio\n");
+                    return 10;
+                }
+            }
+        }
+
+        if (params.detect_language || params.language == "auto") {
+            result.language = whisper_lang_str(whisper_full_lang_id(ctx));
+        }
+        const int n_segments = whisper_full_n_segments(ctx);
+        result.segments.resize(n_segments);
+
+        for (int i = 0; i < n_segments; ++i) {
+            const char * text = whisper_full_get_segment_text(ctx, i);
+            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);
+            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);
+
+            result.segments[i].emplace_back(to_timestamp(t0, params.comma_in_time));
+            result.segments[i].emplace_back(to_timestamp(t1, params.comma_in_time));
+            result.segments[i].emplace_back(text);
+        }
+
+        whisper_print_timings(ctx);
+        whisper_free(ctx);
+
+        return 0;
+    }
+};
+
+Napi::Value whisper(const Napi::CallbackInfo& info) {
+  Napi::Env env = info.Env();
+  if (info.Length() <= 0 || !info[0].IsObject()) {
+    Napi::TypeError::New(env, "object expected").ThrowAsJavaScriptException();
+  }
+  whisper_params params;
+
+  Napi::Object whisper_params = info[0].As<Napi::Object>();
+  std::string language = whisper_params.Get("language").As<Napi::String>();
+  std::string model = whisper_params.Get("model").As<Napi::String>();
+  std::string input = whisper_params.Get("fname_inp").As<Napi::String>();
+
+  bool use_gpu = true;
+  if (whisper_params.Has("use_gpu") && whisper_params.Get("use_gpu").IsBoolean()) {
+    use_gpu = whisper_params.Get("use_gpu").As<Napi::Boolean>();
+  }
+
+  bool flash_attn = false;
+  if (whisper_params.Has("flash_attn") && whisper_params.Get("flash_attn").IsBoolean()) {
+    flash_attn = whisper_params.Get("flash_attn").As<Napi::Boolean>();
+  }
+
+  bool no_prints = false;
+  if (whisper_params.Has("no_prints") && whisper_params.Get("no_prints").IsBoolean()) {
+    no_prints = whisper_params.Get("no_prints").As<Napi::Boolean>();
+  }
+
+  bool no_timestamps = false;
+  if (whisper_params.Has("no_timestamps") && whisper_params.Get("no_timestamps").IsBoolean()) {
+    no_timestamps = whisper_params.Get("no_timestamps").As<Napi::Boolean>();
+  }
+
+  bool detect_language = false;
+  if (whisper_params.Has("detect_language") && whisper_params.Get("detect_language").IsBoolean()) {
+    detect_language = whisper_params.Get("detect_language").As<Napi::Boolean>();
+  }
+
+  int32_t audio_ctx = 0;
+  if (whisper_params.Has("audio_ctx") && whisper_params.Get("audio_ctx").IsNumber()) {
+    audio_ctx = whisper_params.Get("audio_ctx").As<Napi::Number>();
+  }
+
+  bool comma_in_time = true;
+  if (whisper_params.Has("comma_in_time") && whisper_params.Get("comma_in_time").IsBoolean()) {
+    comma_in_time = whisper_params.Get("comma_in_time").As<Napi::Boolean>();
+  }
+
+  int32_t max_len = 0;
+  if (whisper_params.Has("max_len") && whisper_params.Get("max_len").IsNumber()) {
+    max_len = whisper_params.Get("max_len").As<Napi::Number>();
+  }
+
+  // Add support for max_context
+  int32_t max_context = -1;
+  if (whisper_params.Has("max_context") && whisper_params.Get("max_context").IsNumber()) {
+    max_context = whisper_params.Get("max_context").As<Napi::Number>();
+  }
+
+  // support prompt
+  std::string prompt = "";
+  if (whisper_params.Has("prompt") && whisper_params.Get("prompt").IsString()) {
+    prompt = whisper_params.Get("prompt").As<Napi::String>();
+  }
+
+  // Add support for print_progress
+  bool print_progress = false;
+  if (whisper_params.Has("print_progress") && whisper_params.Get("print_progress").IsBoolean()) {
+    print_progress = whisper_params.Get("print_progress").As<Napi::Boolean>();
+  }
+  // Add support for progress_callback
+  Napi::Function progress_callback;
+  if (whisper_params.Has("progress_callback") && whisper_params.Get("progress_callback").IsFunction()) {
+    progress_callback = whisper_params.Get("progress_callback").As<Napi::Function>();
+  }
+
+  // Add support for VAD parameters
+  bool vad = false;
+  if (whisper_params.Has("vad") && whisper_params.Get("vad").IsBoolean()) {
+    vad = whisper_params.Get("vad").As<Napi::Boolean>();
+  }
+
+  std::string vad_model = "";
+  if (whisper_params.Has("vad_model") && whisper_params.Get("vad_model").IsString()) {
+    vad_model = whisper_params.Get("vad_model").As<Napi::String>();
+  }
+
+  float vad_threshold = 0.5f;
+  if (whisper_params.Has("vad_threshold") && whisper_params.Get("vad_threshold").IsNumber()) {
+    vad_threshold = whisper_params.Get("vad_threshold").As<Napi::Number>();
+  }
+
+  int vad_min_speech_duration_ms = 250;
+  if (whisper_params.Has("vad_min_speech_duration_ms") && whisper_params.Get("vad_min_speech_duration_ms").IsNumber()) {
+    vad_min_speech_duration_ms = whisper_params.Get("vad_min_speech_duration_ms").As<Napi::Number>();
+  }
+
+  int vad_min_silence_duration_ms = 100;
+  if (whisper_params.Has("vad_min_silence_duration_ms") && whisper_params.Get("vad_min_silence_duration_ms").IsNumber()) {
+    vad_min_silence_duration_ms = whisper_params.Get("vad_min_silence_duration_ms").As<Napi::Number>();
+  }
+
+  float vad_max_speech_duration_s = FLT_MAX;
+  if (whisper_params.Has("vad_max_speech_duration_s") && whisper_params.Get("vad_max_speech_duration_s").IsNumber()) {
+    vad_max_speech_duration_s = whisper_params.Get("vad_max_speech_duration_s").As<Napi::Number>();
+  }
+
+  int vad_speech_pad_ms = 30;
+  if (whisper_params.Has("vad_speech_pad_ms") && whisper_params.Get("vad_speech_pad_ms").IsNumber()) {
+    vad_speech_pad_ms = whisper_params.Get("vad_speech_pad_ms").As<Napi::Number>();
+  }
+
+  float vad_samples_overlap = 0.1f;
+  if (whisper_params.Has("vad_samples_overlap") && whisper_params.Get("vad_samples_overlap").IsNumber()) {
+    vad_samples_overlap = whisper_params.Get("vad_samples_overlap").As<Napi::Number>();
+  }
+
+  Napi::Value pcmf32Value = whisper_params.Get("pcmf32");
+  std::vector<float> pcmf32_vec;
+  if (pcmf32Value.IsTypedArray()) {
+    Napi::Float32Array pcmf32 = pcmf32Value.As<Napi::Float32Array>();
+    size_t length = pcmf32.ElementLength();
+    pcmf32_vec.reserve(length);
+    for (size_t i = 0; i < length; i++) {
+      pcmf32_vec.push_back(pcmf32[i]);
+    }
+  }
+
+  params.language = language;
+  params.model = model;
+  params.fname_inp.emplace_back(input);
+  params.use_gpu = use_gpu;
+  params.flash_attn = flash_attn;
+  params.no_prints = no_prints;
+  params.no_timestamps = no_timestamps;
+  params.audio_ctx = audio_ctx;
+  params.pcmf32 = pcmf32_vec;
+  params.comma_in_time = comma_in_time;
+  params.max_len = max_len;
+  params.max_context = max_context;
+  params.print_progress = print_progress;
+  params.prompt = prompt;
+  params.detect_language = detect_language;
+
+  // Set VAD parameters
+  params.vad = vad;
+  params.vad_model = vad_model;
+  params.vad_threshold = vad_threshold;
+  params.vad_min_speech_duration_ms = vad_min_speech_duration_ms;
+  params.vad_min_silence_duration_ms = vad_min_silence_duration_ms;
+  params.vad_max_speech_duration_s = vad_max_speech_duration_s;
+  params.vad_speech_pad_ms = vad_speech_pad_ms;
+  params.vad_samples_overlap = vad_samples_overlap;
+
+  Napi::Function callback = info[1].As<Napi::Function>();
+  // Create a new Worker class with progress callback support
+  ProgressWorker* worker = new ProgressWorker(callback, params, progress_callback, env);
+  worker->Queue();
+  return env.Undefined();
+}
+
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+  exports.Set(
+      Napi::String::New(env, "whisper"),
+      Napi::Function::New(env, whisper)
+  );
+  return exports;
+}
+
+NODE_API_MODULE(whisper, Init);

examples/addon.node/index.js

@@ -0,0 +1,59 @@
+const path = require('path');
+const os = require('os');
+
+const isWindows = os.platform() === 'win32';
+const buildPath = isWindows ? "../../build/bin/Release/addon.node" : "../../build/Release/addon.node";
+
+const { whisper } = require(path.join(__dirname, buildPath));
+const { promisify } = require("util");
+
+const whisperAsync = promisify(whisper);
+
+const whisperParams = {
+  language: "en",
+  model: path.join(__dirname, "../../models/ggml-base.en.bin"),
+  fname_inp: path.join(__dirname, "../../samples/jfk.wav"),
+  use_gpu: true,
+  flash_attn: false,
+  no_prints: true,
+  comma_in_time: false,
+  translate: true,
+  no_timestamps: false,
+  detect_language: false,
+  audio_ctx: 0,
+  max_len: 0,
+  progress_callback: (progress) => {
+      console.log(`progress: ${progress}%`);
+    }
+};
+
+const arguments = process.argv.slice(2);
+const params = Object.fromEntries(
+  arguments.reduce((pre, item) => {
+    if (item.startsWith("--")) {
+      const [key, value] = item.slice(2).split("=");
+      if (key === "audio_ctx") {
+        whisperParams[key] = parseInt(value);
+      } else if (key === "detect_language") {
+        whisperParams[key] = value === "true";
+      } else {
+        whisperParams[key] = value;
+      }
+      return pre;
+    }
+    return pre;
+  }, [])
+);
+
+for (const key in params) {
+  if (whisperParams.hasOwnProperty(key)) {
+    whisperParams[key] = params[key];
+  }
+}
+
+console.log("whisperParams =", whisperParams);
+
+whisperAsync(whisperParams).then((result) => {
+  console.log();
+  console.log(result);
+});

examples/addon.node/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "addon.node",
+  "version": "0.0.0",
+  "description": "",
+  "main": "index.js",
+  "author": "Qanhe Chen",
+  "license": "MIT",
+  "scripts": {
+    "test": "jest"
+  },
+  "devDependencies": {
+    "cmake-js": "^7.1.1",
+    "jest": "^29.4.0",
+    "node-addon-api": "^5.0.0"
+  }
+}

examples/addon.node/vad-example.js

@@ -0,0 +1,132 @@
+const path = require("path");
+const { whisper } = require(path.join(
+  __dirname,
+  "../../build/Release/addon.node"
+));
+const { promisify } = require("util");
+
+const whisperAsync = promisify(whisper);
+
+// Example with VAD enabled
+const vadParams = {
+  language: "en",
+  model: path.join(__dirname, "../../models/ggml-base.en.bin"),
+  fname_inp: path.join(__dirname, "../../samples/jfk.wav"),
+  use_gpu: true,
+  flash_attn: false,
+  no_prints: false,
+  comma_in_time: true,
+  translate: false,
+  no_timestamps: false,
+  detect_language: false,
+  audio_ctx: 0,
+  max_len: 0,
+  // VAD parameters
+  vad: true,
+  vad_model: path.join(__dirname, "../../models/ggml-silero-v6.2.0.bin"), // You need to download this model
+  vad_threshold: 0.5,
+  vad_min_speech_duration_ms: 250,
+  vad_min_silence_duration_ms: 100,
+  vad_max_speech_duration_s: 30.0,
+  vad_speech_pad_ms: 30,
+  vad_samples_overlap: 0.1,
+  progress_callback: (progress) => {
+    console.log(`VAD Transcription progress: ${progress}%`);
+  }
+};
+
+// Example without VAD (traditional approach)
+const traditionalParams = {
+  language: "en",
+  model: path.join(__dirname, "../../models/ggml-base.en.bin"),
+  fname_inp: path.join(__dirname, "../../samples/jfk.wav"),
+  use_gpu: true,
+  flash_attn: false,
+  no_prints: false,
+  comma_in_time: true,
+  translate: false,
+  no_timestamps: false,
+  detect_language: false,
+  audio_ctx: 0,
+  max_len: 0,
+  vad: false, // Explicitly disable VAD
+  progress_callback: (progress) => {
+    console.log(`Traditional transcription progress: ${progress}%`);
+  }
+};
+
+async function runVADExample() {
+  try {
+    console.log("=== Whisper.cpp Node.js VAD Example ===\n");
+    
+    // Check if VAD model exists
+    const fs = require('fs');
+    if (!fs.existsSync(vadParams.vad_model)) {
+      console.log("⚠️  VAD model not found. Please download the VAD model first:");
+      console.log("   ./models/download-vad-model.sh silero-v6.2.0");
+      console.log("   Or run: python models/convert-silero-vad-to-ggml.py");
+      console.log("\n   Falling back to traditional transcription without VAD...\n");
+      
+      // Run without VAD
+      console.log("🎵 Running traditional transcription...");
+      const traditionalResult = await whisperAsync(traditionalParams);
+      console.log("\n📝 Traditional transcription result:");
+      console.log(traditionalResult);
+      return;
+    }
+    
+    console.log("🎵 Running transcription with VAD enabled...");
+    console.log("VAD Parameters:");
+    console.log(`  - Threshold: ${vadParams.vad_threshold}`);
+    console.log(`  - Min speech duration: ${vadParams.vad_min_speech_duration_ms}ms`);
+    console.log(`  - Min silence duration: ${vadParams.vad_min_silence_duration_ms}ms`);
+    console.log(`  - Max speech duration: ${vadParams.vad_max_speech_duration_s}s`);
+    console.log(`  - Speech padding: ${vadParams.vad_speech_pad_ms}ms`);
+    console.log(`  - Samples overlap: ${vadParams.vad_samples_overlap}\n`);
+    
+    const startTime = Date.now();
+    const vadResult = await whisperAsync(vadParams);
+    const vadDuration = Date.now() - startTime;
+    
+    console.log("\n✅ VAD transcription completed!");
+    console.log(`⏱️  Processing time: ${vadDuration}ms`);
+    console.log("\n📝 VAD transcription result:");
+    console.log(vadResult);
+    
+    // Compare with traditional approach
+    console.log("\n🔄 Running traditional transcription for comparison...");
+    const traditionalStartTime = Date.now();
+    const traditionalResult = await whisperAsync(traditionalParams);
+    const traditionalDuration = Date.now() - traditionalStartTime;
+    
+    console.log("\n✅ Traditional transcription completed!");
+    console.log(`⏱️  Processing time: ${traditionalDuration}ms`);
+    console.log("\n📝 Traditional transcription result:");
+    console.log(traditionalResult);
+    
+    // Performance comparison
+    console.log("\n📊 Performance Comparison:");
+    console.log(`VAD:         ${vadDuration}ms`);
+    console.log(`Traditional: ${traditionalDuration}ms`);
+    const speedup = traditionalDuration / vadDuration;
+    if (speedup > 1) {
+      console.log(`🚀 VAD is ${speedup.toFixed(2)}x faster!`);
+    } else {
+      console.log(`ℹ️  Traditional approach was ${(1/speedup).toFixed(2)}x faster in this case.`);
+    }
+    
+  } catch (error) {
+    console.error("❌ Error during transcription:", error);
+  }
+}
+
+// Run the example
+if (require.main === module) {
+  runVADExample();
+}
+
+module.exports = {
+  runVADExample,
+  vadParams,
+  traditionalParams
+}; 

examples/bench.wasm/CMakeLists.txt

@@ -0,0 +1,49 @@
+#
+# libbench
+#
+
+set(TARGET libbench)
+
+add_executable(${TARGET}
+    emscripten.cpp
+    )
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE
+    whisper
+    )
+
+unset(EXTRA_FLAGS)
+
+if (WHISPER_WASM_SINGLE_FILE)
+    set(EXTRA_FLAGS "-s SINGLE_FILE=1")
+    message(STATUS "Embedding WASM inside bench.js")
+
+    add_custom_command(
+        TARGET ${TARGET} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${CMAKE_BINARY_DIR}/bin/libbench.js
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/bench.wasm/bench.js
+        )
+endif()
+
+set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
+    --bind \
+    -s USE_PTHREADS=1 \
+    -s PTHREAD_POOL_SIZE_STRICT=0 \
+    -s INITIAL_MEMORY=2000MB \
+    -s TOTAL_MEMORY=2000MB \
+    -s FORCE_FILESYSTEM=1 \
+    -s EXPORTED_RUNTIME_METHODS=\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\" \
+    ${EXTRA_FLAGS} \
+    ")
+
+#
+# bench.wasm
+#
+
+set(TARGET bench.wasm)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)

examples/bench.wasm/README.md

@@ -0,0 +1,47 @@
+# bench.wasm
+
+Benchmark the performance of whisper.cpp in the browser using WebAssembly
+
+Link: https://ggml.ai/whisper.cpp/bench.wasm/
+
+Terminal version: [examples/bench](/examples/bench)
+
+## Build instructions
+
+```bash
+# build using Emscripten (v3.1.2)
+git clone https://github.com/ggerganov/whisper.cpp
+cd whisper.cpp
+mkdir build-em && cd build-em
+emcmake cmake ..
+make -j
+```
+The example can then be started by running a local HTTP server:
+```console
+python3 examples/server.py
+```
+And then opening a browser to the following URL:
+http://localhost:8000/bench.wasm
+
+To run the example in a different server, you need to copy the following files
+to the server's HTTP path:
+```
+# copy the produced page to your HTTP path
+cp bin/bench.wasm/*       /path/to/html/
+cp bin/libbench.js        /path/to/html/
+cp bin/libbench.worker.js /path/to/html/
+```
+
+> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`
+> which means that that a separate .wasm file will not be generated. Instead, the
+> WASM module is embedded in the main JS file as a base64 encoded string. To
+> generate a separate .wasm file, you need to disable this option by passing
+> `-DWHISPER_WASM_SINGLE_FILE=OFF`:
+> ```console
+> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF
+> ```
+> This will generate a `libbench.wasm` file in the build/bin directory.
+
+> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no
+> longer generated and the worker is embedded in the main JS file. So the worker
+> file will not be generated for versions later than `3.1.58`.

examples/bench.wasm/emscripten.cpp

@@ -0,0 +1,87 @@
+#include "whisper.h"
+
+#include <emscripten.h>
+#include <emscripten/bind.h>
+
+#include <cmath>
+#include <string>
+#include <thread>
+#include <vector>
+
+constexpr int N_THREAD = 8;
+
+// TODO: get rid of this vector of contexts - bad idea in the first place
+std::vector<struct whisper_context *> g_contexts(4, nullptr);
+
+std::thread g_worker;
+
+void bench_main(size_t index) {
+    const int n_threads = std::min(N_THREAD, (int) std::thread::hardware_concurrency());
+
+    // whisper context
+    auto & ctx = g_contexts[index];
+
+    fprintf(stderr, "%s: running benchmark with %d threads - please wait...\n", __func__, n_threads);
+
+    const int n_mels = whisper_model_n_mels(ctx);
+
+    if (int ret = whisper_set_mel(ctx, nullptr, 0, n_mels)) {
+        fprintf(stderr, "error: failed to set mel: %d\n", ret);
+        return;
+    }
+
+    {
+        fprintf(stderr, "\n");
+        fprintf(stderr, "system_info: n_threads = %d / %d | %s\n", n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());
+    }
+
+    if (int ret = whisper_encode(ctx, 0, n_threads) != 0) {
+        fprintf(stderr, "error: failed to encode model: %d\n", ret);
+        return;
+    }
+
+    whisper_print_timings(ctx);
+
+    fprintf(stderr, "\n");
+    fprintf(stderr, "If you wish, you can submit these results here:\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "  https://github.com/ggml-org/whisper.cpp/issues/89\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "Please include the following information:\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "  - CPU model\n");
+    fprintf(stderr, "  - Operating system\n");
+    fprintf(stderr, "  - Browser\n");
+    fprintf(stderr, "\n");
+}
+
+EMSCRIPTEN_BINDINGS(bench) {
+    emscripten::function("init", emscripten::optional_override([](const std::string & path_model) {
+        for (size_t i = 0; i < g_contexts.size(); ++i) {
+            if (g_contexts[i] == nullptr) {
+                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());
+                if (g_contexts[i] != nullptr) {
+                    if (g_worker.joinable()) {
+                        g_worker.join();
+                    }
+                    g_worker = std::thread([i]() {
+                        bench_main(i);
+                    });
+
+                    return i + 1;
+                } else {
+                    return (size_t) 0;
+                }
+            }
+        }
+
+        return (size_t) 0;
+    }));
+
+    emscripten::function("free", emscripten::optional_override([](size_t index) {
+        if (index < g_contexts.size()) {
+            whisper_free(g_contexts[index]);
+            g_contexts[index] = nullptr;
+        }
+    }));
+}

examples/bench.wasm/index-tmpl.html

@@ -0,0 +1,285 @@
+<!doctype html>
+<html lang="en-us">
+    <head>
+        <title>bench : Benchmark whisper.cpp performance in the browser</title>
+
+        <style>
+            #output {
+                width: 100%;
+                height: 100%;
+                margin: 0 auto;
+                margin-top: 10px;
+                border-left: 0px;
+                border-right: 0px;
+                padding-left: 0px;
+                padding-right: 0px;
+                display: block;
+                background-color: black;
+                color: white;
+                font-size: 10px;
+                font-family: 'Lucida Console', Monaco, monospace;
+                outline: none;
+                white-space: pre;
+                overflow-wrap: normal;
+                overflow-x: scroll;
+            }
+        </style>
+        <script src="../coi-serviceworker.js"></script>
+        <link rel="icon" href="data:,">
+    </head>
+    <body>
+        <div id="main-container">
+            <b>bench : Benchmark whisper.cpp performance in the browser</b>
+
+            <br><br>
+
+            You can find more about this project on <a href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/bench.wasm">GitHub</a>.
+
+            <br><br>
+
+            <b>More examples:</b>
+                <a href="../">main</a> |
+                <a href="../bench.wasm/">bench</a> |
+                <a href="../stream.wasm">stream</a> |
+                <a href="../command.wasm/">command</a> |
+                <a href="../wchess.wasm/">wchess</a> |
+
+            <br><br>
+
+            <hr>
+
+            Select the model you would like to use and click the "Bench" button.<br>
+            The results will be displayed in the textarea below.
+
+            <br><br>
+
+            <div id="model-whisper">
+                Whisper model: <span id="model-whisper-status"></span>
+                <button id="fetch-whisper-tiny-en"  onclick="loadWhisper('tiny.en')">tiny.en (75 MB)</button>
+                <button id="fetch-whisper-base-en"  onclick="loadWhisper('base.en')">base.en (142 MB)</button>
+                <button id="fetch-whisper-small-en" onclick="loadWhisper('small.en')">small.en (466 MB)</button>
+                <input type="file" id="whisper-file" name="file" onchange="loadFile(event, 'whisper.bin')" />
+                <br><br>
+                Quantized models:<br><br>
+                <button id="fetch-whisper-tiny-en-q5_1"   onclick="loadWhisper('tiny-en-q5_1')">tiny.en (Q5_1, 31 MB)</button>
+                <button id="fetch-whisper-base-en-q5_1"   onclick="loadWhisper('base-en-q5_1')">base.en (Q5_1, 57 MB)</button>
+                <button id="fetch-whisper-small-en-q5_1"  onclick="loadWhisper('small-en-q5_1')">small.en (Q5_1, 182 MB)</button>
+                <button id="fetch-whisper-medium-en-q5_0" onclick="loadWhisper('medium-en-q5_0')">medium.en (Q5_0, 515 MB)</button>
+                <button id="fetch-whisper-large-q5_0"     onclick="loadWhisper('large-q5_0')">large (Q5_0, 1030 MB)</button>
+                <span id="fetch-whisper-progress"></span>
+            </div>
+
+            <br>
+
+            <div id="input">
+                <button id="bench" onclick="onBench()" disabled>Bench</button>
+                <button id="clear" onclick="clearCache()">Clear Cache</button>
+            </div>
+
+            <hr>
+
+            Debug output:
+            <textarea id="output" rows="20"></textarea>
+
+            <br>
+
+            <b>Troubleshooting</b>
+
+            <br><br>
+
+            The page does some heavy computations, so make sure:
+
+            <ul>
+                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>
+                <li>To use a fast desktop or laptop computer (i.e. not a mobile phone)</li>
+                <li>Your browser supports WASM <a href="https://webassembly.org/roadmap/">Fixed-width SIMD</a></li>
+            </ul>
+
+            <div class="cell-version">
+                <span>
+                    |
+                    Build time: <span class="nav-link">@GIT_DATE@</span> |
+                    Commit hash: <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@">@GIT_SHA1@</a> |
+                    Commit subject: <span class="nav-link">@GIT_COMMIT_SUBJECT@</span> |
+                    <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/bench.wasm">Source Code</a> |
+                </span>
+            </div>
+        </div>
+
+        <script type="text/javascript" src="helpers.js"></script>
+        <script type='text/javascript'>
+            // the bench instance
+            var instance = null;
+
+            // model name
+            var model_whisper = null;
+
+            var Module = {
+                print: printTextarea,
+                printErr: printTextarea,
+                setStatus: function(text) {
+                    printTextarea('js: ' + text);
+                },
+                monitorRunDependencies: function(left) {
+                },
+                preRun: function() {
+                    printTextarea('js: Preparing ...');
+                },
+                postRun: function() {
+                    printTextarea('js: Initialized successfully!');
+                }
+            };
+
+            //
+            // fetch models
+            //
+
+            let dbVersion = 1
+            let dbName    = 'whisper.ggerganov.com';
+            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB
+
+            function storeFS(fname, buf) {
+                // write to WASM file using FS_createDataFile
+                // if the file exists, delete it
+                try {
+                    Module.FS_unlink(fname);
+                } catch (e) {
+                    // ignore
+                }
+
+                Module.FS_createDataFile("/", fname, buf, true, true);
+
+                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);
+
+                model_whisper = fname;
+
+                document.getElementById('model-whisper-status').innerHTML = 'loaded "' + model_whisper + '"!';
+
+                if (model_whisper != null) {
+                    document.getElementById('bench').disabled = false;
+                }
+            }
+
+            function loadFile(event, fname) {
+                var file = event.target.files[0] || null;
+                if (file == null) {
+                    return;
+                }
+
+                printTextarea("loadFile: loading model: " + file.name + ", size: " + file.size + " bytes");
+                printTextarea('loadFile: please wait ...');
+
+                var reader = new FileReader();
+                reader.onload = function(event) {
+                    var buf = new Uint8Array(reader.result);
+                    storeFS(fname, buf);
+                }
+                reader.readAsArrayBuffer(file);
+
+                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';
+                document.getElementById('fetch-whisper-base-en').style.display = 'none';
+                document.getElementById('fetch-whisper-small-en').style.display = 'none';
+
+                document.getElementById('fetch-whisper-tiny-en-q5_1'  ).style.display = 'none';
+                document.getElementById('fetch-whisper-base-en-q5_1'  ).style.display = 'none';
+                document.getElementById('fetch-whisper-small-en-q5_1' ).style.display = 'none';
+                document.getElementById('fetch-whisper-medium-en-q5_0').style.display = 'none';
+                document.getElementById('fetch-whisper-large-q5_0'    ).style.display = 'none';
+
+                document.getElementById('whisper-file'         ).style.display = 'none';
+                document.getElementById('model-whisper-status' ).innerHTML = 'loaded model: ' + file.name;
+            }
+
+            function loadWhisper(model) {
+                let urls = {
+                    'tiny.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',
+                    'base.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',
+                    'small.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin',
+
+                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',
+                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',
+                    'small-en-q5_1': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en-q5_1.bin',
+                    'medium-en-q5_0':'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en-q5_0.bin',
+                    'large-q5_0':    'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-q5_0.bin',
+                };
+
+                let sizes = {
+                    'tiny.en': 75,
+                    'base.en': 142,
+                    'small.en': 466,
+
+                    'tiny-en-q5_1':   31,
+                    'base-en-q5_1':   57,
+                    'small-en-q5_1':  182,
+                    'medium-en-q5_0': 515,
+                    'large-q5_0':     1030,
+                };
+
+                let url     = urls[model];
+                let dst     = 'whisper.bin';
+                let size_mb = sizes[model];
+
+                document.getElementById('fetch-whisper-tiny-en').style.display  = 'none';
+                document.getElementById('fetch-whisper-base-en').style.display  = 'none';
+                document.getElementById('fetch-whisper-small-en').style.display = 'none';
+
+                document.getElementById('fetch-whisper-tiny-en-q5_1'  ).style.display = 'none';
+                document.getElementById('fetch-whisper-base-en-q5_1'  ).style.display = 'none';
+                document.getElementById('fetch-whisper-small-en-q5_1' ).style.display = 'none';
+                document.getElementById('fetch-whisper-medium-en-q5_0').style.display = 'none';
+                document.getElementById('fetch-whisper-large-q5_0'    ).style.display = 'none';
+
+                document.getElementById('whisper-file'        ).style.display = 'none';
+                document.getElementById('model-whisper-status').innerHTML = 'loading "' + model + '" ... ';
+
+                cbProgress = function(p) {
+                    let el = document.getElementById('fetch-whisper-progress');
+                    el.innerHTML = Math.round(100*p) + '%';
+                };
+
+                cbCancel = function() {
+                    var el;
+                    el = document.getElementById('fetch-whisper-tiny-en');  if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base-en');  if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-small-en'); if (el) el.style.display = 'inline-block';
+
+                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'  ); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base-en-q5_1'  ); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-small-en-q5_1' ); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-medium-en-q5_0'); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-large-q5_0'    ); if (el) el.style.display = 'inline-block';
+
+                    el = document.getElementById('whisper-file'        ); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('model-whisper-status'); if (el) el.innerHTML = '';
+                };
+
+                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);
+            }
+
+            //
+            // main
+            //
+
+            function onBench() {
+                if (instance) {
+                    Module.free(instance);
+                }
+
+                instance = Module.init('whisper.bin');
+
+                if (instance) {
+                    printTextarea("js: whisper initialized, instance: " + instance);
+                }
+
+                document.getElementById('bench').disabled = true;
+
+                if (!instance) {
+                    printTextarea("js: failed to initialize whisper");
+                    return;
+                }
+            }
+
+        </script>
+        <script type="text/javascript" src="bench.js"></script>
+    </body>
+</html>

examples/bench/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(TARGET whisper-bench)
+add_executable(${TARGET} bench.cpp)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE whisper ${CMAKE_THREAD_LIBS_INIT})
+
+install(TARGETS ${TARGET} RUNTIME)

examples/bench/README.md

@@ -0,0 +1,51 @@
+# whisper.cpp/examples/bench
+
+A very basic tool for benchmarking the inference performance on your device. The tool simply runs the Encoder part of
+the transformer on some random audio data and records the execution time. This way we can have an objective comparison
+of the performance of the model for various setups.
+
+Benchmark results are tracked in the following Github issue: https://github.com/ggml-org/whisper.cpp/issues/89
+
+```bash
+# run the bench too on the small.en model using 4 threads
+$ ./build/bin/whisper-bench -m ./models/ggml-small.en.bin -t 4
+
+whisper_model_load: loading model from './models/ggml-small.en.bin'
+whisper_model_load: n_vocab       = 51864
+whisper_model_load: n_audio_ctx   = 1500
+whisper_model_load: n_audio_state = 768
+whisper_model_load: n_audio_head  = 12
+whisper_model_load: n_audio_layer = 12
+whisper_model_load: n_text_ctx    = 448
+whisper_model_load: n_text_state  = 768
+whisper_model_load: n_text_head   = 12
+whisper_model_load: n_text_layer  = 12
+whisper_model_load: n_mels        = 80
+whisper_model_load: f16           = 1
+whisper_model_load: type          = 3
+whisper_model_load: mem_required  = 1048.00 MB
+whisper_model_load: adding 1607 extra tokens
+whisper_model_load: ggml ctx size = 533.05 MB
+whisper_model_load: memory size =    68.48 MB 
+whisper_model_load: model size  =   464.44 MB
+
+whisper_print_timings:     load time =   240.82 ms
+whisper_print_timings:      mel time =     0.00 ms
+whisper_print_timings:   sample time =     0.00 ms
+whisper_print_timings:   encode time =  1062.21 ms / 88.52 ms per layer
+whisper_print_timings:   decode time =     0.00 ms / 0.00 ms per layer
+whisper_print_timings:    total time =  1303.04 ms
+
+system_info: n_threads = 4 | AVX2 = 0 | AVX512 = 0 | NEON = 1 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | 
+
+If you wish, you can submit these results here:
+
+  https://github.com/ggml-org/whisper.cpp/issues/89
+
+Please include the following information:
+
+  - CPU model
+  - Operating system
+  - Compiler
+
+```

examples/bench/bench.cpp

@@ -0,0 +1,191 @@
+#include "whisper.h"
+
+#include <cstdio>
+#include <cstring>
+#include <string>
+#include <thread>
+
+// command-line parameters
+struct whisper_params {
+    int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t what = 0; // what to benchmark: 0 - whisper encoder, 1 - memcpy, 2 - ggml_mul_mat
+
+    std::string model = "models/ggml-base.en.bin";
+
+    bool use_gpu    = true;
+    bool flash_attn = true;
+};
+
+void whisper_print_usage(int argc, char ** argv, const whisper_params & params);
+
+static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-h" || arg == "--help") {
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+        else if (arg == "-t"     || arg == "--threads")       { params.n_threads  = std::stoi(argv[++i]); }
+        else if (arg == "-m"     || arg == "--model")         { params.model      = argv[++i]; }
+        else if (arg == "-w"     || arg == "--what")          { params.what       = atoi(argv[++i]); }
+        else if (arg == "-ng"    || arg == "--no-gpu")        { params.use_gpu    = false; }
+        else if (arg == "-fa"    || arg == "--flash-attn")    { params.flash_attn = true; }
+        else if (arg == "-nfa"   || arg == "--no-flash-attn") { params.flash_attn = false; }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,       --help          [default] show this help message and exit\n");
+    fprintf(stderr, "  -t N,     --threads N     [%-7d] number of threads to use during computation\n", params.n_threads);
+    fprintf(stderr, "  -m FNAME, --model FNAME   [%-7s] model path\n",                                  params.model.c_str());
+    fprintf(stderr, "  -w N,     --what N        [%-7d] what to benchmark:\n",                          params.what);
+    fprintf(stderr, "                             %-7s  0 - whisper\n",                                 "");
+    fprintf(stderr, "                             %-7s  1 - memcpy\n",                                  "");
+    fprintf(stderr, "                             %-7s  2 - ggml_mul_mat\n",                            "");
+    fprintf(stderr, "  -ng,      --no-gpu        [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -fa,      --flash-attn    [%-7s] enable flash attention\n",                      params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -nfa,     --no-flash-attn [%-7s] disable flash attention\n",                     params.flash_attn ? "false" : "true");
+    fprintf(stderr, "\n");
+}
+
+static int whisper_bench_full(const whisper_params & params) {
+    // whisper init
+
+    struct whisper_context_params cparams = whisper_context_default_params();
+
+    cparams.use_gpu    = params.use_gpu;
+    cparams.flash_attn = params.flash_attn;
+
+    {
+        fprintf(stderr, "\n");
+        fprintf(stderr, "system_info: n_threads = %d / %d | %s\n", params.n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());
+    }
+
+    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
+    if (ctx == nullptr) {
+        fprintf(stderr, "error: failed to initialize whisper context\n");
+        return 2;
+    }
+
+    const int n_mels = whisper_model_n_mels(ctx);
+
+    if (int ret = whisper_set_mel(ctx, nullptr, 0, n_mels)) {
+        fprintf(stderr, "error: failed to set mel: %d\n", ret);
+        return 3;
+    }
+
+    whisper_token tokens[512];
+    memset(tokens, 0, sizeof(tokens));
+
+    // TODO: need 2 loops because of the current graph capture logic in the CUDA backend
+    //       https://github.com/ggml-org/llama.cpp/pull/19754
+    for (int h = 0; h < 2; ++h) {
+        // heat encoder
+        if (int ret = whisper_encode(ctx, 0, params.n_threads) != 0) {
+            fprintf(stderr, "error: failed to encode: %d\n", ret);
+            return 4;
+        }
+
+        // prompt heat
+        if (int ret = whisper_decode(ctx, tokens, 256, 0, params.n_threads) != 0) {
+            fprintf(stderr, "error: failed to decode: %d\n", ret);
+            return 4;
+        }
+
+        // text-generation heat
+        for (int i = 0; i < 256; i++) {
+            if (int ret = whisper_decode(ctx, tokens, 1, i, params.n_threads) != 0) {
+                fprintf(stderr, "error: failed to decode: %d\n", ret);
+                return 4;
+            }
+        }
+
+        // batched heat
+        if (int ret = whisper_decode(ctx, tokens, 5, 0, params.n_threads) != 0) {
+            fprintf(stderr, "error: failed to decode: %d\n", ret);
+            return 4;
+        }
+    }
+
+    whisper_reset_timings(ctx);
+
+    // actual run
+    if (int ret = whisper_encode(ctx, 0, params.n_threads) != 0) {
+        fprintf(stderr, "error: failed to encode: %d\n", ret);
+        return 4;
+    }
+
+    // text-generation
+    for (int i = 0; i < 256; i++) {
+        if (int ret = whisper_decode(ctx, tokens, 1, i, params.n_threads) != 0) {
+            fprintf(stderr, "error: failed to decode: %d\n", ret);
+            return 4;
+        }
+    }
+
+    // batched decoding
+    for (int i = 0; i < 64; i++) {
+        if (int ret = whisper_decode(ctx, tokens, 5, 0, params.n_threads) != 0) {
+            fprintf(stderr, "error: failed to decode: %d\n", ret);
+            return 4;
+        }
+    }
+
+    // prompt processing
+    for (int i = 0; i < 16; i++) {
+        if (int ret = whisper_decode(ctx, tokens, 256, 0, params.n_threads) != 0) {
+            fprintf(stderr, "error: failed to decode: %d\n", ret);
+            return 4;
+        }
+    }
+
+    whisper_print_timings(ctx);
+    whisper_free(ctx);
+
+    fprintf(stderr, "\n");
+    fprintf(stderr, "If you wish, you can submit these results here:\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "  https://github.com/ggml-org/whisper.cpp/issues/89\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "Please include the following information:\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "  - CPU model\n");
+    fprintf(stderr, "  - Operating system\n");
+    fprintf(stderr, "  - Compiler\n");
+    fprintf(stderr, "\n");
+
+    return 0;
+}
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    whisper_params params;
+
+    if (whisper_params_parse(argc, argv, params) == false) {
+        return 1;
+    }
+
+    int ret = -1;
+
+    switch (params.what) {
+        case 0: ret = whisper_bench_full(params);                break;
+        case 1: ret = whisper_bench_memcpy(params.n_threads);       break;
+        case 2: ret = whisper_bench_ggml_mul_mat(params.n_threads); break;
+        default: fprintf(stderr, "error: unknown benchmark: %d\n", params.what); break;
+    }
+
+    return ret;
+}

examples/cli/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(TARGET whisper-cli)
+add_executable(${TARGET} cli.cpp)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+
+install(TARGETS ${TARGET} RUNTIME)

examples/cli/README.md

@@ -0,0 +1,66 @@
+# whisper.cpp/examples/cli
+
+This is the main example demonstrating most of the functionality of the Whisper model.
+It can be used as a reference for using the `whisper.cpp` library in other projects.
+
+```
+./build/bin/whisper-cli -h
+
+usage: ./build/bin/whisper-cli [options] file0 file1 ...
+supported audio formats: flac, mp3, ogg, wav
+
+options:
+  -h,        --help              [default] show this help message and exit
+  -t N,      --threads N         [4      ] number of threads to use during computation
+  -p N,      --processors N      [1      ] number of processors to use during computation
+  -ot N,     --offset-t N        [0      ] time offset in milliseconds
+  -on N,     --offset-n N        [0      ] segment index offset
+  -d  N,     --duration N        [0      ] duration of audio to process in milliseconds
+  -mc N,     --max-context N     [-1     ] maximum number of text context tokens to store
+  -ml N,     --max-len N         [0      ] maximum segment length in characters
+  -sow,      --split-on-word     [false  ] split on word rather than on token
+  -bo N,     --best-of N         [5      ] number of best candidates to keep
+  -bs N,     --beam-size N       [5      ] beam size for beam search
+  -ac N,     --audio-ctx N       [0      ] audio context size (0 - all)
+  -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold
+  -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail
+  -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail
+  -nth N,    --no-speech-thold N [0.60   ] no speech threshold
+  -tp,       --temperature N     [0.00   ] The sampling temperature, between 0 and 1
+  -tpi,      --temperature-inc N [0.20   ] The increment of temperature, between 0 and 1
+  -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)
+  -tr,       --translate         [false  ] translate from source language to english
+  -di,       --diarize           [false  ] stereo audio diarization
+  -tdrz,     --tinydiarize       [false  ] enable tinydiarize (requires a tdrz model)
+  -nf,       --no-fallback       [false  ] do not use temperature fallback while decoding
+  -otxt,     --output-txt        [false  ] output result in a text file
+  -ovtt,     --output-vtt        [false  ] output result in a vtt file
+  -osrt,     --output-srt        [false  ] output result in a srt file
+  -olrc,     --output-lrc        [false  ] output result in a lrc file
+  -owts,     --output-words      [false  ] output script for generating karaoke video
+  -fp,       --font-path         [/System/Library/Fonts/Supplemental/Courier New Bold.ttf] path to a monospace font for karaoke video
+  -ocsv,     --output-csv        [false  ] output result in a CSV file
+  -oj,       --output-json       [false  ] output result in a JSON file
+  -ojf,      --output-json-full  [false  ] include more information in the JSON file
+  -of FNAME, --output-file FNAME [       ] output file path (without file extension)
+  -np,       --no-prints         [false  ] do not print anything other than the results
+  -ps,       --print-special     [false  ] print special tokens
+  -pc,       --print-colors      [false  ] print colors
+  -pp,       --print-progress    [false  ] print progress
+  -nt,       --no-timestamps     [false  ] do not print timestamps
+  -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
+  -dl,       --detect-language   [false  ] exit after automatically detecting language
+             --prompt PROMPT     [       ] initial prompt (max n_text_ctx/2 tokens)
+  -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
+  -f FNAME,  --file FNAME        [       ] input audio file path
+  -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference
+  -dtw MODEL --dtw MODEL         [       ] compute token-level timestamps
+  -ls,       --log-score         [false  ] log best decoder scores of tokens
+  -ng,       --no-gpu            [false  ] disable GPU
+  -fa,       --flash-attn        [false  ] flash attention
+  -sns,      --suppress-nst      [false  ] suppress non-speech tokens
+  --suppress-regex REGEX         [       ] regular expression matching tokens to suppress
+  --grammar GRAMMAR              [       ] GBNF grammar to guide decoding
+  --grammar-rule RULE            [       ] top-level GBNF grammar rule name
+  --grammar-penalty N            [100.0  ] scales down logits of nongrammar tokens
+```

examples/coi-serviceworker.js

@@ -0,0 +1,146 @@
+/*! coi-serviceworker v0.1.7 - Guido Zuidhof and contributors, licensed under MIT */
+let coepCredentialless = false;
+if (typeof window === 'undefined') {
+    self.addEventListener("install", () => self.skipWaiting());
+    self.addEventListener("activate", (event) => event.waitUntil(self.clients.claim()));
+
+    self.addEventListener("message", (ev) => {
+        if (!ev.data) {
+            return;
+        } else if (ev.data.type === "deregister") {
+            self.registration
+                .unregister()
+                .then(() => {
+                    return self.clients.matchAll();
+                })
+                .then(clients => {
+                    clients.forEach((client) => client.navigate(client.url));
+                });
+        } else if (ev.data.type === "coepCredentialless") {
+            coepCredentialless = ev.data.value;
+        }
+    });
+
+    self.addEventListener("fetch", function (event) {
+        const r = event.request;
+        if (r.cache === "only-if-cached" && r.mode !== "same-origin") {
+            return;
+        }
+
+        const request = (coepCredentialless && r.mode === "no-cors")
+            ? new Request(r, {
+                credentials: "omit",
+            })
+            : r;
+        event.respondWith(
+            fetch(request)
+                .then((response) => {
+                    if (response.status === 0) {
+                        return response;
+                    }
+
+                    const newHeaders = new Headers(response.headers);
+                    newHeaders.set("Cross-Origin-Embedder-Policy",
+                        coepCredentialless ? "credentialless" : "require-corp"
+                    );
+                    if (!coepCredentialless) {
+                        newHeaders.set("Cross-Origin-Resource-Policy", "cross-origin");
+                    }
+                    newHeaders.set("Cross-Origin-Opener-Policy", "same-origin");
+
+                    return new Response(response.body, {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers: newHeaders,
+                    });
+                })
+                .catch((e) => console.error(e))
+        );
+    });
+
+} else {
+    (() => {
+        const reloadedBySelf = window.sessionStorage.getItem("coiReloadedBySelf");
+        window.sessionStorage.removeItem("coiReloadedBySelf");
+        const coepDegrading = (reloadedBySelf == "coepdegrade");
+
+        // You can customize the behavior of this script through a global `coi` variable.
+        const coi = {
+            shouldRegister: () => !reloadedBySelf,
+            shouldDeregister: () => false,
+            coepCredentialless: () => true,
+            coepDegrade: () => true,
+            doReload: () => window.location.reload(),
+            quiet: false,
+            ...window.coi
+        };
+
+        const n = navigator;
+        const controlling = n.serviceWorker && n.serviceWorker.controller;
+
+        // Record the failure if the page is served by serviceWorker.
+        if (controlling && !window.crossOriginIsolated) {
+            window.sessionStorage.setItem("coiCoepHasFailed", "true");
+        }
+        const coepHasFailed = window.sessionStorage.getItem("coiCoepHasFailed");
+
+        if (controlling) {
+            // Reload only on the first failure.
+            const reloadToDegrade = coi.coepDegrade() && !(
+                coepDegrading || window.crossOriginIsolated
+            );
+            n.serviceWorker.controller.postMessage({
+                type: "coepCredentialless",
+                value: (reloadToDegrade || coepHasFailed && coi.coepDegrade())
+                    ? false
+                    : coi.coepCredentialless(),
+            });
+            if (reloadToDegrade) {
+                !coi.quiet && console.log("Reloading page to degrade COEP.");
+                window.sessionStorage.setItem("coiReloadedBySelf", "coepdegrade");
+                coi.doReload("coepdegrade");
+            }
+
+            if (coi.shouldDeregister()) {
+                n.serviceWorker.controller.postMessage({ type: "deregister" });
+            }
+        }
+
+        // If we're already coi: do nothing. Perhaps it's due to this script doing its job, or COOP/COEP are
+        // already set from the origin server. Also if the browser has no notion of crossOriginIsolated, just give up here.
+        if (window.crossOriginIsolated !== false || !coi.shouldRegister()) return;
+
+        if (!window.isSecureContext) {
+            !coi.quiet && console.log("COOP/COEP Service Worker not registered, a secure context is required.");
+            return;
+        }
+
+        // In some environments (e.g. Firefox private mode) this won't be available
+        if (!n.serviceWorker) {
+            !coi.quiet && console.error("COOP/COEP Service Worker not registered, perhaps due to private mode.");
+            return;
+        }
+
+        n.serviceWorker.register(window.document.currentScript.src).then(
+            (registration) => {
+                !coi.quiet && console.log("COOP/COEP Service Worker registered", registration.scope);
+
+                registration.addEventListener("updatefound", () => {
+                    !coi.quiet && console.log("Reloading page to make use of updated COOP/COEP Service Worker.");
+                    window.sessionStorage.setItem("coiReloadedBySelf", "updatefound");
+                    coi.doReload();
+                });
+
+                // If the registration is active, but it's not controlling the page
+                if (registration.active && !n.serviceWorker.controller) {
+                    !coi.quiet && console.log("Reloading page to make use of COOP/COEP Service Worker.");
+                    window.sessionStorage.setItem("coiReloadedBySelf", "notcontrolling");
+                    coi.doReload();
+                }
+            },
+            (err) => {
+                !coi.quiet && console.error("COOP/COEP Service Worker failed to register:", err);
+            }
+        );
+    })();
+}

examples/command.wasm/CMakeLists.txt

@@ -0,0 +1,50 @@
+#
+# libcommand
+#
+
+set(TARGET libcommand)
+
+add_executable(${TARGET}
+    emscripten.cpp
+    )
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE
+    common
+    whisper
+    )
+
+unset(EXTRA_FLAGS)
+
+if (WHISPER_WASM_SINGLE_FILE)
+    set(EXTRA_FLAGS "-s SINGLE_FILE=1")
+    message(STATUS "Embedding WASM inside command.js")
+
+    add_custom_command(
+        TARGET ${TARGET} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${CMAKE_BINARY_DIR}/bin/libcommand.js
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/command.wasm/command.js
+        )
+endif()
+
+set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
+    --bind \
+    -s USE_PTHREADS=1 \
+    -s PTHREAD_POOL_SIZE=8 \
+    -s INITIAL_MEMORY=1024MB \
+    -s TOTAL_MEMORY=1024MB \
+    -s FORCE_FILESYSTEM=1 \
+    -s EXPORTED_RUNTIME_METHODS=\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\" \
+    ${EXTRA_FLAGS} \
+    ")
+
+#
+# command.wasm
+#
+
+set(TARGET command.wasm)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)

examples/command.wasm/README.md

@@ -0,0 +1,47 @@
+# command.wasm
+
+This is a basic Voice Assistant example that accepts voice commands from the microphone.
+It runs in fully in the browser via WebAseembly.
+
+Online demo: https://ggml.ai/whisper.cpp/command.wasm/
+
+Terminal version: [examples/command](/examples/command)
+
+## Build instructions
+
+```bash
+# build using Emscripten (v3.1.2)
+git clone https://github.com/ggerganov/whisper.cpp
+cd whisper.cpp
+mkdir build-em && cd build-em
+emcmake cmake ..
+make -j libcommand
+```
+The example can then be started by running a local HTTP server:
+```console
+python3 examples/server.py
+```
+And then opening a browser to the following URL:
+http://localhost:8000/command.wasm/
+
+To run the example in a different server, you need to copy the following files
+to the server's HTTP path:
+```
+cp bin/command.wasm/*       /path/to/html/
+cp bin/libcommand.js        /path/to/html/
+cp bin/libcommand.worker.js /path/to/html/
+```
+
+> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`
+> which means that that a separate .wasm file will not be generated. Instead, the
+> WASM module is embedded in the main JS file as a base64 encoded string. To
+> generate a separate .wasm file, you need to disable this option by passing
+> `-DWHISPER_WASM_SINGLE_FILE=OFF`:
+> ```console
+> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF
+> ```
+> This will generate a `libcommand.wasm` file in the build/bin directory.
+
+> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no
+> longer generated and the worker is embedded in the main JS file. So the worker
+> file will not be generated for versions later than `3.1.58`.

examples/command.wasm/emscripten.cpp

@@ -0,0 +1,327 @@
+#include "ggml.h"
+#include "common.h"
+#include "whisper.h"
+
+#include <emscripten.h>
+#include <emscripten/bind.h>
+
+#include <atomic>
+#include <cmath>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+#include <regex>
+
+constexpr int N_THREAD = 8;
+
+std::vector<struct whisper_context *> g_contexts(4, nullptr);
+
+std::mutex  g_mutex;
+std::thread g_worker;
+
+std::atomic<bool> g_running(false);
+
+std::string g_status        = "";
+std::string g_status_forced = "";
+std::string g_transcribed   = "";
+
+std::vector<float> g_pcmf32;
+
+void command_set_status(const std::string & status) {
+    std::lock_guard<std::mutex> lock(g_mutex);
+    g_status = status;
+}
+
+std::string command_transcribe(whisper_context * ctx, const whisper_full_params & wparams, const std::vector<float> & pcmf32, float & prob, int64_t & t_ms) {
+    const auto t_start = std::chrono::high_resolution_clock::now();
+
+    prob = 0.0f;
+    t_ms = 0;
+
+    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
+        return "";
+    }
+
+    int prob_n = 0;
+    std::string result;
+
+    const int n_segments = whisper_full_n_segments(ctx);
+    for (int i = 0; i < n_segments; ++i) {
+        const char * text = whisper_full_get_segment_text(ctx, i);
+
+        result += text;
+
+        const int n_tokens = whisper_full_n_tokens(ctx, i);
+        for (int j = 0; j < n_tokens; ++j) {
+            const auto token = whisper_full_get_token_data(ctx, i, j);
+
+            prob += token.p;
+            ++prob_n;
+        }
+    }
+
+    if (prob_n > 0) {
+        prob /= prob_n;
+    }
+
+    const auto t_end = std::chrono::high_resolution_clock::now();
+    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();
+
+    return result;
+}
+
+void command_get_audio(int ms, int sample_rate, std::vector<float> & audio) {
+    const int64_t n_samples = (ms * sample_rate) / 1000;
+
+    int64_t n_take = 0;
+    if (n_samples > (int) g_pcmf32.size()) {
+        n_take = g_pcmf32.size();
+    } else {
+        n_take = n_samples;
+    }
+
+    audio.resize(n_take);
+    std::copy(g_pcmf32.end() - n_take, g_pcmf32.end(), audio.begin());
+}
+
+void command_main(size_t index) {
+    command_set_status("loading data ...");
+
+    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);
+
+    wparams.n_threads        = std::min(N_THREAD, (int) std::thread::hardware_concurrency());
+    wparams.offset_ms        = 0;
+    wparams.translate        = false;
+    wparams.no_context       = true;
+    wparams.single_segment   = true;
+    wparams.print_realtime   = false;
+    wparams.print_progress   = false;
+    wparams.print_timestamps = true;
+    wparams.print_special    = false;
+
+    wparams.max_tokens       = 32;
+    wparams.audio_ctx        = 768; // partial encoder context for better performance
+
+    wparams.language         = "en";
+
+    printf("command: using %d threads\n", wparams.n_threads);
+
+    bool have_prompt  = false;
+    bool ask_prompt   = true;
+    bool print_energy = false;
+
+    float prob0 = 0.0f;
+    float prob  = 0.0f;
+
+    std::vector<float> pcmf32_cur;
+    std::vector<float> pcmf32_prompt;
+
+    const std::string k_prompt = "Ok Whisper, start listening for commands.";
+
+    // whisper context
+    auto & ctx = g_contexts[index];
+
+    const int32_t vad_ms     = 2000;
+    const int32_t prompt_ms  = 5000;
+    const int32_t command_ms = 4000;
+
+    const float vad_thold  = 0.1f;
+    const float freq_thold = -1.0f;
+
+    while (g_running) {
+        // delay
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+        if (ask_prompt) {
+            fprintf(stdout, "\n");
+            fprintf(stdout, "%s: Say the following phrase: '%s%s%s'\n", __func__, "\033[1m", k_prompt.c_str(), "\033[0m");
+            fprintf(stdout, "\n");
+
+            {
+                char txt[1024];
+                snprintf(txt, sizeof(txt), "Say the following phrase: '%s'", k_prompt.c_str());
+                command_set_status(txt);
+            }
+
+            ask_prompt = false;
+        }
+
+        int64_t t_ms = 0;
+
+        {
+            command_get_audio(vad_ms, WHISPER_SAMPLE_RATE, pcmf32_cur);
+
+            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, vad_thold, freq_thold, print_energy)) {
+                fprintf(stdout, "%s: Speech detected! Processing ...\n", __func__);
+                command_set_status("Speech detected! Processing ...");
+
+                if (!have_prompt) {
+                    command_get_audio(prompt_ms, WHISPER_SAMPLE_RATE, pcmf32_cur);
+
+                    const auto txt = ::trim(::command_transcribe(ctx, wparams, pcmf32_cur, prob0, t_ms));
+
+                    fprintf(stdout, "%s: Heard '%s%s%s', (t = %d ms)\n", __func__, "\033[1m", txt.c_str(), "\033[0m", (int) t_ms);
+
+                    const float sim = similarity(txt, k_prompt);
+
+                    if (txt.length() < 0.8*k_prompt.length() || txt.length() > 1.2*k_prompt.length() || sim < 0.8f) {
+                        fprintf(stdout, "%s: WARNING: prompt not recognized, try again\n", __func__);
+                        ask_prompt = true;
+                    } else {
+                        fprintf(stdout, "\n");
+                        fprintf(stdout, "%s: The prompt has been recognized!\n", __func__);
+                        fprintf(stdout, "%s: Waiting for voice commands ...\n", __func__);
+                        fprintf(stdout, "\n");
+
+                        {
+                            char txt[1024];
+                            snprintf(txt, sizeof(txt), "Success! Waiting for voice commands ...");
+                            command_set_status(txt);
+                        }
+
+                        // save the audio for the prompt
+                        pcmf32_prompt = pcmf32_cur;
+                        have_prompt = true;
+                    }
+                } else {
+                    command_get_audio(command_ms, WHISPER_SAMPLE_RATE, pcmf32_cur);
+
+                    // prepend the prompt audio
+                    pcmf32_cur.insert(pcmf32_cur.begin(), pcmf32_prompt.begin(), pcmf32_prompt.end());
+
+                    const auto txt = ::trim(::command_transcribe(ctx, wparams, pcmf32_cur, prob, t_ms));
+
+                    prob = 100.0f*(prob - prob0);
+
+                    fprintf(stdout, "%s: heard '%s'\n", __func__, txt.c_str());
+
+                    // find the prompt in the text
+                    float best_sim = 0.0f;
+                    size_t best_len = 0;
+                    for (int n = 0.8*k_prompt.size(); n <= 1.2*k_prompt.size(); ++n) {
+                        const auto prompt = txt.substr(0, n);
+
+                        const float sim = similarity(prompt, k_prompt);
+
+                        //fprintf(stderr, "%s: prompt = '%s', sim = %f\n", __func__, prompt.c_str(), sim);
+
+                        if (sim > best_sim) {
+                            best_sim = sim;
+                            best_len = n;
+                        }
+                    }
+
+                    const std::string command = ::trim(txt.substr(best_len));
+
+                    fprintf(stdout, "%s: Command '%s%s%s', (t = %d ms)\n", __func__, "\033[1m", command.c_str(), "\033[0m", (int) t_ms);
+                    fprintf(stdout, "\n");
+
+                    {
+                        char txt[1024];
+                        snprintf(txt, sizeof(txt), "Command '%s', (t = %d ms)", command.c_str(), (int) t_ms);
+                        command_set_status(txt);
+                    }
+                    {
+                        std::lock_guard<std::mutex> lock(g_mutex);
+                        g_transcribed = command;
+                    }
+                }
+
+                g_pcmf32.clear();
+            }
+        }
+    }
+
+    if (index < g_contexts.size()) {
+        whisper_free(g_contexts[index]);
+        g_contexts[index] = nullptr;
+    }
+}
+
+EMSCRIPTEN_BINDINGS(command) {
+    emscripten::function("init", emscripten::optional_override([](const std::string & path_model) {
+        for (size_t i = 0; i < g_contexts.size(); ++i) {
+            if (g_contexts[i] == nullptr) {
+                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());
+                if (g_contexts[i] != nullptr) {
+                    g_running = true;
+                    if (g_worker.joinable()) {
+                        g_worker.join();
+                    }
+                    g_worker = std::thread([i]() {
+                        command_main(i);
+                    });
+
+                    return i + 1;
+                } else {
+                    return (size_t) 0;
+                }
+            }
+        }
+
+        return (size_t) 0;
+    }));
+
+    emscripten::function("free", emscripten::optional_override([](size_t index) {
+        if (g_running) {
+            g_running = false;
+        }
+    }));
+
+    emscripten::function("set_audio", emscripten::optional_override([](size_t index, const emscripten::val & audio) {
+        --index;
+
+        if (index >= g_contexts.size()) {
+            return -1;
+        }
+
+        if (g_contexts[index] == nullptr) {
+            return -2;
+        }
+
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            const int n = audio["length"].as<int>();
+
+            emscripten::val heap = emscripten::val::module_property("HEAPU8");
+            emscripten::val memory = heap["buffer"];
+
+            g_pcmf32.resize(n);
+
+            emscripten::val memoryView = audio["constructor"].new_(memory, reinterpret_cast<uintptr_t>(g_pcmf32.data()), n);
+            memoryView.call<void>("set", audio);
+        }
+
+        return 0;
+    }));
+
+    emscripten::function("get_transcribed", emscripten::optional_override([]() {
+        std::string transcribed;
+
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            transcribed = std::move(g_transcribed);
+        }
+
+        return transcribed;
+    }));
+
+    emscripten::function("get_status", emscripten::optional_override([]() {
+        std::string status;
+
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            status = g_status_forced.empty() ? g_status : g_status_forced;
+        }
+
+        return status;
+    }));
+
+    emscripten::function("set_status", emscripten::optional_override([](const std::string & status) {
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            g_status_forced = status;
+        }
+    }));
+}

examples/command.wasm/index-tmpl.html

@@ -0,0 +1,415 @@
+<!doctype html>
+<html lang="en-us">
+    <head>
+        <title>command : Voice assistant example using Whisper + WebAssembly</title>
+
+        <style>
+            #output {
+                width: 100%;
+                height: 100%;
+                margin: 0 auto;
+                margin-top: 10px;
+                border-left: 0px;
+                border-right: 0px;
+                padding-left: 0px;
+                padding-right: 0px;
+                display: block;
+                background-color: black;
+                color: white;
+                font-size: 10px;
+                font-family: 'Lucida Console', Monaco, monospace;
+                outline: none;
+                white-space: pre;
+                overflow-wrap: normal;
+                overflow-x: scroll;
+            }
+        </style>
+        <script src="../coi-serviceworker.js"></script>
+        <link rel="icon" href="data:,">
+    </head>
+    <body>
+        <div id="main-container">
+            <b>command : Voice assistant example using Whisper + WebAssembly</b>
+
+            <br><br>
+
+            You can find more about this project on <a href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/command.wasm">GitHub</a>.
+
+            <br><br>
+
+            <b>More examples:</b>
+                <a href="../">main</a> |
+                <a href="../bench.wasm/">bench</a> |
+                <a href="../stream.wasm">stream</a> |
+                <a href="../command.wasm/">command</a> |
+                <a href="../wchess.wasm/">wchess</a> |
+
+            <br><br>
+
+            <hr>
+
+            Select the model you would like to use, click the "Start" button and follow the instructions.
+
+            <br><br>
+
+            <div id="model-whisper">
+                Whisper model: <span id="model-whisper-status"></span>
+                <button id="fetch-whisper-tiny-en" onclick="loadWhisper('tiny.en')">tiny.en (75 MB)</button>
+                <button id="fetch-whisper-base-en" onclick="loadWhisper('base.en')">base.en (142 MB)</button>
+                <br><br>
+                Quantized models:<br><br>
+                <button id="fetch-whisper-tiny-en-q5_1"   onclick="loadWhisper('tiny-en-q5_1')">tiny.en (Q5_1, 31 MB)</button>
+                <button id="fetch-whisper-base-en-q5_1"   onclick="loadWhisper('base-en-q5_1')">base.en (Q5_1, 57 MB)</button>
+                <span id="fetch-whisper-progress"></span>
+
+                <!--
+                    <input type="file" id="file" name="file" onchange="loadFile(event, 'whisper.bin')" />
+                -->
+            </div>
+
+            <br>
+
+            <div id="input">
+                <button id="start" onclick="onStart()" disabled>Start</button>
+                <button id="stop"  onclick="onStop()" disabled>Stop</button>
+                <button id="clear" onclick="clearCache()">Clear Cache</button>
+            </div>
+
+            <br>
+
+            <div id="state">
+                Status: <b><span id="state-status">not started</span></b>
+
+                <pre id="state-transcribed">[The recognized voice commands will be displayed here]</pre>
+            </div>
+
+            <hr>
+
+            Debug output:
+            <textarea id="output" rows="20"></textarea>
+
+            <br>
+
+            <b>Troubleshooting</b>
+
+            <br><br>
+
+            The page does some heavy computations, so make sure:
+
+            <ul>
+                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>
+                <li>To use a fast desktop or laptop computer (i.e. not a mobile phone)</li>
+                <li>Your browser supports WASM <a href="https://webassembly.org/roadmap/">Fixed-width SIMD</a></li>
+            </ul>
+
+            <div class="cell-version">
+                <span>
+                    |
+                    Build time: <span class="nav-link">@GIT_DATE@</span> |
+                    Commit hash: <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@">@GIT_SHA1@</a> |
+                    Commit subject: <span class="nav-link">@GIT_COMMIT_SUBJECT@</span> |
+                    <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/command.wasm">Source Code</a> |
+                </span>
+            </div>
+        </div>
+
+        <script type="text/javascript" src="helpers.js"></script>
+        <script type='text/javascript'>
+            // web audio context
+            var context = null;
+
+            // audio data
+            var audio = null;
+            var audio0 = null;
+
+            // the command instance
+            var instance = null;
+
+            // model name
+            var model_whisper = null;
+
+            var Module = {
+                print: printTextarea,
+                printErr: printTextarea,
+                setStatus: function(text) {
+                    printTextarea('js: ' + text);
+                },
+                monitorRunDependencies: function(left) {
+                },
+                preRun: function() {
+                    printTextarea('js: Preparing ...');
+                },
+                postRun: function() {
+                    printTextarea('js: Initialized successfully!');
+                }
+            };
+
+            //
+            // fetch models
+            //
+
+            let dbVersion = 1
+            let dbName    = 'whisper.ggerganov.com';
+            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB
+
+            function storeFS(fname, buf) {
+                // write to WASM file using FS_createDataFile
+                // if the file exists, delete it
+                try {
+                    Module.FS_unlink(fname);
+                } catch (e) {
+                    // ignore
+                }
+
+                Module.FS_createDataFile("/", fname, buf, true, true);
+
+                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);
+
+                document.getElementById('model-whisper-status').innerHTML = 'loaded "' + model_whisper + '"!';
+
+                if (model_whisper != null) {
+                    document.getElementById('start').disabled = false;
+                    document.getElementById('stop' ).disabled = true;
+                }
+            }
+
+            function loadWhisper(model) {
+                let urls = {
+                    'tiny.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',
+                    'base.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',
+
+                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',
+                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',
+                };
+
+                let sizes = {
+                    'tiny.en': 75,
+                    'base.en': 142,
+
+                    'tiny-en-q5_1':   31,
+                    'base-en-q5_1':   57,
+                };
+
+                let url     = urls[model];
+                let dst     = 'whisper.bin';
+                let size_mb = sizes[model];
+
+                model_whisper = model;
+
+                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';
+                document.getElementById('fetch-whisper-base-en').style.display = 'none';
+
+                document.getElementById('fetch-whisper-tiny-en-q5_1').style.display = 'none';
+                document.getElementById('fetch-whisper-base-en-q5_1').style.display = 'none';
+
+                document.getElementById('model-whisper-status').innerHTML = 'loading "' + model + '" ... ';
+
+                cbProgress = function(p) {
+                    let el = document.getElementById('fetch-whisper-progress');
+                    el.innerHTML = Math.round(100*p) + '%';
+                };
+
+                cbCancel = function() {
+                    var el;
+                    el = document.getElementById('fetch-whisper-tiny-en'); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base-en'); if (el) el.style.display = 'inline-block';
+
+                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base-en-q5_1'); if (el) el.style.display = 'inline-block';
+
+                    el = document.getElementById('model-whisper-status');  if (el) el.innerHTML = '';
+                };
+
+                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);
+            }
+
+            //
+            // microphone
+            //
+
+            const kSampleRate = 16000;
+            const kRestartRecording_s = 120;
+            const kIntervalAudio_ms = 250; // pass the recorded audio to the C++ instance at this rate
+
+            var mediaRecorder = null;
+            var doRecording = false;
+            var startTime = 0;
+
+            window.AudioContext = window.AudioContext || window.webkitAudioContext;
+            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;
+
+            function stopRecording() {
+                Module.set_status("paused");
+                doRecording = false;
+                audio0 = null;
+                audio = null;
+                context = null;
+            }
+
+            function startRecording() {
+                if (!context) {
+                    context = new AudioContext({
+                        sampleRate: kSampleRate,
+                        channelCount: 1,
+                        echoCancellation: false,
+                        autoGainControl:  true,
+                        noiseSuppression: true,
+                    });
+                }
+
+                Module.set_status("");
+
+                document.getElementById('start').disabled = true;
+                document.getElementById('stop').disabled = false;
+
+                doRecording = true;
+                startTime = Date.now();
+
+                var chunks = [];
+                var stream = null;
+
+                navigator.mediaDevices.getUserMedia({audio: true, video: false})
+                    .then(function(s) {
+                        stream = s;
+                        mediaRecorder = new MediaRecorder(stream);
+                        mediaRecorder.ondataavailable = function(e) {
+                            chunks.push(e.data);
+
+                            var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
+                            var reader = new FileReader();
+
+                            reader.onload = function(event) {
+                                var buf = new Uint8Array(reader.result);
+
+                                if (!context) {
+                                    return;
+                                }
+                                context.decodeAudioData(buf.buffer, function(audioBuffer) {
+                                    var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);
+                                    var source = offlineContext.createBufferSource();
+                                    source.buffer = audioBuffer;
+                                    source.connect(offlineContext.destination);
+                                    source.start(0);
+
+                                    offlineContext.startRendering().then(function(renderedBuffer) {
+                                        audio = renderedBuffer.getChannelData(0);
+
+                                        //printTextarea('js: audio recorded, size: ' + audio.length + ', old size: ' + (audio0 == null ? 0 : audio0.length));
+
+                                        var audioAll = new Float32Array(audio0 == null ? audio.length : audio0.length + audio.length);
+                                        if (audio0 != null) {
+                                            audioAll.set(audio0, 0);
+                                        }
+                                        audioAll.set(audio, audio0 == null ? 0 : audio0.length);
+
+                                        if (instance) {
+                                            Module.set_audio(instance, audioAll);
+                                        }
+                                    });
+                                }, function(e) {
+                                    audio = null;
+                                });
+                            }
+
+                            reader.readAsArrayBuffer(blob);
+                        };
+
+                        mediaRecorder.onstop = function(e) {
+                            if (doRecording) {
+                                setTimeout(function() {
+                                    startRecording();
+                                });
+                            }
+                        };
+
+                        mediaRecorder.start(kIntervalAudio_ms);
+                    })
+                    .catch(function(err) {
+                        printTextarea('js: error getting audio stream: ' + err);
+                    });
+
+                var interval = setInterval(function() {
+                    if (!doRecording) {
+                        clearInterval(interval);
+                        mediaRecorder.stop();
+                        stream.getTracks().forEach(function(track) {
+                            track.stop();
+                        });
+
+                        document.getElementById('start').disabled = false;
+                        document.getElementById('stop').disabled  = true;
+
+                        mediaRecorder = null;
+                    }
+
+                    // if audio length is more than kRestartRecording_s seconds, restart recording
+                    if (audio != null && audio.length > kSampleRate*kRestartRecording_s) {
+                        if (doRecording) {
+                            //printTextarea('js: restarting recording');
+
+                            clearInterval(interval);
+                            audio0 = audio;
+                            audio = null;
+                            mediaRecorder.stop();
+                            stream.getTracks().forEach(function(track) {
+                                track.stop();
+                            });
+                        }
+                    }
+                }, 100);
+            }
+
+            //
+            // main
+            //
+
+            var nLines = 0;
+            var intervalUpdate = null;
+            var transcribedAll = '';
+
+            function onStart() {
+                if (!instance) {
+                    instance = Module.init('whisper.bin');
+
+                    if (instance) {
+                        printTextarea("js: whisper initialized, instance: " + instance);
+                    }
+                }
+
+                if (!instance) {
+                    printTextarea("js: failed to initialize whisper");
+                    return;
+                }
+
+                startRecording();
+
+                intervalUpdate = setInterval(function() {
+                    var transcribed = Module.get_transcribed();
+
+                    if (transcribed != null && transcribed.length > 1) {
+                        transcribedAll += transcribed + '<br>';
+                        nLines++;
+
+                        // if more than 10 lines, remove the first line
+                        if (nLines > 10) {
+                            var i = transcribedAll.indexOf('<br>');
+                            if (i > 0) {
+                                transcribedAll = transcribedAll.substring(i + 4);
+                                nLines--;
+                            }
+                        }
+                    }
+
+                    document.getElementById('state-status').innerHTML = Module.get_status();
+                    document.getElementById('state-transcribed').innerHTML = transcribedAll;
+                }, 100);
+            }
+
+            function onStop() {
+                stopRecording();
+            }
+
+        </script>
+        <script type="text/javascript" src="command.js"></script>
+    </body>
+</html>

examples/command/CMakeLists.txt

@@ -0,0 +1,10 @@
+if (WHISPER_SDL2)
+    set(TARGET whisper-command)
+    add_executable(${TARGET} command.cpp)
+
+    include(DefaultTargetOptions)
+
+    target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})
+
+    install(TARGETS ${TARGET} RUNTIME)
+endif ()

examples/command/README.md

@@ -0,0 +1,52 @@
+# whisper.cpp/examples/command
+
+This is a basic Voice Assistant example that accepts voice commands from the microphone.
+More info is available in [issue #171](https://github.com/ggerganov/whisper.cpp/issues/171).
+
+```bash
+# Run with default arguments and small model
+./whisper-command -m ./models/ggml-small.en.bin -t 8
+
+# On Raspberry Pi, use tiny or base models + "-ac 768" for better performance
+./whisper-command -m ./models/ggml-tiny.en.bin -ac 768 -t 3 -c 0
+```
+
+https://user-images.githubusercontent.com/1991296/204038393-2f846eae-c255-4099-a76d-5735c25c49da.mp4
+
+Web version: [examples/command.wasm](/examples/command.wasm)
+
+## Guided mode
+
+"Guided mode" allows you to specify a list of commands (i.e. strings) and the transcription will be guided to classify your command into one from the list. This can be useful in situations where a device is listening only for a small subset of commands.
+
+Initial tests show that this approach might be extremely efficient in terms of performance, since it integrates very well with the "partial Encoder" idea from #137.
+
+```bash
+# Run in guided mode, the list of allowed commands is in commands.txt
+./whisper-command -m ./models/ggml-base.en.bin -cmd ./examples/command/commands.txt
+
+# On Raspberry Pi, in guided mode you can use "-ac 128" for extra performance
+./whisper-command -m ./models/ggml-tiny.en.bin -cmd ./examples/command/commands.txt -ac 128 -t 3 -c 0
+```
+
+https://user-images.githubusercontent.com/1991296/207435352-8fc4ed3f-bde5-4555-9b8b-aeeb76bee969.mp4
+
+
+## Building
+
+The `whisper-command` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
+
+```bash
+# Install SDL2
+# On Debian based linux distributions:
+sudo apt-get install libsdl2-dev
+
+# On Fedora Linux:
+sudo dnf install SDL2 SDL2-devel
+
+# Install SDL2 on Mac OS
+brew install sdl2
+
+cmake -B build -DWHISPER_SDL2=ON
+cmake --build build --config Release
+```

examples/command/command.cpp

@@ -0,0 +1,802 @@
+// Voice assistant example
+//
+// Speak short text commands to the microphone.
+// This program will detect your voice command and convert them to text.
+//
+// ref: https://github.com/ggml-org/whisper.cpp/issues/171
+//
+
+#include "common-sdl.h"
+#include "common.h"
+#include "whisper.h"
+#include "grammar-parser.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <fstream>
+#include <map>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+// command-line parameters
+struct whisper_params {
+    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t prompt_ms  = 5000;
+    int32_t command_ms = 8000;
+    int32_t capture_id = -1;
+    int32_t max_tokens = 32;
+    int32_t audio_ctx  = 0;
+
+    float vad_thold  = 0.6f;
+    float freq_thold = 100.0f;
+
+    float grammar_penalty = 100.0f;
+
+    grammar_parser::parse_state grammar_parsed;
+
+    bool translate     = false;
+    bool print_special = false;
+    bool print_energy  = false;
+    bool no_timestamps = true;
+    bool use_gpu       = true;
+    bool flash_attn    = true;
+
+    std::string language  = "en";
+    std::string model     = "models/ggml-base.en.bin";
+    std::string fname_out;
+    std::string commands;
+    std::string prompt;
+    std::string context;
+    std::string grammar;
+
+    // A regular expression that matches tokens to suppress
+    std::string suppress_regex;
+};
+
+void whisper_print_usage(int argc, char ** argv, const whisper_params & params);
+
+static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-h" || arg == "--help") {
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+        else if (arg == "-t"     || arg == "--threads")       { params.n_threads     = std::stoi(argv[++i]); }
+        else if (arg == "-pms"   || arg == "--prompt-ms")     { params.prompt_ms     = std::stoi(argv[++i]); }
+        else if (arg == "-cms"   || arg == "--command-ms")    { params.command_ms    = std::stoi(argv[++i]); }
+        else if (arg == "-c"     || arg == "--capture")       { params.capture_id    = std::stoi(argv[++i]); }
+        else if (arg == "-mt"    || arg == "--max-tokens")    { params.max_tokens    = std::stoi(argv[++i]); }
+        else if (arg == "-ac"    || arg == "--audio-ctx")     { params.audio_ctx     = std::stoi(argv[++i]); }
+        else if (arg == "-vth"   || arg == "--vad-thold")     { params.vad_thold     = std::stof(argv[++i]); }
+        else if (arg == "-fth"   || arg == "--freq-thold")    { params.freq_thold    = std::stof(argv[++i]); }
+        else if (arg == "-tr"    || arg == "--translate")     { params.translate     = true; }
+        else if (arg == "-ps"    || arg == "--print-special") { params.print_special = true; }
+        else if (arg == "-pe"    || arg == "--print-energy")  { params.print_energy  = true; }
+        else if (arg == "-ng"    || arg == "--no-gpu")        { params.use_gpu       = false; }
+        else if (arg == "-fa"    || arg == "--flash-attn")    { params.flash_attn    = true; }
+        else if (arg == "-nfa"   || arg == "--no-flash-attn") { params.flash_attn    = false; }
+        else if (arg == "-l"     || arg == "--language")      { params.language      = argv[++i]; }
+        else if (arg == "-m"     || arg == "--model")         { params.model         = argv[++i]; }
+        else if (arg == "-f"     || arg == "--file")          { params.fname_out     = argv[++i]; }
+        else if (arg == "-cmd"   || arg == "--commands")      { params.commands      = argv[++i]; }
+        else if (arg == "-p"     || arg == "--prompt")        { params.prompt        = argv[++i]; }
+        else if (arg == "-ctx"   || arg == "--context")       { params.context       = argv[++i]; }
+        else if (                   arg == "--grammar")       { params.grammar       = argv[++i]; }
+        else if (                   arg == "--grammar-penalty") { params.grammar_penalty = std::stof(argv[++i]); }
+        else if (                   arg == "--suppress-regex") { params.suppress_regex = argv[++i]; }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,         --help           [default] show this help message and exit\n");
+    fprintf(stderr, "  -t N,       --threads N      [%-7d] number of threads to use during computation\n", params.n_threads);
+    fprintf(stderr, "  -pms N,     --prompt-ms N    [%-7d] prompt duration in milliseconds\n",             params.prompt_ms);
+    fprintf(stderr, "  -cms N,     --command-ms N   [%-7d] command duration in milliseconds\n",            params.command_ms);
+    fprintf(stderr, "  -c ID,      --capture ID     [%-7d] capture device ID\n",                           params.capture_id);
+    fprintf(stderr, "  -mt N,      --max-tokens N   [%-7d] maximum number of tokens per audio chunk\n",    params.max_tokens);
+    fprintf(stderr, "  -ac N,      --audio-ctx N    [%-7d] audio context size (0 - all)\n",                params.audio_ctx);
+    fprintf(stderr, "  -vth N,     --vad-thold N    [%-7.2f] voice activity detection threshold\n",        params.vad_thold);
+    fprintf(stderr, "  -fth N,     --freq-thold N   [%-7.2f] high-pass frequency cutoff\n",                params.freq_thold);
+    fprintf(stderr, "  -tr,        --translate      [%-7s] translate from source language to english\n",   params.translate ? "true" : "false");
+    fprintf(stderr, "  -ps,        --print-special  [%-7s] print special tokens\n",                        params.print_special ? "true" : "false");
+    fprintf(stderr, "  -pe,        --print-energy   [%-7s] print sound energy (for debugging)\n",          params.print_energy ? "true" : "false");
+    fprintf(stderr, "  -ng,        --no-gpu         [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -fa,        --flash-attn     [%-7s] enbale flash attention\n",                      params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -nfa,       --no-flash-attn  [%-7s] disable flash attention\n",                     params.flash_attn ? "false" : "true");
+    fprintf(stderr, "  -l LANG,    --language LANG  [%-7s] spoken language\n",                             params.language.c_str());
+    fprintf(stderr, "  -m FNAME,   --model FNAME    [%-7s] model path\n",                                  params.model.c_str());
+    fprintf(stderr, "  -f FNAME,   --file FNAME     [%-7s] text output file name\n",                       params.fname_out.c_str());
+    fprintf(stderr, "  -cmd FNAME, --commands FNAME [%-7s] text file with allowed commands\n",             params.commands.c_str());
+    fprintf(stderr, "  -p,         --prompt         [%-7s] the required activation prompt\n",              params.prompt.c_str());
+    fprintf(stderr, "  -ctx,       --context        [%-7s] sample text to help the transcription\n",       params.context.c_str());
+    fprintf(stderr, "  --grammar GRAMMAR            [%-7s] GBNF grammar to guide decoding\n",              params.grammar.c_str());
+    fprintf(stderr, "  --grammar-penalty N          [%-7.1f] scales down logits of nongrammar tokens\n",   params.grammar_penalty);
+    fprintf(stderr, "  --suppress-regex REGEX       [%-7s] regular expression matching tokens to suppress\n", params.suppress_regex.c_str());
+    fprintf(stderr, "\n");
+}
+
+static std::string transcribe(
+                 whisper_context * ctx,
+            const whisper_params & params,
+        const std::vector<float> & pcmf32,
+               const std::string & grammar_rule,
+                           float & logprob_min,
+                           float & logprob_sum,
+                             int & n_tokens,
+                         int64_t & t_ms) {
+    const auto t_start = std::chrono::high_resolution_clock::now();
+
+    logprob_min = 0.0f;
+    logprob_sum = 0.0f;
+    n_tokens    = 0;
+    t_ms = 0;
+
+    //whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH);
+
+    wparams.print_progress   = false;
+    wparams.print_special    = params.print_special;
+    wparams.print_realtime   = false;
+    wparams.print_timestamps = !params.no_timestamps;
+    wparams.translate        = params.translate;
+    wparams.no_context       = true;
+    wparams.no_timestamps    = params.no_timestamps;
+    wparams.single_segment   = true;
+    wparams.max_tokens       = params.max_tokens;
+    wparams.language         = params.language.c_str();
+    wparams.n_threads        = params.n_threads;
+
+    wparams.audio_ctx = params.audio_ctx;
+
+    wparams.temperature     = 0.4f;
+    wparams.temperature_inc = 1.0f;
+    wparams.greedy.best_of  = 5;
+
+    wparams.beam_search.beam_size = 5;
+
+    wparams.initial_prompt = params.context.data();
+
+    wparams.suppress_regex = params.suppress_regex.c_str();
+
+    const auto & grammar_parsed = params.grammar_parsed;
+    auto grammar_rules = grammar_parsed.c_rules();
+
+    if (!params.grammar_parsed.rules.empty() && !grammar_rule.empty()) {
+        if (grammar_parsed.symbol_ids.find(grammar_rule) == grammar_parsed.symbol_ids.end()) {
+            fprintf(stderr, "%s: warning: grammar rule '%s' not found - skipping grammar sampling\n", __func__, grammar_rule.c_str());
+        } else {
+            wparams.grammar_rules   = grammar_rules.data();
+            wparams.n_grammar_rules = grammar_rules.size();
+            wparams.i_start_rule    = grammar_parsed.symbol_ids.at(grammar_rule);
+            wparams.grammar_penalty = params.grammar_penalty;
+        }
+    }
+
+    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
+        return "";
+    }
+
+    std::string result;
+
+    const int n_segments = whisper_full_n_segments(ctx);
+    for (int i = 0; i < n_segments; ++i) {
+        const char * text = whisper_full_get_segment_text(ctx, i);
+
+        result += text;
+
+        const int n = whisper_full_n_tokens(ctx, i);
+        for (int j = 0; j < n; ++j) {
+            const auto token = whisper_full_get_token_data(ctx, i, j);
+
+            if(token.plog > 0.0f) exit(0);
+            logprob_min = std::min(logprob_min, token.plog);
+            logprob_sum += token.plog;
+            ++n_tokens;
+        }
+    }
+
+    const auto t_end = std::chrono::high_resolution_clock::now();
+    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();
+
+    return result;
+}
+
+static std::vector<std::string> read_allowed_commands(const std::string & fname) {
+    std::vector<std::string> allowed_commands;
+
+    std::ifstream ifs(fname);
+    if (!ifs.is_open()) {
+        return allowed_commands;
+    }
+
+    std::string line;
+    while (std::getline(ifs, line)) {
+        line = ::trim(line);
+        if (line.empty()) {
+            continue;
+        }
+
+        std::transform(line.begin(), line.end(),line.begin(), ::tolower);
+        allowed_commands.push_back(std::move(line));
+    }
+
+    return allowed_commands;
+}
+
+static std::vector<std::string> get_words(const std::string &txt) {
+    std::vector<std::string> words;
+
+    std::istringstream iss(txt);
+    std::string word;
+    while (iss >> word) {
+        words.push_back(word);
+    }
+
+    return words;
+}
+
+// command-list mode
+// guide the transcription to match the most likely command from a provided list
+static int process_command_list(struct whisper_context * ctx, audio_async &audio, const whisper_params &params, std::ofstream &fout) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "%s: guided mode\n", __func__);
+
+    std::vector<std::string> allowed_commands = read_allowed_commands(params.commands);
+
+    if (allowed_commands.empty()) {
+        fprintf(stderr, "%s: error: failed to read allowed commands from '%s'\n", __func__, params.commands.c_str());
+        return 2;
+    }
+
+    int max_len = 0;
+
+    std::vector<std::vector<whisper_token>> allowed_tokens;
+
+    for (const auto & cmd : allowed_commands) {
+        whisper_token tokens[1024];
+        allowed_tokens.emplace_back();
+
+        for (int l = 0; l < (int) cmd.size(); ++l) {
+            // NOTE: very important to add the whitespace !
+            //       the reason is that the first decoded token starts with a whitespace too!
+            std::string ss = std::string(" ") + cmd.substr(0, l + 1);
+
+            const int n = whisper_tokenize(ctx, ss.c_str(), tokens, 1024);
+            if (n < 0) {
+                fprintf(stderr, "%s: error: failed to tokenize command '%s'\n", __func__, cmd.c_str());
+                return 3;
+            }
+
+            if (n == 1) {
+                allowed_tokens.back().push_back(tokens[0]);
+            }
+        }
+
+        max_len = std::max(max_len, (int) cmd.size());
+    }
+
+    fprintf(stderr, "%s: allowed commands [ tokens ]:\n", __func__);
+    fprintf(stderr, "\n");
+    for (int i = 0; i < (int) allowed_commands.size(); ++i) {
+        fprintf(stderr, "  - \033[1m%-*s\033[0m = [", max_len, allowed_commands[i].c_str());
+        for (const auto & token : allowed_tokens[i]) {
+            fprintf(stderr, " %5d", token);
+        }
+        fprintf(stderr, " ]\n");
+    }
+
+    std::string k_prompt = "select one from the available words: ";
+    for (int i = 0; i < (int) allowed_commands.size(); ++i) {
+        if (i > 0) {
+            k_prompt += ", ";
+        }
+        k_prompt += allowed_commands[i];
+    }
+    k_prompt += ". selected word: ";
+
+    // tokenize prompt
+    std::vector<whisper_token> k_tokens;
+    {
+        k_tokens.resize(1024);
+        const int n = whisper_tokenize(ctx, k_prompt.c_str(), k_tokens.data(), 1024);
+        if (n < 0) {
+            fprintf(stderr, "%s: error: failed to tokenize prompt '%s'\n", __func__, k_prompt.c_str());
+            return 4;
+        }
+        k_tokens.resize(n);
+    }
+
+    fprintf(stderr, "\n");
+    fprintf(stderr, "%s: prompt: '%s'\n", __func__, k_prompt.c_str());
+    fprintf(stderr, "%s: tokens: [", __func__);
+    for (const auto & token : k_tokens) {
+        fprintf(stderr, " %d", token);
+    }
+    fprintf(stderr, " ]\n");
+
+    fprintf(stderr, "\n");
+    fprintf(stderr, "%s: listening for a command ...\n", __func__);
+    fprintf(stderr, "\n");
+
+    bool is_running  = true;
+
+    std::vector<float> pcmf32_cur;
+    std::vector<float> pcmf32_prompt;
+
+    // main loop
+    while (is_running) {
+        // handle Ctrl + C
+        is_running = sdl_poll_events();
+
+        // delay
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+        audio.get(2000, pcmf32_cur);
+
+        if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {
+            fprintf(stdout, "%s: Speech detected! Processing ...\n", __func__);
+
+            const auto t_start = std::chrono::high_resolution_clock::now();
+
+            whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+
+            wparams.print_progress   = false;
+            wparams.print_special    = params.print_special;
+            wparams.print_realtime   = false;
+            wparams.print_timestamps = !params.no_timestamps;
+            wparams.translate        = params.translate;
+            wparams.no_context       = true;
+            wparams.single_segment   = true;
+            wparams.max_tokens       = 1;
+            wparams.language         = params.language.c_str();
+            wparams.n_threads        = params.n_threads;
+
+            wparams.audio_ctx        = params.audio_ctx;
+
+            wparams.prompt_tokens    = k_tokens.data();
+            wparams.prompt_n_tokens  = k_tokens.size();
+
+            // run the transformer and a single decoding pass
+            if (whisper_full(ctx, wparams, pcmf32_cur.data(), pcmf32_cur.size()) != 0) {
+                fprintf(stderr, "%s: ERROR: whisper_full() failed\n", __func__);
+                break;
+            }
+
+            // estimate command probability
+            // NOTE: not optimal
+            {
+                const auto * logits = whisper_get_logits(ctx);
+
+                std::vector<float> probs(whisper_n_vocab(ctx), 0.0f);
+
+                // compute probs from logits via softmax
+                {
+                    float max = -1e9;
+                    for (int i = 0; i < (int) probs.size(); ++i) {
+                        max = std::max(max, logits[i]);
+                    }
+
+                    float sum = 0.0f;
+                    for (int i = 0; i < (int) probs.size(); ++i) {
+                        probs[i] = expf(logits[i] - max);
+                        sum += probs[i];
+                    }
+
+                    for (int i = 0; i < (int) probs.size(); ++i) {
+                        probs[i] /= sum;
+                    }
+                }
+
+                std::vector<std::pair<float, int>> probs_id;
+
+                double psum = 0.0;
+                for (int i = 0; i < (int) allowed_commands.size(); ++i) {
+                    probs_id.emplace_back(probs[allowed_tokens[i][0]], i);
+                    for (int j = 1; j < (int) allowed_tokens[i].size(); ++j) {
+                        probs_id.back().first += probs[allowed_tokens[i][j]];
+                    }
+                    probs_id.back().first /= allowed_tokens[i].size();
+                    psum += probs_id.back().first;
+                }
+
+                // normalize
+                for (auto & p : probs_id) {
+                    p.first /= psum;
+                }
+
+                // sort descending
+                {
+                    using pair_type = decltype(probs_id)::value_type;
+                    std::sort(probs_id.begin(), probs_id.end(), [](const pair_type & a, const pair_type & b) {
+                        return a.first > b.first;
+                    });
+                }
+
+                // print the commands and the respective probabilities
+                {
+                    fprintf(stdout, "\n");
+                    for (const auto & cmd : probs_id) {
+                        fprintf(stdout, "%s: %s%-*s%s = %f | ", __func__, "\033[1m", max_len, allowed_commands[cmd.second].c_str(), "\033[0m", cmd.first);
+                        for (int token : allowed_tokens[cmd.second]) {
+                            fprintf(stdout, "'%4s' %f ", whisper_token_to_str(ctx, token), probs[token]);
+                        }
+                        fprintf(stdout, "\n");
+                    }
+                }
+
+                // best command
+                {
+                    const auto t_end = std::chrono::high_resolution_clock::now();
+
+                    const float prob = probs_id[0].first;
+                    const int index = probs_id[0].second;
+                    const char * best_command = allowed_commands[index].c_str();
+
+                    fprintf(stdout, "\n");
+                    fprintf(stdout, "%s: detected command: %s%s%s | p = %f | t = %d ms\n", __func__,
+                            "\033[1m", best_command, "\033[0m", prob,
+                            (int) std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count());
+                    fprintf(stdout, "\n");
+                    if (fout.is_open()) {
+                        fout << best_command << std::endl;
+                    }
+                }
+            }
+
+            audio.clear();
+        }
+    }
+
+    return 0;
+}
+
+// always-prompt mode
+// transcribe the voice into text after valid prompt
+static int always_prompt_transcription(struct whisper_context * ctx, audio_async & audio, const whisper_params & params, std::ofstream & fout) {
+    bool is_running = true;
+    bool ask_prompt = true;
+
+    float logprob_min = 0.0f;
+    float logprob_sum = 0.0f;
+    int   n_tokens    = 0;
+
+    std::vector<float> pcmf32_cur;
+
+    const std::string k_prompt = params.prompt;
+
+    const int k_prompt_length = get_words(k_prompt).size();
+
+    fprintf(stderr, "\n");
+    fprintf(stderr, "%s: always-prompt mode\n", __func__);
+
+    // main loop
+    while (is_running) {
+        // handle Ctrl + C
+        is_running = sdl_poll_events();
+
+        // delay
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+        if (ask_prompt) {
+            fprintf(stdout, "\n");
+            fprintf(stdout, "%s: The prompt is: '%s%s%s'\n", __func__, "\033[1m", k_prompt.c_str(), "\033[0m");
+            fprintf(stdout, "\n");
+
+            ask_prompt = false;
+        }
+
+        {
+            audio.get(2000, pcmf32_cur);
+
+            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {
+                fprintf(stdout, "%s: Speech detected! Processing ...\n", __func__);
+
+                int64_t t_ms = 0;
+
+                // detect the commands
+                audio.get(params.command_ms, pcmf32_cur);
+
+                const auto txt = ::trim(::transcribe(ctx, params, pcmf32_cur, "", logprob_min, logprob_sum, n_tokens, t_ms));
+
+                const auto words = get_words(txt);
+
+                std::string prompt;
+                std::string command;
+
+                for (int i = 0; i < (int) words.size(); ++i) {
+                    if (i < k_prompt_length) {
+                        prompt += words[i] + " ";
+                    } else {
+                        command += words[i] + " ";
+                    }
+                }
+
+                const float sim = similarity(prompt, k_prompt);
+
+                //debug
+                //fprintf(stdout, "command size: %i\n", command_length);
+
+                if ((sim > 0.7f) && (command.size() > 0)) {
+                    fprintf(stdout, "%s: Command '%s%s%s', (t = %d ms)\n", __func__, "\033[1m", command.c_str(), "\033[0m", (int) t_ms);
+                    if (fout.is_open()) {
+                        fout << command << std::endl;
+                    }
+                }
+
+                fprintf(stdout, "\n");
+
+                audio.clear();
+            }
+        }
+    }
+
+    return 0;
+}
+
+// general-purpose mode
+// freely transcribe the voice into text
+static int process_general_transcription(struct whisper_context * ctx, audio_async & audio, const whisper_params & params, std::ofstream & fout) {
+    bool is_running  = true;
+    bool have_prompt = false;
+    bool ask_prompt  = true;
+
+    float logprob_min0 = 0.0f;
+    float logprob_min  = 0.0f;
+
+    float logprob_sum0 = 0.0f;
+    float logprob_sum  = 0.0f;
+
+    int n_tokens0 = 0;
+    int n_tokens  = 0;
+
+    std::vector<float> pcmf32_cur;
+    std::vector<float> pcmf32_prompt;
+
+    std::string k_prompt = "Ok Whisper, start listening for commands.";
+    if (!params.prompt.empty()) {
+        k_prompt = params.prompt;
+    }
+
+    fprintf(stderr, "\n");
+    fprintf(stderr, "%s: general-purpose mode\n", __func__);
+
+    // main loop
+    while (is_running) {
+        // handle Ctrl + C
+        is_running = sdl_poll_events();
+
+        // delay
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+        if (ask_prompt) {
+            fprintf(stdout, "\n");
+            fprintf(stdout, "%s: Say the following phrase: '%s%s%s'\n", __func__, "\033[1m", k_prompt.c_str(), "\033[0m");
+            fprintf(stdout, "\n");
+
+            ask_prompt = false;
+        }
+
+        {
+            audio.get(2000, pcmf32_cur);
+
+            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {
+                fprintf(stdout, "%s: Speech detected! Processing ...\n", __func__);
+
+                int64_t t_ms = 0;
+
+                if (!have_prompt) {
+                    // wait for activation phrase
+                    audio.get(params.prompt_ms, pcmf32_cur);
+
+                    const auto txt = ::trim(::transcribe(ctx, params, pcmf32_cur, "prompt", logprob_min0, logprob_sum0, n_tokens0, t_ms));
+
+                    const float p = 100.0f * std::exp(logprob_min0);
+
+                    fprintf(stdout, "%s: Heard '%s%s%s', (t = %d ms, p = %.2f%%)\n", __func__, "\033[1m", txt.c_str(), "\033[0m", (int) t_ms, p);
+
+                    const float sim = similarity(txt, k_prompt);
+
+                    if (txt.length() < 0.8*k_prompt.length() || txt.length() > 1.2*k_prompt.length() || sim < 0.8f) {
+                        fprintf(stdout, "%s: WARNING: prompt not recognized, try again\n", __func__);
+                        ask_prompt = true;
+                    } else {
+                        fprintf(stdout, "\n");
+                        fprintf(stdout, "%s: The prompt has been recognized!\n", __func__);
+                        fprintf(stdout, "%s: Waiting for voice commands ...\n", __func__);
+                        fprintf(stdout, "\n");
+
+                        // save the audio for the prompt
+                        pcmf32_prompt = pcmf32_cur;
+                        have_prompt = true;
+                    }
+                } else {
+                    // we have heard the activation phrase, now detect the commands
+                    audio.get(params.command_ms, pcmf32_cur);
+
+                    //printf("len prompt:  %.4f\n", pcmf32_prompt.size() / (float) WHISPER_SAMPLE_RATE);
+                    //printf("len command: %.4f\n", pcmf32_cur.size() / (float) WHISPER_SAMPLE_RATE);
+
+                    // prepend 3 second of silence
+                    pcmf32_cur.insert(pcmf32_cur.begin(), 3.0f*WHISPER_SAMPLE_RATE, 0.0f);
+
+                    // prepend the prompt audio
+                    pcmf32_cur.insert(pcmf32_cur.begin(), pcmf32_prompt.begin(), pcmf32_prompt.end());
+
+                    const auto txt = ::trim(::transcribe(ctx, params, pcmf32_cur, "root", logprob_min, logprob_sum, n_tokens, t_ms));
+
+                    //const float p = 100.0f * std::exp((logprob - logprob0) / (n_tokens - n_tokens0));
+                    const float p = 100.0f * std::exp(logprob_min);
+
+                    //fprintf(stdout, "%s: heard '%s'\n", __func__, txt.c_str());
+
+                    // find the prompt in the text
+                    float best_sim = 0.0f;
+                    size_t best_len = 0;
+                    for (size_t n = 0.8*k_prompt.size(); n <= 1.2*k_prompt.size(); ++n) {
+                        if (n >= txt.size()) {
+                            break;
+                        }
+
+                        const auto prompt = txt.substr(0, n);
+
+                        const float sim = similarity(prompt, k_prompt);
+
+                        //fprintf(stderr, "%s: prompt = '%s', sim = %f\n", __func__, prompt.c_str(), sim);
+
+                        if (sim > best_sim) {
+                            best_sim = sim;
+                            best_len = n;
+                        }
+                    }
+
+                    fprintf(stdout, "%s:   DEBUG: txt = '%s', prob = %.2f%%\n", __func__, txt.c_str(), p);
+                    if (best_len == 0) {
+                        fprintf(stdout, "%s: WARNING: command not recognized, try again\n", __func__);
+                    } else {
+                        // cut the prompt from the decoded text
+                        const std::string command = ::trim(txt.substr(best_len));
+                        fprintf(stdout, "%s: Command '%s%s%s', (t = %d ms)\n", __func__, "\033[1m", command.c_str(), "\033[0m", (int) t_ms);
+                        if (fout.is_open()) {
+                            fout << command << std::endl;
+                        }
+                    }
+
+                    fprintf(stdout, "\n");
+                }
+
+                audio.clear();
+            }
+        }
+    }
+
+    return 0;
+}
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    whisper_params params;
+
+    if (whisper_params_parse(argc, argv, params) == false) {
+        return 1;
+    }
+
+    if (whisper_lang_id(params.language.c_str()) == -1) {
+        fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+        whisper_print_usage(argc, argv, params);
+        exit(0);
+    }
+
+    // whisper init
+
+    struct whisper_context_params cparams = whisper_context_default_params();
+
+    cparams.use_gpu    = params.use_gpu;
+    cparams.flash_attn = params.flash_attn;
+
+    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
+    if (ctx == nullptr) {
+        fprintf(stderr, "error: failed to initialize whisper context\n");
+        return 2;
+    }
+
+    // print some info about the processing
+    {
+        fprintf(stderr, "\n");
+        if (!whisper_is_multilingual(ctx)) {
+            if (params.language != "en" || params.translate) {
+                params.language = "en";
+                params.translate = false;
+                fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
+            }
+        }
+        fprintf(stderr, "%s: processing, %d threads, lang = %s, task = %s, timestamps = %d ...\n",
+                __func__,
+                params.n_threads,
+                params.language.c_str(),
+                params.translate ? "translate" : "transcribe",
+                params.no_timestamps ? 0 : 1);
+
+        fprintf(stderr, "\n");
+    }
+
+    // init audio
+
+    audio_async audio(30*1000);
+    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {
+        fprintf(stderr, "%s: audio.init() failed!\n", __func__);
+        return 1;
+    }
+
+    audio.resume();
+
+    // wait for 1 second to avoid any buffered noise
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    audio.clear();
+
+    int  ret_val = 0;
+
+    if (!params.grammar.empty()) {
+        auto & grammar = params.grammar_parsed;
+        if (is_file_exist(params.grammar.c_str())) {
+            // read grammar from file
+            std::ifstream ifs(params.grammar.c_str());
+            const std::string txt = std::string((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
+            grammar = grammar_parser::parse(txt.c_str());
+        } else {
+            // read grammar from string
+            grammar = grammar_parser::parse(params.grammar.c_str());
+        }
+
+        // will be empty (default) if there are parse errors
+        if (grammar.rules.empty()) {
+            ret_val = 1;
+        } else {
+            fprintf(stderr, "%s: grammar:\n", __func__);
+            grammar_parser::print_grammar(stderr, grammar);
+            fprintf(stderr, "\n");
+        }
+    }
+
+    std::ofstream fout;
+    if (params.fname_out.length() > 0) {
+        fout.open(params.fname_out);
+        if (!fout.is_open()) {
+            fprintf(stderr, "%s: failed to open output file '%s'!\n", __func__, params.fname_out.c_str());
+            return 1;
+        }
+    }
+
+    if (ret_val == 0) {
+        if (!params.commands.empty()) {
+            ret_val = process_command_list(ctx, audio, params, fout);
+        } else if (!params.prompt.empty() && params.grammar_parsed.rules.empty()) {
+            ret_val = always_prompt_transcription(ctx, audio, params, fout);
+        } else {
+            ret_val = process_general_transcription(ctx, audio, params, fout);
+        }
+    }
+
+    audio.pause();
+
+    whisper_print_timings(ctx);
+    whisper_free(ctx);
+
+    return ret_val;
+}

examples/command/commands.txt

@@ -0,0 +1,9 @@
+enable
+disable
+cat
+dog
+apple
+red
+blue
+green
+lightblue

examples/common-ggml.cpp

@@ -0,0 +1,246 @@
+#include "common-ggml.h"
+
+#include <regex>
+#include <map>
+
+static const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {
+    {"q4_0", GGML_FTYPE_MOSTLY_Q4_0},
+    {"q4_1", GGML_FTYPE_MOSTLY_Q4_1},
+    {"q5_0", GGML_FTYPE_MOSTLY_Q5_0},
+    {"q5_1", GGML_FTYPE_MOSTLY_Q5_1},
+    {"q8_0", GGML_FTYPE_MOSTLY_Q8_0},
+    {"q2_k", GGML_FTYPE_MOSTLY_Q2_K},
+    {"q3_k", GGML_FTYPE_MOSTLY_Q3_K},
+    {"q4_k", GGML_FTYPE_MOSTLY_Q4_K},
+    {"q5_k", GGML_FTYPE_MOSTLY_Q5_K},
+    {"q6_k", GGML_FTYPE_MOSTLY_Q6_K},
+};
+
+void ggml_print_ftypes(FILE * fp) {
+    for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {
+        fprintf(fp, "  type = \"%s\" or %d\n", it->first.c_str(), it->second);
+    }
+}
+
+enum ggml_ftype ggml_parse_ftype(const char * str) {
+    enum ggml_ftype ftype;
+    if (str[0] == 'q') {
+        const auto it = GGML_FTYPE_MAP.find(str);
+        if (it == GGML_FTYPE_MAP.end()) {
+            fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str);
+            return GGML_FTYPE_UNKNOWN;
+        }
+        ftype = it->second;
+    } else {
+        ftype = (enum ggml_ftype) atoi(str);
+    }
+
+    return ftype;
+}
+
+bool ggml_common_quantize_0(
+        std::ifstream & finp,
+        std::ofstream & fout,
+        const ggml_ftype ftype,
+        const std::vector<std::string> & to_quant,
+        const std::vector<std::string> & to_skip) {
+
+    ggml_type qtype = GGML_TYPE_F32;
+
+    switch (ftype) {
+        case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;
+        case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;
+        case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;
+        case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;
+        case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;
+        case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;
+        case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;
+        case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;
+        case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;
+        case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;
+        case GGML_FTYPE_UNKNOWN:
+        case GGML_FTYPE_ALL_F32:
+        case GGML_FTYPE_MOSTLY_F16:
+        case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:
+        case GGML_FTYPE_MOSTLY_IQ2_XXS:
+        case GGML_FTYPE_MOSTLY_IQ2_XS:
+        case GGML_FTYPE_MOSTLY_IQ2_S:
+        case GGML_FTYPE_MOSTLY_IQ3_XXS:
+        case GGML_FTYPE_MOSTLY_IQ3_S:
+        case GGML_FTYPE_MOSTLY_IQ1_S:
+        case GGML_FTYPE_MOSTLY_IQ4_NL:
+        case GGML_FTYPE_MOSTLY_IQ4_XS:
+        case GGML_FTYPE_MOSTLY_IQ1_M:
+        case GGML_FTYPE_MOSTLY_BF16:
+        case GGML_FTYPE_MOSTLY_MXFP4:
+        case GGML_FTYPE_MOSTLY_NVFP4:
+        case GGML_FTYPE_MOSTLY_Q1_0:
+        case GGML_FTYPE_MOSTLY_Q2_0:
+                {
+                    fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
+                    return false;
+                }
+    };
+
+    if (!ggml_is_quantized(qtype)) {
+        fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype));
+        return false;
+    }
+
+    size_t total_size_org = 0;
+    size_t total_size_new = 0;
+
+    std::vector<float> work;
+
+    std::vector<uint8_t>     data_u8;
+    std::vector<ggml_fp16_t> data_f16;
+    std::vector<float>       data_f32;
+
+    while (true) {
+        int32_t n_dims;
+        int32_t length;
+        int32_t ttype;
+
+        finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+        finp.read(reinterpret_cast<char *>(&length), sizeof(length));
+        finp.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+        if (finp.eof()) {
+            break;
+        }
+
+        int32_t nelements = 1;
+        int32_t ne[4] = { 1, 1, 1, 1 };
+        for (int i = 0; i < n_dims; ++i) {
+            finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+            nelements *= ne[i];
+        }
+
+        std::string name(length, 0);
+        finp.read (&name[0], length);
+
+        printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));
+
+        bool quantize = false;
+
+        // check if we should quantize this tensor
+        for (const auto & s : to_quant) {
+            if (std::regex_match(name, std::regex(s))) {
+                quantize = true;
+                break;
+            }
+        }
+
+        // check if we should skip this tensor
+        for (const auto & s : to_skip) {
+            if (std::regex_match(name, std::regex(s))) {
+                quantize = false;
+                break;
+            }
+        }
+
+        // quantize only 2D tensors
+        quantize &= (n_dims == 2);
+
+        if (quantize) {
+            if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {
+                fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
+                return false;
+            }
+
+            if (ttype == GGML_TYPE_F16) {
+                data_f16.resize(nelements);
+                finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));
+                data_f32.resize(nelements);
+                for (int i = 0; i < nelements; ++i) {
+                    data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);
+                }
+            } else {
+                data_f32.resize(nelements);
+                finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));
+            }
+
+            ttype = qtype;
+        } else {
+            const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);
+
+            data_u8.resize(nelements*bpe);
+            finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);
+        }
+
+        fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+        fout.write(reinterpret_cast<char *>(&length), sizeof(length));
+        fout.write(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+        for (int i = 0; i < n_dims; ++i) {
+            fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+        }
+        fout.write(&name[0], length);
+
+        if (quantize) {
+            work.resize(nelements); // for quantization
+
+            size_t cur_size = 0;
+            switch ((ggml_type) ttype) {
+                case GGML_TYPE_Q4_0:
+                case GGML_TYPE_Q4_1:
+                case GGML_TYPE_Q5_0:
+                case GGML_TYPE_Q5_1:
+                case GGML_TYPE_Q8_0:
+                case GGML_TYPE_Q2_K:
+                case GGML_TYPE_Q3_K:
+                case GGML_TYPE_Q4_K:
+                case GGML_TYPE_Q5_K:
+                case GGML_TYPE_Q6_K:
+                    {
+                        cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);
+                    } break;
+                case GGML_TYPE_F32:
+                case GGML_TYPE_F16:
+                case GGML_TYPE_I8:
+                case GGML_TYPE_I16:
+                case GGML_TYPE_I32:
+                case GGML_TYPE_I64:
+                case GGML_TYPE_F64:
+                case GGML_TYPE_Q8_1:
+                case GGML_TYPE_Q8_K:
+                case GGML_TYPE_IQ2_XXS:
+                case GGML_TYPE_IQ2_XS:
+                case GGML_TYPE_IQ2_S:
+                case GGML_TYPE_IQ3_XXS:
+                case GGML_TYPE_IQ3_S:
+                case GGML_TYPE_IQ1_S:
+                case GGML_TYPE_IQ4_NL:
+                case GGML_TYPE_IQ4_XS:
+                case GGML_TYPE_IQ1_M:
+                case GGML_TYPE_BF16:
+                case GGML_TYPE_TQ1_0:
+                case GGML_TYPE_TQ2_0:
+                case GGML_TYPE_MXFP4:
+                case GGML_TYPE_NVFP4:
+                case GGML_TYPE_Q1_0:
+                case GGML_TYPE_Q2_0:
+                case GGML_TYPE_COUNT:
+                    {
+                        fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
+                        return false;
+                    }
+            }
+
+            fout.write(reinterpret_cast<char *>(work.data()), cur_size);
+            total_size_new += cur_size;
+
+            printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);
+        } else {
+            printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0);
+            fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());
+            total_size_new += data_u8.size();
+        }
+
+        total_size_org += nelements * sizeof(float);
+    }
+
+    printf("%s: model size  = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
+    printf("%s: quant size  = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));
+
+    return true;
+}

examples/common-ggml.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "ggml.h"
+
+#include <fstream>
+#include <vector>
+#include <string>
+
+enum ggml_ftype ggml_parse_ftype(const char * str);
+
+void ggml_print_ftypes(FILE * fp = stderr);
+
+bool ggml_common_quantize_0(
+        std::ifstream & finp,
+        std::ofstream & fout,
+        const ggml_ftype ftype,
+        const std::vector<std::string> & to_quant,
+        const std::vector<std::string> & to_skip);

examples/common-sdl.cpp

@@ -0,0 +1,227 @@
+#include "common-sdl.h"
+
+#include <cstdio>
+
+audio_async::audio_async(int len_ms) {
+    m_len_ms = len_ms;
+
+    m_running = false;
+}
+
+audio_async::~audio_async() {
+    if (m_dev_id_in) {
+        SDL_CloseAudioDevice(m_dev_id_in);
+    }
+}
+
+bool audio_async::init(int capture_id, int sample_rate) {
+    SDL_LogSetPriority(SDL_LOG_CATEGORY_APPLICATION, SDL_LOG_PRIORITY_INFO);
+
+    if (SDL_Init(SDL_INIT_AUDIO) < 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't initialize SDL: %s\n", SDL_GetError());
+        return false;
+    }
+
+    SDL_SetHintWithPriority(SDL_HINT_AUDIO_RESAMPLING_MODE, "medium", SDL_HINT_OVERRIDE);
+
+    {
+        int nDevices = SDL_GetNumAudioDevices(SDL_TRUE);
+        fprintf(stderr, "%s: found %d capture devices:\n", __func__, nDevices);
+        for (int i = 0; i < nDevices; i++) {
+            fprintf(stderr, "%s:    - Capture device #%d: '%s'\n", __func__, i, SDL_GetAudioDeviceName(i, SDL_TRUE));
+        }
+    }
+
+    SDL_AudioSpec capture_spec_requested;
+    SDL_AudioSpec capture_spec_obtained;
+
+    SDL_zero(capture_spec_requested);
+    SDL_zero(capture_spec_obtained);
+
+    capture_spec_requested.freq     = sample_rate;
+    capture_spec_requested.format   = AUDIO_F32;
+    capture_spec_requested.channels = 1;
+    capture_spec_requested.samples  = 1024;
+    capture_spec_requested.callback = [](void * userdata, uint8_t * stream, int len) {
+        audio_async * audio = (audio_async *) userdata;
+        audio->callback(stream, len);
+    };
+    capture_spec_requested.userdata = this;
+
+    if (capture_id >= 0) {
+        fprintf(stderr, "%s: attempt to open capture device %d : '%s' ...\n", __func__, capture_id, SDL_GetAudioDeviceName(capture_id, SDL_TRUE));
+        m_dev_id_in = SDL_OpenAudioDevice(SDL_GetAudioDeviceName(capture_id, SDL_TRUE), SDL_TRUE, &capture_spec_requested, &capture_spec_obtained, 0);
+    } else {
+        fprintf(stderr, "%s: attempt to open default capture device ...\n", __func__);
+        m_dev_id_in = SDL_OpenAudioDevice(nullptr, SDL_TRUE, &capture_spec_requested, &capture_spec_obtained, 0);
+    }
+
+    if (!m_dev_id_in) {
+        fprintf(stderr, "%s: couldn't open an audio device for capture: %s!\n", __func__, SDL_GetError());
+        m_dev_id_in = 0;
+
+        return false;
+    } else {
+        fprintf(stderr, "%s: obtained spec for input device (SDL Id = %d):\n", __func__, m_dev_id_in);
+        fprintf(stderr, "%s:     - sample rate:       %d\n",                   __func__, capture_spec_obtained.freq);
+        fprintf(stderr, "%s:     - format:            %d (required: %d)\n",    __func__, capture_spec_obtained.format,
+                capture_spec_requested.format);
+        fprintf(stderr, "%s:     - channels:          %d (required: %d)\n",    __func__, capture_spec_obtained.channels,
+                capture_spec_requested.channels);
+        fprintf(stderr, "%s:     - samples per frame: %d\n",                   __func__, capture_spec_obtained.samples);
+    }
+
+    m_sample_rate = capture_spec_obtained.freq;
+
+    m_audio.resize((m_sample_rate*m_len_ms)/1000);
+
+    return true;
+}
+
+bool audio_async::resume() {
+    if (!m_dev_id_in) {
+        fprintf(stderr, "%s: no audio device to resume!\n", __func__);
+        return false;
+    }
+
+    if (m_running) {
+        fprintf(stderr, "%s: already running!\n", __func__);
+        return false;
+    }
+
+    SDL_PauseAudioDevice(m_dev_id_in, 0);
+
+    m_running = true;
+
+    return true;
+}
+
+bool audio_async::pause() {
+    if (!m_dev_id_in) {
+        fprintf(stderr, "%s: no audio device to pause!\n", __func__);
+        return false;
+    }
+
+    if (!m_running) {
+        fprintf(stderr, "%s: already paused!\n", __func__);
+        return false;
+    }
+
+    SDL_PauseAudioDevice(m_dev_id_in, 1);
+
+    m_running = false;
+
+    return true;
+}
+
+bool audio_async::clear() {
+    if (!m_dev_id_in) {
+        fprintf(stderr, "%s: no audio device to clear!\n", __func__);
+        return false;
+    }
+
+    if (!m_running) {
+        fprintf(stderr, "%s: not running!\n", __func__);
+        return false;
+    }
+
+    {
+        std::lock_guard<std::mutex> lock(m_mutex);
+
+        m_audio_pos = 0;
+        m_audio_len = 0;
+    }
+
+    return true;
+}
+
+// callback to be called by SDL
+void audio_async::callback(uint8_t * stream, int len) {
+    if (!m_running) {
+        return;
+    }
+
+    size_t n_samples = len / sizeof(float);
+
+    if (n_samples > m_audio.size()) {
+        n_samples = m_audio.size();
+
+        stream += (len - (n_samples * sizeof(float)));
+    }
+
+    //fprintf(stderr, "%s: %zu samples, pos %zu, len %zu\n", __func__, n_samples, m_audio_pos, m_audio_len);
+
+    {
+        std::lock_guard<std::mutex> lock(m_mutex);
+
+        if (m_audio_pos + n_samples > m_audio.size()) {
+            const size_t n0 = m_audio.size() - m_audio_pos;
+
+            memcpy(&m_audio[m_audio_pos], stream, n0 * sizeof(float));
+            memcpy(&m_audio[0], stream + n0 * sizeof(float), (n_samples - n0) * sizeof(float));
+        } else {
+            memcpy(&m_audio[m_audio_pos], stream, n_samples * sizeof(float));
+        }
+        m_audio_pos = (m_audio_pos + n_samples) % m_audio.size();
+        m_audio_len = std::min(m_audio_len + n_samples, m_audio.size());
+    }
+}
+
+void audio_async::get(int ms, std::vector<float> & result) {
+    if (!m_dev_id_in) {
+        fprintf(stderr, "%s: no audio device to get audio from!\n", __func__);
+        return;
+    }
+
+    if (!m_running) {
+        fprintf(stderr, "%s: not running!\n", __func__);
+        return;
+    }
+
+    result.clear();
+
+    {
+        std::lock_guard<std::mutex> lock(m_mutex);
+
+        if (ms <= 0) {
+            ms = m_len_ms;
+        }
+
+        size_t n_samples = (m_sample_rate * ms) / 1000;
+        if (n_samples > m_audio_len) {
+            n_samples = m_audio_len;
+        }
+
+        result.resize(n_samples);
+
+        int s0 = m_audio_pos - n_samples;
+        if (s0 < 0) {
+            s0 += m_audio.size();
+        }
+
+        if (s0 + n_samples > m_audio.size()) {
+            const size_t n0 = m_audio.size() - s0;
+
+            memcpy(result.data(), &m_audio[s0], n0 * sizeof(float));
+            memcpy(&result[n0], &m_audio[0], (n_samples - n0) * sizeof(float));
+        } else {
+            memcpy(result.data(), &m_audio[s0], n_samples * sizeof(float));
+        }
+    }
+}
+
+bool sdl_poll_events() {
+    SDL_Event event;
+    while (SDL_PollEvent(&event)) {
+        switch (event.type) {
+            case SDL_QUIT:
+                {
+                    return false;
+                }
+            default:
+                break;
+        }
+    }
+
+    return true;
+}

examples/common-sdl.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <SDL.h>
+#include <SDL_audio.h>
+
+#include <atomic>
+#include <cstdint>
+#include <vector>
+#include <mutex>
+
+//
+// SDL Audio capture
+//
+
+class audio_async {
+public:
+    audio_async(int len_ms);
+    ~audio_async();
+
+    bool init(int capture_id, int sample_rate);
+
+    // start capturing audio via the provided SDL callback
+    // keep last len_ms seconds of audio in a circular buffer
+    bool resume();
+    bool pause();
+    bool clear();
+
+    // callback to be called by SDL
+    void callback(uint8_t * stream, int len);
+
+    // get audio data from the circular buffer
+    void get(int ms, std::vector<float> & audio);
+
+private:
+    SDL_AudioDeviceID m_dev_id_in = 0;
+
+    int m_len_ms = 0;
+    int m_sample_rate = 0;
+
+    std::atomic_bool m_running;
+    std::mutex       m_mutex;
+
+    std::vector<float> m_audio;
+    size_t             m_audio_pos = 0;
+    size_t             m_audio_len = 0;
+};
+
+// Return false if need to quit
+bool sdl_poll_events();

examples/common-whisper.cpp

@@ -0,0 +1,247 @@
+#define _USE_MATH_DEFINES // for M_PI
+
+#include "common-whisper.h"
+
+#include "common.h"
+
+#include "whisper.h"
+
+// third-party utilities
+// use your favorite implementations
+#define STB_VORBIS_HEADER_ONLY
+#include "stb_vorbis.c"    /* Enables Vorbis decoding. */
+
+#ifdef _WIN32
+#ifndef NOMINMAX
+    #define NOMINMAX
+#endif
+#endif
+
+#define MA_NO_DEVICE_IO
+#define MA_NO_THREADING
+#define MA_NO_ENCODING
+#define MA_NO_GENERATION
+#define MA_NO_RESOURCE_MANAGER
+#define MA_NO_NODE_GRAPH
+#define MINIAUDIO_IMPLEMENTATION
+#include "miniaudio.h"
+
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+#endif
+
+#include <cstring>
+#include <fstream>
+
+#ifdef WHISPER_COMMON_FFMPEG
+// as implemented in ffmpeg-trancode.cpp only embedded in common lib if whisper built with ffmpeg support
+extern bool ffmpeg_decode_audio(const std::string & ifname, std::vector<uint8_t> & wav_data, int out_sample_rate = WHISPER_SAMPLE_RATE);
+#endif
+
+// extract f32 PCM frames from an initialized decoder, downmix to mono and keep the stereo split
+static bool read_audio_from_decoder(ma_decoder & decoder, std::vector<float> & pcmf32, std::vector<std::vector<float>> & pcmf32s, bool stereo) {
+    ma_result result;
+    ma_uint64 frame_count;
+    ma_uint64 frames_read;
+
+    if ((result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count)) != MA_SUCCESS) {
+        fprintf(stderr, "error: failed to retrieve the length of the audio data (%s)\n", ma_result_description(result));
+        return false;
+    }
+
+    pcmf32.resize(stereo ? frame_count*2 : frame_count);
+
+    if ((result = ma_decoder_read_pcm_frames(&decoder, pcmf32.data(), frame_count, &frames_read)) != MA_SUCCESS) {
+        fprintf(stderr, "error: failed to read the frames of the audio data (%s)\n", ma_result_description(result));
+        return false;
+    }
+
+    if (stereo) {
+        std::vector<float> stereo_data = pcmf32;
+        pcmf32.resize(frame_count);
+        for (uint64_t i = 0; i < frame_count; i++) {
+            pcmf32[i] = (stereo_data[2*i] + stereo_data[2*i + 1]);
+        }
+        pcmf32s.resize(2);
+        pcmf32s[0].resize(frame_count);
+        pcmf32s[1].resize(frame_count);
+        for (uint64_t i = 0; i < frame_count; i++) {
+            pcmf32s[0][i] = stereo_data[2*i];
+            pcmf32s[1][i] = stereo_data[2*i + 1];
+        }
+    }
+
+    return true;
+}
+
+bool read_audio_data(const std::string & fname, std::vector<float> & pcmf32, std::vector<std::vector<float>> & pcmf32s, bool stereo) {
+    std::vector<uint8_t> audio_data; // used for pipe input from stdin or ffmpeg decoding output
+
+    ma_result result;
+    ma_decoder_config decoder_config;
+
+    struct decoder_guard {
+        ma_decoder decoder;
+        bool initialized = false;
+        ma_decoder * operator&() { return &decoder; }
+        ~decoder_guard() {
+            if (initialized) {
+                ma_decoder_uninit(&decoder);
+            }
+        }
+    };
+    decoder_guard decoder{};
+
+    decoder_config = ma_decoder_config_init(ma_format_f32, stereo ? 2 : 1, WHISPER_SAMPLE_RATE);
+
+    if (fname == "-") {
+#ifdef _WIN32
+        _setmode(_fileno(stdin), _O_BINARY);
+#endif
+
+        uint8_t buf[1024];
+        while (true)
+        {
+            const size_t n = fread(buf, 1, sizeof(buf), stdin);
+            if (n == 0) {
+                break;
+            }
+            audio_data.insert(audio_data.end(), buf, buf + n);
+        }
+
+        result = ma_decoder_init_memory(audio_data.data(), audio_data.size(), &decoder_config, &decoder);
+        if (result != MA_SUCCESS) {
+            fprintf(stderr, "%s: failed to open audio data from stdin (%s)\n", __func__, ma_result_description(result));
+            return false;
+        }
+        decoder.initialized = true;
+
+        fprintf(stderr, "%s: read %zu bytes from stdin\n", __func__, audio_data.size());
+    } else {
+        fprintf(stderr, "%s: reading audio data from '%s' ...\n", __func__, fname.c_str());
+
+        // first try miniaudio. if it fails (or skipped) - try ffmpeg
+        {
+            const char * skip = getenv("WHISPER_COMMON_MINIAUDIO_SKIP");
+            if (!skip || strlen(skip) == 0 || strcmp(skip, "0") == 0) {
+                fprintf(stderr, "%s: trying to decode with miniaudio\n", __func__);
+
+                result = ma_decoder_init_file(fname.c_str(), &decoder_config, &decoder);
+                if (result == MA_SUCCESS) {
+                    decoder.initialized = true;
+                }
+            } else {
+                fprintf(stderr, "%s: skipping miniaudio\n", __func__);
+            }
+        }
+
+#if defined(WHISPER_COMMON_FFMPEG)
+        if (!decoder.initialized) {
+            fprintf(stderr, "%s: trying to decode with ffmpeg\n", __func__);
+
+            if (ffmpeg_decode_audio(fname, audio_data) != 0) {
+                fprintf(stderr, "%s: failed to ffmpeg decode\n", __func__);
+                return false;
+            }
+            result = ma_decoder_init_memory(audio_data.data(), audio_data.size(), &decoder_config, &decoder);
+            if (result != MA_SUCCESS) {
+                fprintf(stderr, "%s: failed to read audio data as wav (%s)\n", __func__, ma_result_description(result));
+                return false;
+            }
+            decoder.initialized = true;
+        }
+#endif
+
+        if (!decoder.initialized) {
+            fprintf(stderr, "%s: failed to read audio data\n", __func__);
+            return false;
+        }
+    }
+
+    return read_audio_from_decoder(decoder.decoder, pcmf32, pcmf32s, stereo);
+}
+
+// decode audio bytes already held in memory
+bool read_audio_data(const char * buffer, size_t buffer_size, std::vector<float> & pcmf32, std::vector<std::vector<float>> & pcmf32s, bool stereo) {
+    ma_decoder_config decoder_config = ma_decoder_config_init(ma_format_f32, stereo ? 2 : 1, WHISPER_SAMPLE_RATE);
+    ma_decoder decoder;
+
+    if (ma_decoder_init_memory(buffer, buffer_size, &decoder_config, &decoder) != MA_SUCCESS) {
+        fprintf(stderr, "error: failed to decode audio data from memory buffer\n");
+        return false;
+    }
+
+    bool ok = read_audio_from_decoder(decoder, pcmf32, pcmf32s, stereo);
+    ma_decoder_uninit(&decoder);
+    return ok;
+}
+
+//  500 -> 00:05.000
+// 6000 -> 01:00.000
+std::string to_timestamp(int64_t t, bool comma) {
+    int64_t msec = t * 10;
+    int64_t hr = msec / (1000 * 60 * 60);
+    msec = msec - hr * (1000 * 60 * 60);
+    int64_t min = msec / (1000 * 60);
+    msec = msec - min * (1000 * 60);
+    int64_t sec = msec / 1000;
+    msec = msec - sec * 1000;
+
+    char buf[32];
+    snprintf(buf, sizeof(buf), "%02d:%02d:%02d%s%03d", (int) hr, (int) min, (int) sec, comma ? "," : ".", (int) msec);
+
+    return std::string(buf);
+}
+
+int timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate) {
+    return std::max(0, std::min((int) n_samples - 1, (int) ((t*whisper_sample_rate)/100)));
+}
+
+int utf8_trailing_bytes_needed(const std::string & s) {
+    const int n = (int) s.size();
+    int i = n - 1;
+    while (i >= 0 && ((unsigned char) s[i] & 0xC0) == 0x80) {
+        --i;
+    }
+    if (i < 0) {
+        return 0;
+    }
+
+    const unsigned char c = (unsigned char) s[i];
+    int expected;
+    if ((c & 0x80) == 0x00) {
+        expected = 1;
+    } else if ((c & 0xE0) == 0xC0) {
+        expected = 2;
+    } else if ((c & 0xF0) == 0xE0) {
+        expected = 3;
+    } else if ((c & 0xF8) == 0xF0) {
+        expected = 4;
+    } else {
+        return 0;
+    }
+
+    const int have = n - i;
+    return have >= expected ? 0 : (expected - have);
+}
+
+bool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id) {
+    std::ofstream speak_file(path.c_str());
+    if (speak_file.fail()) {
+        fprintf(stderr, "%s: failed to open speak_file\n", __func__);
+        return false;
+    } else {
+        speak_file.write(text.c_str(), text.size());
+        speak_file.close();
+        int ret = system((command + " " + std::to_string(voice_id) + " " + path).c_str());
+        if (ret != 0) {
+            fprintf(stderr, "%s: failed to speak\n", __func__);
+            return false;
+        }
+    }
+    return true;
+}
+
+#undef STB_VORBIS_HEADER_ONLY
+#include "stb_vorbis.c"

examples/common-whisper.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+// Read WAV audio file and store the PCM data into pcmf32
+// fname can be a buffer of WAV data instead of a filename
+// The sample rate of the audio must be equal to COMMON_SAMPLE_RATE
+// If stereo flag is set and the audio has 2 channels, the pcmf32s will contain 2 channel PCM
+bool read_audio_data(
+        const std::string & fname,
+        std::vector<float> & pcmf32,
+        std::vector<std::vector<float>> & pcmf32s,
+        bool stereo);
+
+// decode audio bytes already held in memory (uploaded file, network buffer)
+bool read_audio_data(
+        const char * buffer,
+        size_t buffer_size,
+        std::vector<float> & pcmf32,
+        std::vector<std::vector<float>> & pcmf32s,
+        bool stereo);
+
+// convert timestamp to string, 6000 -> 01:00.000
+std::string to_timestamp(int64_t t, bool comma = false);
+
+// given a timestamp get the sample
+int timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate);
+
+// Returns the number of trailing bytes still needed for s to end on a complete UTF-8 codepoint.
+int utf8_trailing_bytes_needed(const std::string & s);
+
+// write text to file, and call system("command voice_id file")
+bool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id);

examples/common.cpp

@@ -0,0 +1,675 @@
+#define _USE_MATH_DEFINES // for M_PI
+
+#include "common.h"
+
+#include <cmath>
+#include <codecvt>
+#include <cstring>
+#include <fstream>
+#include <locale>
+#include <regex>
+#include <sstream>
+
+// Function to check if the next argument exists
+static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {
+    if (i + 1 < argc && argv[i + 1][0] != '-') {
+        return argv[++i];
+    } else {
+        fprintf(stderr, "error: %s requires one argument.\n", flag.c_str());
+        gpt_print_usage(argc, argv, params);
+        exit(0);
+    }
+}
+
+bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-s" || arg == "--seed") {
+            params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-t" || arg == "--threads") {
+            params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-p" || arg == "--prompt") {
+            params.prompt = get_next_arg(i, argc, argv, arg, params);
+        } else if (arg == "-n" || arg == "--n_predict") {
+            params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-np" || arg == "--n_parallel") {
+            params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "--top_k") {
+            params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "--top_p") {
+            params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "--temp") {
+            params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "--repeat-last-n") {
+            params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "--repeat-penalty") {
+            params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-b" || arg == "--batch_size") {
+            params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-c" || arg == "--context") {
+            params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
+            params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "--ignore-eos") {
+            params.ignore_eos = true;
+        } else if (arg == "-m" || arg == "--model") {
+            params.model = get_next_arg(i, argc, argv, arg, params);
+        } else if (arg == "-i" || arg == "--interactive") {
+            params.interactive = true;
+        } else if (arg == "-ip" || arg == "--interactive-port") {
+            params.interactive = true;
+            params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));
+        } else if (arg == "-h" || arg == "--help") {
+            gpt_print_usage(argc, argv, params);
+            exit(0);
+        } else if (arg == "-f" || arg == "--file") {
+            get_next_arg(i, argc, argv, arg, params);
+            std::ifstream file(argv[i]);
+            if (!file) {
+                fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
+                break;
+            }
+            std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
+            if (params.prompt.back() == '\n') {
+                params.prompt.pop_back();
+            }
+        } else if (arg == "-tt" || arg == "--token_test") {
+            params.token_test = get_next_arg(i, argc, argv, arg, params);
+        }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            gpt_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
+    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h, --help            show this help message and exit\n");
+    fprintf(stderr, "  -s SEED, --seed SEED  RNG seed (default: -1)\n");
+    fprintf(stderr, "  -t N, --threads N     number of threads to use during computation (default: %d)\n", params.n_threads);
+    fprintf(stderr, "  -p PROMPT, --prompt PROMPT\n");
+    fprintf(stderr, "                        prompt to start generation with (default: random)\n");
+    fprintf(stderr, "  -f FNAME, --file FNAME\n");
+    fprintf(stderr, "                        load prompt from a file\n");
+    fprintf(stderr, "  -tt TOKEN_TEST, --token_test TOKEN_TEST\n");
+    fprintf(stderr, "                        test tokenization\n");
+    fprintf(stderr, "  -n N, --n_predict N   number of tokens to predict (default: %d)\n", params.n_predict);
+    fprintf(stderr, "  --top_k N             top-k sampling (default: %d)\n", params.top_k);
+    fprintf(stderr, "  --top_p N             top-p sampling (default: %.1f)\n", params.top_p);
+    fprintf(stderr, "  --temp N              temperature (default: %.1f)\n", params.temp);
+    fprintf(stderr, "  --repeat-last-n N     last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n);
+    fprintf(stderr, "  --repeat-penalty N    penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty);
+    fprintf(stderr, "  -b N, --batch_size N  batch size for prompt processing (default: %d)\n", params.n_batch);
+    fprintf(stderr, "  -c N, --context N     context / KV cache size (default: %d)\n", params.n_ctx);
+    fprintf(stderr, "  --ignore-eos          ignore EOS token during generation\n");
+    fprintf(stderr, "  -ngl N, --gpu-layers N  number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers);
+    fprintf(stderr, "  -m FNAME, --model FNAME\n");
+    fprintf(stderr, "                        model path (default: %s)\n", params.model.c_str());
+    fprintf(stderr, "\n");
+}
+
+std::string gpt_random_prompt(std::mt19937 & rng) {
+    const int r = rng() % 10;
+    switch (r) {
+        case 0: return "So";
+        case 1: return "Once upon a time";
+        case 2: return "When";
+        case 3: return "The";
+        case 4: return "After";
+        case 5: return "If";
+        case 6: return "import";
+        case 7: return "He";
+        case 8: return "She";
+        case 9: return "They";
+    }
+
+    return "The";
+}
+
+std::string trim(const std::string & s) {
+    std::regex e("^\\s+|\\s+$");
+    return std::regex_replace(s, e, "");
+}
+
+std::string replace(const std::string & s, const std::string & from, const std::string & to) {
+    std::string result = s;
+    size_t pos = 0;
+    while ((pos = result.find(from, pos)) != std::string::npos) {
+        result.replace(pos, from.length(), to);
+        pos += to.length();
+    }
+    return result;
+}
+
+void gpt_vocab::add_special_token(const std::string & token) {
+    special_tokens.push_back(token);
+}
+
+std::map<std::string, int32_t> json_parse(const std::string & fname) {
+    std::map<std::string, int32_t> result;
+
+    // read file into string
+    std::string json;
+    {
+        std::ifstream ifs(fname);
+        if (!ifs) {
+            fprintf(stderr, "Failed to open %s\n", fname.c_str());
+            exit(1);
+        }
+
+        json = std::string((std::istreambuf_iterator<char>(ifs)),
+                (std::istreambuf_iterator<char>()));
+    }
+
+    if (json[0] != '{') {
+        return result;
+    }
+
+    // parse json
+    {
+        bool has_key  = false;
+        bool in_token = false;
+
+        std::string str_key = "";
+        std::string str_val = "";
+
+        int n = json.size();
+        for (int i = 1; i < n; ++i) {
+            if (!in_token) {
+                if (json[i] == ' ') continue;
+                if (json[i] == '"') {
+                    in_token = true;
+                    continue;
+                }
+            } else {
+                if (json[i] == '\\' && i+1 < n) {
+                    if (has_key == false) {
+                        str_key += json[i];
+                    } else {
+                        str_val += json[i];
+                    }
+                    ++i;
+                } else if (json[i] == '"') {
+                    if (has_key == false) {
+                        has_key = true;
+                        ++i;
+                        while (json[i] == ' ') ++i;
+                        ++i; // :
+                        while (json[i] == ' ') ++i;
+                        if (json[i] != '\"') {
+                            while (json[i] != ',' && json[i] != '}') {
+                                str_val += json[i++];
+                            }
+                            has_key = false;
+                        } else {
+                            in_token = true;
+                            continue;
+                        }
+                    } else {
+                        has_key = false;
+                    }
+
+                    str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
+                    str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
+                    str_key = ::replace(str_key, "\\\"",    "\""); // \\\"   -> "
+
+                    try {
+                        result[str_key] = std::stoi(str_val);
+                    } catch (...) {
+                        //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
+
+                    }
+                    str_key = "";
+                    str_val = "";
+                    in_token = false;
+                    continue;
+                }
+                if (has_key == false) {
+                    str_key += json[i];
+                } else {
+                    str_val += json[i];
+                }
+            }
+        }
+    }
+
+    return result;
+}
+
+void gpt_split_words(std::string str, std::vector<std::string>& words) {
+    const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
+    const std::regex re(pattern);
+    std::smatch m;
+
+    while (std::regex_search(str, m, re)) {
+        for (auto x : m) {
+            words.push_back(x);
+        }
+        str = m.suffix();
+    }
+}
+
+std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
+    std::vector<std::string> words;
+
+    // first split the text into words
+    {
+        std::string str = text;
+
+        // Generate the subpattern from the special_tokens vector if it's not empty
+        if (!vocab.special_tokens.empty()) {
+            const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])");
+            std::string special_tokens_subpattern;
+            for (const auto & token : vocab.special_tokens) {
+                if (!special_tokens_subpattern.empty()) {
+                    special_tokens_subpattern += "|";
+                }
+                special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)");
+            }
+
+            std::regex re(special_tokens_subpattern);
+            std::smatch m;
+            // Split the text by special tokens.
+            while (std::regex_search(str, m, re)) {
+                // Split the substrings in-between special tokens into words.
+                gpt_split_words(m.prefix(), words);
+                // Add matched special tokens as words.
+                for (auto x : m) {
+                    words.push_back(x);
+                }
+                str = m.suffix();
+            }
+            // Remaining text without special tokens will be handled below.
+        }
+
+        gpt_split_words(str, words);
+    }
+
+    // find the longest token that forms each word in words:
+    std::vector<gpt_vocab::id> tokens;
+    for (const auto & word : words) {
+        for (int i = 0; i < (int) word.size(); ){
+            for (int j = word.size() - 1; j >= i; j--){
+                auto cand = word.substr(i, j-i+1);
+                auto it = vocab.token_to_id.find(cand);
+                if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab
+                    tokens.push_back(it->second);
+                    i = j + 1;
+                    break;
+                }
+                else if (j == i){ // word.substr(i, 1) has no matching
+                    fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
+                    i++;
+                }
+            }
+        }
+    }
+
+    return tokens;
+}
+
+static std::vector<gpt_vocab::id> parse_tokens_from_string(const std::string& input, char delimiter) {
+    std::vector<gpt_vocab::id> output;
+    std::stringstream ss(input);
+    std::string token;
+
+    while (std::getline(ss, token, delimiter)) {
+        output.push_back(std::stoi(token));
+    }
+
+    return output;
+}
+
+static std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){
+    if (fpath_test.empty()){
+        fprintf(stderr, "%s : No test file found.\n", __func__);
+        return std::map<std::string, std::vector<gpt_vocab::id>>();
+    }
+
+    std::map<std::string, std::vector<gpt_vocab::id>> tests;
+
+    auto fin = std::ifstream(fpath_test, std::ios_base::in);
+    const char * delimeter = " => ";
+    const char del_tok = ',';
+    std::string line;
+    while (std::getline(fin, line)) {
+        size_t delimiterPos = line.find(delimeter);
+        if (delimiterPos != std::string::npos) {
+            std::string text = line.substr(0, delimiterPos);
+            std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));
+            tests[text] = parse_tokens_from_string(s_tokens, del_tok);
+        }
+    }
+    return tests;
+}
+
+void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){
+    std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test);
+
+    size_t n_fails = 0;
+
+    for (const auto & test : tests) {
+        std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first);
+
+        if (tokens != test.second){
+            n_fails++;
+
+            // print out failure cases
+            fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str());
+            fprintf(stderr, "%s : tokens in hf:   ", __func__);
+            for (const auto & t : test.second) {
+                fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
+            }
+            fprintf(stderr, "\n");
+            fprintf(stderr, "%s : tokens in ggml: ", __func__);
+            for (const auto & t : tokens) {
+                fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
+            }
+            fprintf(stderr, "\n");
+        }
+    }
+
+    fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size());
+}
+
+bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
+    printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
+
+    vocab.token_to_id = ::json_parse(fname);
+
+    for (const auto & kv : vocab.token_to_id) {
+        vocab.id_to_token[kv.second] = kv.first;
+    }
+
+    printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
+
+    // print the vocabulary
+    //for (auto kv : vocab.token_to_id) {
+    //    printf("'%s' -> %d\n", kv.first.data(), kv.second);
+    //}
+
+    return true;
+}
+
+gpt_vocab::id gpt_sample_top_k_top_p(
+        const gpt_vocab & vocab,
+        const float * logits,
+        int    top_k,
+        double top_p,
+        double temp,
+        std::mt19937 & rng) {
+    int n_logits = vocab.id_to_token.size();
+
+    std::vector<std::pair<double, gpt_vocab::id>> logits_id;
+    logits_id.reserve(n_logits);
+
+    {
+        const double scale = 1.0/temp;
+        for (int i = 0; i < n_logits; ++i) {
+            logits_id.push_back(std::make_pair(logits[i]*scale, i));
+        }
+    }
+
+    // find the top K tokens
+    std::partial_sort(
+            logits_id.begin(),
+            logits_id.begin() + top_k, logits_id.end(),
+            [](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
+        return a.first > b.first;
+    });
+
+    logits_id.resize(top_k);
+
+    double maxl = -INFINITY;
+    for (const auto & kv : logits_id) {
+        maxl = std::max(maxl, kv.first);
+    }
+
+    // compute probs for the top K tokens
+    std::vector<double> probs;
+    probs.reserve(logits_id.size());
+
+    double sum = 0.0;
+    for (const auto & kv : logits_id) {
+        double p = exp(kv.first - maxl);
+        probs.push_back(p);
+        sum += p;
+    }
+
+    // normalize the probs
+    for (auto & p : probs) {
+        p /= sum;
+    }
+
+    if (top_p < 1.0f) {
+        double cumsum = 0.0f;
+        for (int i = 0; i < top_k; i++) {
+            cumsum += probs[i];
+            if (cumsum >= top_p) {
+                top_k = i + 1;
+                probs.resize(top_k);
+                logits_id.resize(top_k);
+                break;
+            }
+        }
+
+        cumsum = 1.0/cumsum;
+        for (int i = 0; i < (int) probs.size(); i++) {
+            probs[i] *= cumsum;
+        }
+    }
+
+    //printf("\n");
+    //for (int i = 0; i < (int) probs.size(); i++) {
+    //    printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
+    //}
+    //exit(0);
+
+    std::discrete_distribution<> dist(probs.begin(), probs.end());
+    int idx = dist(rng);
+
+    return logits_id[idx].second;
+}
+
+gpt_vocab::id gpt_sample_top_k_top_p_repeat(
+        const gpt_vocab & vocab,
+        const float * logits,
+        const int32_t * last_n_tokens_data,
+        size_t last_n_tokens_data_size,
+        int    top_k,
+        double top_p,
+        double temp,
+        int repeat_last_n,
+        float repeat_penalty,
+        std::mt19937 & rng) {
+
+    int n_logits = vocab.id_to_token.size();
+
+    const auto * plogits = logits;
+
+    const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);
+
+    if (temp <= 0) {
+        // select the token with the highest logit directly
+        float max_logit = plogits[0];
+        gpt_vocab::id max_id = 0;
+
+        for (int i = 1; i < n_logits; ++i) {
+            if (plogits[i] > max_logit) {
+                max_logit = plogits[i];
+                max_id = i;
+            }
+        }
+        return max_id;
+    }
+
+
+    std::vector<std::pair<double, gpt_vocab::id>> logits_id;
+    logits_id.reserve(n_logits);
+
+    {
+        const float scale = 1.0f/temp;
+        for (int i = 0; i < n_logits; ++i) {
+            // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
+            // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
+            if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {
+                // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
+                if (plogits[i] < 0.0f) {
+                    logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
+                } else {
+                    logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
+                }
+            } else {
+                logits_id.push_back(std::make_pair(plogits[i]*scale, i));
+            }
+        }
+    }
+
+    // find the top K tokens
+    std::partial_sort(
+            logits_id.begin(),
+            logits_id.begin() + top_k, logits_id.end(),
+            [](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
+        return a.first > b.first;
+    });
+
+    logits_id.resize(top_k);
+
+    double maxl = -INFINITY;
+    for (const auto & kv : logits_id) {
+        maxl = std::max(maxl, kv.first);
+    }
+
+    // compute probs for the top K tokens
+    std::vector<double> probs;
+    probs.reserve(logits_id.size());
+
+    double sum = 0.0;
+    for (const auto & kv : logits_id) {
+        double p = exp(kv.first - maxl);
+        probs.push_back(p);
+        sum += p;
+    }
+
+    // normalize the probs
+    for (auto & p : probs) {
+        p /= sum;
+    }
+
+    if (top_p < 1.0f) {
+        double cumsum = 0.0f;
+        for (int i = 0; i < top_k; i++) {
+            cumsum += probs[i];
+            if (cumsum >= top_p) {
+                top_k = i + 1;
+                probs.resize(top_k);
+                logits_id.resize(top_k);
+                break;
+            }
+        }
+
+        cumsum = 1.0/cumsum;
+        for (int i = 0; i < (int) probs.size(); i++) {
+            probs[i] *= cumsum;
+        }
+    }
+
+//    printf("\n");
+//    for (int i = 0; i < (int) probs.size(); i++) {
+//    for (int i = 0; i < 10; i++) {
+//        printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
+//    }
+
+    std::discrete_distribution<> dist(probs.begin(), probs.end());
+    int idx = dist(rng);
+
+    return logits_id[idx].second;
+
+}
+
+void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {
+    const float rc = 1.0f / (2.0f * M_PI * cutoff);
+    const float dt = 1.0f / sample_rate;
+    const float alpha = dt / (rc + dt);
+
+    float y = data[0];
+
+    for (size_t i = 1; i < data.size(); i++) {
+        y = alpha * (y + data[i] - data[i - 1]);
+        data[i] = y;
+    }
+}
+
+bool vad_simple(std::vector<float> & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {
+    const int n_samples      = pcmf32.size();
+    const int n_samples_last = (sample_rate * last_ms) / 1000;
+
+    if (n_samples_last >= n_samples) {
+        // not enough samples - assume no speech
+        return false;
+    }
+
+    if (freq_thold > 0.0f) {
+        high_pass_filter(pcmf32, freq_thold, sample_rate);
+    }
+
+    float energy_all  = 0.0f;
+    float energy_last = 0.0f;
+
+    for (int i = 0; i < n_samples; i++) {
+        energy_all += fabsf(pcmf32[i]);
+
+        if (i >= n_samples - n_samples_last) {
+            energy_last += fabsf(pcmf32[i]);
+        }
+    }
+
+    energy_all  /= n_samples;
+    energy_last /= n_samples_last;
+
+    if (verbose) {
+        fprintf(stderr, "%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\n", __func__, energy_all, energy_last, vad_thold, freq_thold);
+    }
+
+    if (energy_last > vad_thold*energy_all) {
+        return false;
+    }
+
+    return true;
+}
+
+float similarity(const std::string & s0, const std::string & s1) {
+    const size_t len0 = s0.size() + 1;
+    const size_t len1 = s1.size() + 1;
+
+    std::vector<int> col(len1, 0);
+    std::vector<int> prevCol(len1, 0);
+
+    for (size_t i = 0; i < len1; i++) {
+        prevCol[i] = i;
+    }
+
+    for (size_t i = 0; i < len0; i++) {
+        col[0] = i;
+        for (size_t j = 1; j < len1; j++) {
+            col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));
+        }
+        col.swap(prevCol);
+    }
+
+    const float dist = prevCol[len1 - 1];
+
+    return 1.0f - (dist / std::max(s0.size(), s1.size()));
+}
+
+bool is_file_exist(const char * filename) {
+    std::ifstream infile(filename);
+    return infile.good();
+}

examples/common.h

@@ -0,0 +1,322 @@
+// Various helper functions and utilities
+
+#pragma once
+
+#include <string>
+#include <map>
+#include <vector>
+#include <random>
+#include <thread>
+#include <ctime>
+#include <fstream>
+#include <sstream>
+
+//
+// GPT CLI argument parsing
+//
+
+struct gpt_params {
+    int32_t seed         = -1;   // RNG seed
+    int32_t n_threads    = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t n_predict    = 200;  // new tokens to predict
+    int32_t n_parallel   = 1;    // number of parallel streams
+    int32_t n_batch      = 32;   // batch size for prompt processing
+    int32_t n_ctx        = 2048; // context size (this is the KV cache max size)
+    int32_t n_gpu_layers = 0;    // number of layers to offlload to the GPU
+
+    bool ignore_eos = false; // ignore EOS token when generating text
+
+    // sampling parameters
+    int32_t top_k          = 40;
+    float   top_p          = 0.9f;
+    float   temp           = 0.9f;
+    int32_t repeat_last_n  = 64;
+    float   repeat_penalty = 1.00f;
+
+    std::string model      = "models/gpt-2-117M/ggml-model.bin"; // model path
+    std::string prompt     = "";
+    std::string token_test = "";
+
+    bool    interactive      = false;
+    int32_t interactive_port = -1;
+};
+
+bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
+
+void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
+
+std::string gpt_random_prompt(std::mt19937 & rng);
+
+//
+// Vocab utils
+//
+
+std::string trim(const std::string & s);
+
+std::string replace(
+        const std::string & s,
+        const std::string & from,
+        const std::string & to);
+
+struct gpt_vocab {
+    using id    = int32_t;
+    using token = std::string;
+
+    std::map<token, id> token_to_id;
+    std::map<id, token> id_to_token;
+    std::vector<std::string> special_tokens;
+
+    void add_special_token(const std::string & token);
+};
+
+// poor-man's JSON parsing
+std::map<std::string, int32_t> json_parse(const std::string & fname);
+
+std::string convert_to_utf8(const std::wstring & input);
+
+std::wstring convert_to_wstring(const std::string & input);
+
+void gpt_split_words(std::string str, std::vector<std::string>& words);
+
+// split text into tokens
+//
+// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
+//
+// Regex (Python):
+// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+//
+// Regex (C++):
+// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
+//
+std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
+
+// test outputs of gpt_tokenize
+//
+//   - compare with tokens generated by the huggingface tokenizer
+//   - test cases are chosen based on the model's main language (under 'prompt' directory)
+//   - if all sentences are tokenized identically, print 'All tests passed.'
+//   - otherwise, print sentence, huggingface tokens, ggml tokens
+//
+void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);
+
+// load the tokens from encoder.json
+bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
+
+// sample next token given probabilities for each embedding
+//
+//   - consider only the top K tokens
+//   - from them, consider only the top tokens with cumulative probability > P
+//
+// TODO: not sure if this implementation is correct
+// TODO: temperature is not implemented
+//
+gpt_vocab::id gpt_sample_top_k_top_p(
+        const gpt_vocab & vocab,
+        const float * logits,
+        int    top_k,
+        double top_p,
+        double temp,
+        std::mt19937 & rng);
+
+gpt_vocab::id gpt_sample_top_k_top_p_repeat(
+        const gpt_vocab & vocab,
+        const float * logits,
+        const int32_t * last_n_tokens_data,
+        size_t last_n_tokens_data_size,
+        int    top_k,
+        double top_p,
+        double temp,
+        int repeat_last_n,
+        float repeat_penalty,
+        std::mt19937 & rng);
+
+//
+// Audio utils
+//
+
+// Write PCM data into WAV audio file
+class wav_writer {
+private:
+    std::ofstream file;
+    uint32_t dataSize = 0;
+    std::string wav_filename;
+
+    bool write_header(const uint32_t sample_rate,
+                      const uint16_t bits_per_sample,
+                      const uint16_t channels) {
+
+        file.write("RIFF", 4);
+        file.write("\0\0\0\0", 4);    // Placeholder for file size
+        file.write("WAVE", 4);
+        file.write("fmt ", 4);
+
+        const uint32_t sub_chunk_size = 16;
+        const uint16_t audio_format = 1;      // PCM format
+        const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;
+        const uint16_t block_align = channels * bits_per_sample / 8;
+
+        file.write(reinterpret_cast<const char *>(&sub_chunk_size), 4);
+        file.write(reinterpret_cast<const char *>(&audio_format), 2);
+        file.write(reinterpret_cast<const char *>(&channels), 2);
+        file.write(reinterpret_cast<const char *>(&sample_rate), 4);
+        file.write(reinterpret_cast<const char *>(&byte_rate), 4);
+        file.write(reinterpret_cast<const char *>(&block_align), 2);
+        file.write(reinterpret_cast<const char *>(&bits_per_sample), 2);
+        file.write("data", 4);
+        file.write("\0\0\0\0", 4);    // Placeholder for data size
+
+        return true;
+    }
+
+    // It is assumed that PCM data is normalized to a range from -1 to 1
+    bool write_audio(const float * data, size_t length) {
+        for (size_t i = 0; i < length; ++i) {
+            const int16_t intSample = int16_t(data[i] * 32767);
+            file.write(reinterpret_cast<const char *>(&intSample), sizeof(int16_t));
+            dataSize += sizeof(int16_t);
+        }
+        if (file.is_open()) {
+            file.seekp(4, std::ios::beg);
+            uint32_t fileSize = 36 + dataSize;
+            file.write(reinterpret_cast<char *>(&fileSize), 4);
+            file.seekp(40, std::ios::beg);
+            file.write(reinterpret_cast<char *>(&dataSize), 4);
+            file.seekp(0, std::ios::end);
+        }
+        return true;
+    }
+
+    bool open_wav(const std::string & filename) {
+        if (filename != wav_filename) {
+            if (file.is_open()) {
+                file.close();
+            }
+        }
+        if (!file.is_open()) {
+            file.open(filename, std::ios::binary);
+            wav_filename = filename;
+            dataSize = 0;
+        }
+        return file.is_open();
+    }
+
+public:
+    bool open(const std::string & filename,
+              const    uint32_t   sample_rate,
+              const    uint16_t   bits_per_sample,
+              const    uint16_t   channels) {
+
+        if (open_wav(filename)) {
+            write_header(sample_rate, bits_per_sample, channels);
+        } else {
+            return false;
+        }
+
+        return true;
+    }
+
+    bool close() {
+        file.close();
+        return true;
+    }
+
+    bool write(const float * data, size_t length) {
+        return write_audio(data, length);
+    }
+
+    ~wav_writer() {
+        if (file.is_open()) {
+            file.close();
+        }
+    }
+};
+
+
+// Apply a high-pass frequency filter to PCM audio
+// Suppresses frequencies below cutoff Hz
+void high_pass_filter(
+        std::vector<float> & data,
+        float cutoff,
+        float sample_rate);
+
+// Basic voice activity detection (VAD) using audio energy adaptive threshold
+bool vad_simple(
+        std::vector<float> & pcmf32,
+        int   sample_rate,
+        int   last_ms,
+        float vad_thold,
+        float freq_thold,
+        bool  verbose);
+
+// compute similarity between two strings using Levenshtein distance
+float similarity(const std::string & s0, const std::string & s1);
+
+//
+// Terminal utils
+//
+
+#define SQR(X)    ((X) * (X))
+#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40
+
+/**
+ * Quantizes 24-bit RGB to xterm256 code range [16,256).
+ */
+static int rgb2xterm256(int r, int g, int b) {
+    unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};
+    int av, ir, ig, ib, il, qr, qg, qb, ql;
+    av = r * .299 + g * .587 + b * .114 + .5;
+    ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;
+    qr = cube[(ir = UNCUBE(r))];
+    qg = cube[(ig = UNCUBE(g))];
+    qb = cube[(ib = UNCUBE(b))];
+    if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=
+        SQR(ql - r) + SQR(ql - g) + SQR(ql - b))
+        return ir * 36 + ig * 6 + ib + 020;
+    return il + 0350;
+}
+
+static std::string set_xterm256_foreground(int r, int g, int b) {
+    int x = rgb2xterm256(r, g, b);
+    std::ostringstream oss;
+    oss << "\033[38;5;" << x << "m";
+    return oss.str();
+}
+
+// Lowest is red, middle is yellow, highest is green. Color scheme from
+// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault
+const std::vector<std::string> k_colors = {
+    set_xterm256_foreground(220,   5,  12),
+    set_xterm256_foreground(232,  96,  28),
+    set_xterm256_foreground(241, 147,  45),
+    set_xterm256_foreground(246, 193,  65),
+    set_xterm256_foreground(247, 240,  86),
+    set_xterm256_foreground(144, 201, 135),
+    set_xterm256_foreground( 78, 178, 101),
+};
+
+// ANSI formatting codes
+static std::string set_inverse() {
+    return "\033[7m";
+}
+
+static std::string set_underline() {
+    return "\033[4m";
+}
+
+static std::string set_dim() {
+    return "\033[2m";
+}
+
+// Style scheme for different confidence levels
+const std::vector<std::string> k_styles = {
+    set_inverse(),   // Low confidence - inverse (highlighted)
+    set_underline(), // Medium confidence - underlined
+    set_dim(),       // High confidence - dim
+};
+
+//
+// Other utils
+//
+
+// check if file exists using ifstream
+bool is_file_exist(const char * filename);

examples/deprecation-warning/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_executable(main ./deprecation-warning.cpp)
+add_executable(bench ./deprecation-warning.cpp)
+if (WHISPER_SDL2)
+    add_executable(stream ./deprecation-warning.cpp)
+    add_executable(command ./deprecation-warning.cpp)
+endif()

examples/deprecation-warning/README.md

@@ -0,0 +1,17 @@
+# Migration notice for binary filenames
+
+> [!IMPORTANT]
+[2024 Dec 20] Binaries have been renamed w/ a `whisper-` prefix. `main` is now `whisper-cli`, `server` is `whisper-server`, etc (https://github.com/ggerganov/whisper.cpp/pull/2648)
+
+This migration was important, but it is a breaking change that may not always be immediately obvious to users.
+
+Please update all scripts and workflows to use the new binary names.
+
+| Old Filename | New Filename |
+| ---- | ---- |
+| main | whisper-cli |
+| bench | whisper-bench |
+| stream | whisper-stream |
+| command | whisper-command |
+| server | whisper-server |
+| talk-llama | whisper-talk-llama |

examples/deprecation-warning/deprecation-warning.cpp

@@ -0,0 +1,38 @@
+// Warns users that this filename was deprecated, and provides a link for more information.
+
+#include <cstdio>
+#include <string>
+
+// Main
+int main(int argc, char** argv) {
+    std::string filename = "main";
+    if (argc >= 1) {
+        filename = argv[0];
+    }
+
+    // Get only the program name from the full path
+    size_t pos = filename.find_last_of("/\\");
+    if (pos != std::string::npos) {
+        filename = filename.substr(pos+1);
+    }
+
+    // Append "whisper-" to the beginning of filename to get the replacemnt filename
+    std::string replacement_filename = "whisper-" + filename;
+
+    // The exception is if the filename is "main", then our replacement filename is "whisper-cli"
+    if (filename == "main") {
+        replacement_filename = "whisper-cli";
+    }
+
+    if (filename == "main.exe") {
+        replacement_filename = "whisper-cli.exe";
+    }
+
+    fprintf(stdout, "\n");
+    fprintf(stdout, "WARNING: The binary '%s' is deprecated.\n", filename.c_str());
+    fprintf(stdout, " Please use '%s' instead.\n", replacement_filename.c_str());
+    fprintf(stdout, " See https://github.com/ggerganov/whisper.cpp/tree/master/examples/deprecation-warning/README.md for more information.\n");
+    fprintf(stdout, "\n");
+
+    return EXIT_FAILURE;
+}

examples/ffmpeg-transcode.cpp

@@ -0,0 +1,238 @@
+#ifdef WHISPER_COMMON_FFMPEG
+
+#include <string>
+#include <vector>
+#include <cstdio>
+#include <cstring>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libswresample/swresample.h>
+}
+
+// Write a minimal WAV header into the output buffer.
+// Returns the number of bytes written (44 for a standard PCM WAV header).
+static size_t wav_header_write(uint8_t * buf, int num_channels, int sample_rate, int bits_per_sample, uint32_t data_size) {
+    // RIFF header
+    memcpy(buf, "RIFF", 4);
+    uint32_t chunk_size = 36 + data_size;
+    memcpy(buf + 4, &chunk_size, 4);
+    memcpy(buf + 8, "WAVE", 4);
+
+    // fmt subchunk
+    memcpy(buf + 12, "fmt ", 4);
+    uint32_t subchunk1_size = 16;
+    memcpy(buf + 16, &subchunk1_size, 4);
+    uint16_t audio_format = 1; // PCM
+    memcpy(buf + 20, &audio_format, 2);
+    memcpy(buf + 22, &num_channels, 2);
+    memcpy(buf + 24, &sample_rate, 4);
+
+    int bytes_per_sample = (bits_per_sample / 8) * num_channels;
+    int byte_rate = sample_rate * bytes_per_sample;
+    memcpy(buf + 28, &byte_rate, 4);
+    memcpy(buf + 32, &bytes_per_sample, 2);
+    memcpy(buf + 34, &bits_per_sample, 2);
+
+    // data subchunk
+    memcpy(buf + 36, "data", 4);
+    memcpy(buf + 40, &data_size, 4);
+
+    return 44;
+}
+
+bool ffmpeg_decode_audio(const std::string & ifname, std::vector<uint8_t> & wav_data, int out_sample_rate) {
+    {
+        const char * verbose = getenv("WHISPER_COMMON_FFMPEG_VERBOSE");
+        if (verbose && strcmp(verbose, "2") == 0) {
+            av_log_set_level(AV_LOG_DEBUG);
+        } else if (verbose && strcmp(verbose, "1") == 0) {
+            av_log_set_level(AV_LOG_VERBOSE);
+        } else {
+            av_log_set_level(AV_LOG_WARNING);
+        }
+    }
+
+    AVFormatContext * fmt_ctx = nullptr;
+    if (avformat_open_input(&fmt_ctx, ifname.c_str(), nullptr, nullptr) != 0) {
+        fprintf(stderr, "error: failed to open input file '%s'\n", ifname.c_str());
+        return true;
+    }
+
+    if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) {
+        fprintf(stderr, "error: failed to find stream information\n");
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    // Find the first audio stream
+    int audio_stream_idx = -1;
+    for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) {
+        if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+            audio_stream_idx = i;
+            break;
+        }
+    }
+
+    if (audio_stream_idx == -1) {
+        fprintf(stderr, "error: failed to find an audio stream in '%s'\n", ifname.c_str());
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    AVStream * audio_stream = fmt_ctx->streams[audio_stream_idx];
+
+    // Open the decoder
+    const AVCodec * codec = avcodec_find_decoder(audio_stream->codecpar->codec_id);
+    if (!codec) {
+        fprintf(stderr, "error: failed to find decoder for codec id %d\n", audio_stream->codecpar->codec_id);
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    AVCodecContext * codec_ctx = avcodec_alloc_context3(codec);
+    if (!codec_ctx) {
+        fprintf(stderr, "error: failed to allocate codec context\n");
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    if (avcodec_parameters_to_context(codec_ctx, audio_stream->codecpar) < 0) {
+        fprintf(stderr, "error: failed to copy codec parameters to context\n");
+        avcodec_free_context(&codec_ctx);
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    if (avcodec_open2(codec_ctx, codec, nullptr) < 0) {
+        fprintf(stderr, "error: failed to open codec\n");
+        avcodec_free_context(&codec_ctx);
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    // Setup resampler: convert to 16-bit signed PCM, mono, 16000 Hz
+    const enum AVSampleFormat out_sample_fmt = AV_SAMPLE_FMT_S16;
+
+    AVChannelLayout out_ch_layout = AV_CHANNEL_LAYOUT_MONO;
+
+    SwrContext * swr_ctx = nullptr;
+    if (swr_alloc_set_opts2(&swr_ctx, &out_ch_layout, out_sample_fmt, out_sample_rate,
+                            &codec_ctx->ch_layout, codec_ctx->sample_fmt, codec_ctx->sample_rate,
+                            0, nullptr) < 0) {
+        fprintf(stderr, "error: failed to allocate swr context\n");
+        avcodec_free_context(&codec_ctx);
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    if (swr_init(swr_ctx) < 0) {
+        fprintf(stderr, "error: failed to initialize swr context\n");
+        swr_free(&swr_ctx);
+        avcodec_free_context(&codec_ctx);
+        avformat_close_input(&fmt_ctx);
+        return true;
+    }
+
+    // Decode and resample
+    AVPacket * packet = av_packet_alloc();
+    AVFrame * frame = av_frame_alloc();
+
+    // Buffer to collect resampled output
+    std::vector<int16_t> pcm_data;
+
+    // Max output samples per swr_convert call
+    const int max_out_samples = 16 * 1024;
+    std::vector<int16_t> out_buffer(max_out_samples);
+
+    while (av_read_frame(fmt_ctx, packet) >= 0) {
+        if (packet->stream_index != audio_stream_idx) {
+            av_packet_unref(packet);
+            continue;
+        }
+
+        int ret = avcodec_send_packet(codec_ctx, packet);
+        av_packet_unref(packet);
+
+        if (ret < 0) {
+            continue;
+        }
+
+        while (ret >= 0) {
+            ret = avcodec_receive_frame(codec_ctx, frame);
+            if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+                break;
+            }
+            if (ret < 0) {
+                break;
+            }
+
+            // Resample
+            int out_samples = av_rescale_rnd(swr_get_delay(swr_ctx, out_sample_rate) + frame->nb_samples,
+                                              out_sample_rate, out_sample_rate, AV_ROUND_UP);
+            if (out_samples > (int)out_buffer.size()) {
+                out_buffer.resize(out_samples);
+            }
+
+            const uint8_t * in_data[16] = {0};
+            for (int p = 0; p < (int)codec_ctx->ch_layout.nb_channels && p < 16; p++) {
+                in_data[p] = frame->data[p];
+            }
+            uint8_t * out_data[16] = {0};
+            out_data[0] = (uint8_t *)out_buffer.data();
+
+            int got_samples = swr_convert(swr_ctx, out_data, out_samples, in_data, frame->nb_samples);
+            if (got_samples > 0) {
+                pcm_data.insert(pcm_data.end(), out_buffer.begin(), out_buffer.begin() + got_samples);
+            }
+        }
+    }
+
+    // Flush the decoder
+    avcodec_send_packet(codec_ctx, nullptr);
+    while (avcodec_receive_frame(codec_ctx, frame) >= 0) {
+        int out_samples = av_rescale_rnd(swr_get_delay(swr_ctx, out_sample_rate) + frame->nb_samples,
+                                          out_sample_rate, out_sample_rate, AV_ROUND_UP);
+        if (out_samples > (int)out_buffer.size()) {
+            out_buffer.resize(out_samples);
+        }
+        const uint8_t * in_data[16] = {0};
+        for (int p = 0; p < (int)codec_ctx->ch_layout.nb_channels && p < 16; p++) {
+            in_data[p] = frame->data[p];
+        }
+        uint8_t * out_data[16] = {0};
+        out_data[0] = (uint8_t *)out_buffer.data();
+
+        int got_samples = swr_convert(swr_ctx, out_data, out_samples, in_data, frame->nb_samples);
+        if (got_samples > 0) {
+            pcm_data.insert(pcm_data.end(), out_buffer.begin(), out_buffer.begin() + got_samples);
+        }
+    }
+
+    // Flush the resampler
+    uint8_t * out_data[16] = {0};
+    out_data[0] = (uint8_t *)out_buffer.data();
+    int flush_samples = swr_convert(swr_ctx, out_data, max_out_samples, nullptr, 0);
+    if (flush_samples > 0) {
+        pcm_data.insert(pcm_data.end(), out_buffer.begin(), out_buffer.begin() + flush_samples);
+    }
+
+    // Build WAV output
+    uint32_t data_size = pcm_data.size() * sizeof(int16_t);
+    wav_data.resize(44 + data_size);
+
+    wav_header_write(wav_data.data(), 1, out_sample_rate, 16, data_size);
+    memcpy(wav_data.data() + 44, pcm_data.data(), data_size);
+
+    // Cleanup
+    av_frame_free(&frame);
+    av_packet_free(&packet);
+    swr_free(&swr_ctx);
+    avcodec_free_context(&codec_ctx);
+    avformat_close_input(&fmt_ctx);
+
+    return false; // success
+}
+
+#endif // WHISPER_COMMON_FFMPEG

examples/generate-karaoke.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Simple tool to record audio from the microphone and generate a karaoke video
+# Usage:
+#
+#  cd whisper.cpp
+#  make
+#
+#  ./examples/generate-karaoke.sh [model] [step_ms]
+#
+# Press Ctrl+C to stop recording
+#
+
+executable="./build/bin/whisper-cli"
+model="base.en"
+model_path="models/ggml-$model.bin"
+
+# require sox and ffmpeg to be installed
+if ! command -v sox &> /dev/null
+then
+    echo "sox could not be found"
+    exit 1
+fi
+
+if ! command -v ffmpeg &> /dev/null
+then
+    echo "ffmpeg could not be found"
+    exit 2
+fi
+
+if [ ! -f "$executable" ]; then
+    echo "'$executable' does not exist. Please build it first."
+    exit 3
+fi
+
+if [ ! -f "$model_path" ]; then
+    echo "'$model_path' does not exist. Please download it first."
+    exit 4
+fi
+
+# record some raw audio
+sox -d rec.wav
+
+# run Whisper
+echo "Processing ..."
+${executable} -m models/ggml-base.en.bin rec.wav -owts > /dev/null 2>&1
+
+# generate Karaoke video
+echo "Generating video ..."
+source rec.wav.wts > /dev/null 2>&1
+
+# play the video
+echo "Playing ./rec16.wav.mp4 ..."
+ffplay -loglevel 0 -autoexit ./rec.wav.mp4
+
+echo "Done"
+exit 0

examples/grammar-parser.cpp

@@ -0,0 +1,423 @@
+#include "grammar-parser.h"
+#include <cstdint>
+#include <cwchar>
+#include <string>
+#include <utility>
+#include <stdexcept>
+#include <exception>
+
+namespace grammar_parser {
+    // NOTE: assumes valid utf8 (but checks for overrun)
+    // copied from whisper.cpp
+    static std::pair<uint32_t, const char *> decode_utf8(const char * src) {
+        static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
+        uint8_t  first_byte = static_cast<uint8_t>(*src);
+        uint8_t  highbits   = first_byte >> 4;
+        int      len        = lookup[highbits];
+        uint8_t  mask       = (1 << (8 - len)) - 1;
+        uint32_t value      = first_byte & mask;
+        const char * end    = src + len; // may overrun!
+        const char * pos    = src + 1;
+        for ( ; pos < end && *pos; pos++) {
+            value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);
+        }
+        return std::make_pair(value, pos);
+    }
+
+    static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) {
+        uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
+        auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id));
+        return result.first->second;
+    }
+
+    static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) {
+        uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
+        state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id;
+        return next_id;
+    }
+
+    static void add_rule(
+            parse_state & state,
+            uint32_t      rule_id,
+            const std::vector<whisper_grammar_element> & rule) {
+        if (state.rules.size() <= rule_id) {
+            state.rules.resize(rule_id + 1);
+        }
+        state.rules[rule_id] = rule;
+    }
+
+    static bool is_word_char(char c) {
+        return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || ('0' <= c && c <= '9');
+    }
+
+    static std::pair<uint32_t, const char *> parse_hex(const char * src, int size) {
+        const char * pos   = src;
+        const char * end   = src + size;
+        uint32_t     value = 0;
+        for ( ; pos < end && *pos; pos++) {
+            value <<= 4;
+            char c = *pos;
+            if ('a' <= c && c <= 'f') {
+                value += c - 'a' + 10;
+            } else if ('A' <= c && c <= 'F') {
+                value += c - 'A' + 10;
+            } else if ('0' <= c && c <= '9') {
+                value += c - '0';
+            } else {
+                break;
+            }
+        }
+        if (pos != end) {
+            throw std::runtime_error("expecting " + std::to_string(size) + " hex chars at " + src);
+        }
+        return std::make_pair(value, pos);
+    }
+
+    static const char * parse_space(const char * src, bool newline_ok) {
+        const char * pos = src;
+        while (*pos == ' ' || *pos == '\t' || *pos == '#' ||
+                (newline_ok && (*pos == '\r' || *pos == '\n'))) {
+            if (*pos == '#') {
+                while (*pos && *pos != '\r' && *pos != '\n') {
+                    pos++;
+                }
+            } else {
+                pos++;
+            }
+        }
+        return pos;
+    }
+
+    static const char * parse_name(const char * src) {
+        const char * pos = src;
+        while (is_word_char(*pos)) {
+            pos++;
+        }
+        if (pos == src) {
+            throw std::runtime_error(std::string("expecting name at ") + src);
+        }
+        return pos;
+    }
+
+    static std::pair<uint32_t, const char *> parse_char(const char * src) {
+        if (*src == '\\') {
+            switch (src[1]) {
+                case 'x': return parse_hex(src + 2, 2);
+                case 'u': return parse_hex(src + 2, 4);
+                case 'U': return parse_hex(src + 2, 8);
+                case 't': return std::make_pair('\t', src + 2);
+                case 'r': return std::make_pair('\r', src + 2);
+                case 'n': return std::make_pair('\n', src + 2);
+                case '\\':
+                case '"':
+                case '[':
+                case ']':
+                    return std::make_pair(src[1], src + 2);
+                default:
+                    throw std::runtime_error(std::string("unknown escape at ") + src);
+            }
+        } else if (*src) {
+            return decode_utf8(src);
+        }
+        throw std::runtime_error("unexpected end of input");
+    }
+
+    static const char * parse_alternates(
+            parse_state       & state,
+            const char        * src,
+            const std::string & rule_name,
+            uint32_t            rule_id,
+            bool                is_nested);
+
+    static const char * parse_sequence(
+            parse_state                        & state,
+            const char                         * src,
+            const std::string                  & rule_name,
+            std::vector<whisper_grammar_element> & out_elements,
+            bool                                 is_nested) {
+        size_t last_sym_start = out_elements.size();
+        const char * pos = src;
+        while (*pos) {
+            if (*pos == '"') { // literal string
+                pos++;
+                last_sym_start = out_elements.size();
+                while (*pos != '"') {
+                    auto char_pair = parse_char(pos);
+                         pos       = char_pair.second;
+                    out_elements.push_back({WHISPER_GRETYPE_CHAR, char_pair.first});
+                }
+                pos = parse_space(pos + 1, is_nested);
+            } else if (*pos == '[') { // char range(s)
+                pos++;
+                enum whisper_gretype start_type = WHISPER_GRETYPE_CHAR;
+                if (*pos == '^') {
+                    pos++;
+                    start_type = WHISPER_GRETYPE_CHAR_NOT;
+                }
+                last_sym_start = out_elements.size();
+                while (*pos != ']') {
+                    auto char_pair = parse_char(pos);
+                         pos       = char_pair.second;
+                    enum whisper_gretype type = last_sym_start < out_elements.size()
+                        ? WHISPER_GRETYPE_CHAR_ALT
+                        : start_type;
+
+                    out_elements.push_back({type, char_pair.first});
+                    if (pos[0] == '-' && pos[1] != ']') {
+                        auto endchar_pair = parse_char(pos + 1);
+                             pos          = endchar_pair.second;
+                        out_elements.push_back({WHISPER_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});
+                    }
+                }
+                pos = parse_space(pos + 1, is_nested);
+            } else if (is_word_char(*pos)) { // rule reference
+                const char * name_end    = parse_name(pos);
+                uint32_t     ref_rule_id = get_symbol_id(state, pos, name_end - pos);
+                pos = parse_space(name_end, is_nested);
+                last_sym_start = out_elements.size();
+                out_elements.push_back({WHISPER_GRETYPE_RULE_REF, ref_rule_id});
+            } else if (*pos == '(') { // grouping
+                // parse nested alternates into synthesized rule
+                pos = parse_space(pos + 1, true);
+                uint32_t sub_rule_id = generate_symbol_id(state, rule_name);
+                pos = parse_alternates(state, pos, rule_name, sub_rule_id, true);
+                last_sym_start = out_elements.size();
+                // output reference to synthesized rule
+                out_elements.push_back({WHISPER_GRETYPE_RULE_REF, sub_rule_id});
+                if (*pos != ')') {
+                    throw std::runtime_error(std::string("expecting ')' at ") + pos);
+                }
+                pos = parse_space(pos + 1, is_nested);
+            } else if (*pos == '*' || *pos == '+' || *pos == '?') { // repetition operator
+                if (last_sym_start == out_elements.size()) {
+                    throw std::runtime_error(std::string("expecting preceding item to */+/? at ") + pos);
+                }
+
+                // apply transformation to previous symbol (last_sym_start to end) according to
+                // rewrite rules:
+                // S* --> S' ::= S S' |
+                // S+ --> S' ::= S S' | S
+                // S? --> S' ::= S |
+                uint32_t sub_rule_id = generate_symbol_id(state, rule_name);
+                std::vector<whisper_grammar_element> sub_rule;
+                // add preceding symbol to generated rule
+                sub_rule.insert(
+                    sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end());
+                if (*pos == '*' || *pos == '+') {
+                    // cause generated rule to recurse
+                    sub_rule.push_back({WHISPER_GRETYPE_RULE_REF, sub_rule_id});
+                }
+                // mark start of alternate def
+                sub_rule.push_back({WHISPER_GRETYPE_ALT, 0});
+                if (*pos == '+') {
+                    // add preceding symbol as alternate only for '+' (otherwise empty)
+                    sub_rule.insert(
+                        sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end());
+                }
+                sub_rule.push_back({WHISPER_GRETYPE_END, 0});
+                add_rule(state, sub_rule_id, sub_rule);
+
+                // in original rule, replace previous symbol with reference to generated rule
+                out_elements.resize(last_sym_start);
+                out_elements.push_back({WHISPER_GRETYPE_RULE_REF, sub_rule_id});
+
+                pos = parse_space(pos + 1, is_nested);
+            } else {
+                break;
+            }
+        }
+        return pos;
+    }
+
+    static const char * parse_alternates(
+            parse_state       & state,
+            const char        * src,
+            const std::string & rule_name,
+            uint32_t            rule_id,
+            bool                is_nested) {
+        std::vector<whisper_grammar_element> rule;
+        const char * pos = parse_sequence(state, src, rule_name, rule, is_nested);
+        while (*pos == '|') {
+            rule.push_back({WHISPER_GRETYPE_ALT, 0});
+            pos = parse_space(pos + 1, true);
+            pos = parse_sequence(state, pos, rule_name, rule, is_nested);
+        }
+        rule.push_back({WHISPER_GRETYPE_END, 0});
+        add_rule(state, rule_id, rule);
+        return pos;
+    }
+
+    static const char * parse_rule(parse_state & state, const char * src) {
+        const char * name_end = parse_name(src);
+        const char * pos      = parse_space(name_end, false);
+        size_t       name_len = name_end - src;
+        uint32_t     rule_id  = get_symbol_id(state, src, name_len);
+        const std::string name(src, name_len);
+
+        if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) {
+            throw std::runtime_error(std::string("expecting ::= at ") + pos);
+        }
+        pos = parse_space(pos + 3, true);
+
+        pos = parse_alternates(state, pos, name, rule_id, false);
+
+        if (*pos == '\r') {
+            pos += pos[1] == '\n' ? 2 : 1;
+        } else if (*pos == '\n') {
+            pos++;
+        } else if (*pos) {
+            throw std::runtime_error(std::string("expecting newline or end at ") + pos);
+        }
+        return parse_space(pos, true);
+    }
+
+    parse_state parse(const char * src) {
+        try {
+            parse_state state;
+            const char * pos = parse_space(src, true);
+            while (*pos) {
+                pos = parse_rule(state, pos);
+            }
+            return state;
+        } catch (const std::exception & err) {
+            fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what());
+            return parse_state();
+        }
+    }
+
+    static void print_grammar_char(FILE * file, uint32_t c) {
+        if (0x20 <= c && c <= 0x7f) {
+            fprintf(file, "%c", static_cast<char>(c));
+        } else {
+            // cop out of encoding UTF-8
+            fprintf(file, "<U+%04X>", c);
+        }
+    }
+
+    static bool is_char_element(whisper_grammar_element elem) {
+        switch (elem.type) {
+            case WHISPER_GRETYPE_CHAR:           return true;
+            case WHISPER_GRETYPE_CHAR_NOT:       return true;
+            case WHISPER_GRETYPE_CHAR_ALT:       return true;
+            case WHISPER_GRETYPE_CHAR_RNG_UPPER: return true;
+            default:                           return false;
+        }
+    }
+
+    static void print_rule_binary(FILE * file, const std::vector<whisper_grammar_element> & rule) {
+        for (auto elem : rule) {
+            switch (elem.type) {
+                case WHISPER_GRETYPE_END:            fprintf(file, "END");            break;
+                case WHISPER_GRETYPE_ALT:            fprintf(file, "ALT");            break;
+                case WHISPER_GRETYPE_RULE_REF:       fprintf(file, "RULE_REF");       break;
+                case WHISPER_GRETYPE_CHAR:           fprintf(file, "CHAR");           break;
+                case WHISPER_GRETYPE_CHAR_NOT:       fprintf(file, "CHAR_NOT");       break;
+                case WHISPER_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break;
+                case WHISPER_GRETYPE_CHAR_ALT:       fprintf(file, "CHAR_ALT");       break;
+            }
+            switch (elem.type) {
+                case WHISPER_GRETYPE_END:
+                case WHISPER_GRETYPE_ALT:
+                case WHISPER_GRETYPE_RULE_REF:
+                    fprintf(file, "(%u) ", elem.value);
+                    break;
+                case WHISPER_GRETYPE_CHAR:
+                case WHISPER_GRETYPE_CHAR_NOT:
+                case WHISPER_GRETYPE_CHAR_RNG_UPPER:
+                case WHISPER_GRETYPE_CHAR_ALT:
+                    fprintf(file, "(\"");
+                    print_grammar_char(file, elem.value);
+                    fprintf(file, "\") ");
+                    break;
+            }
+        }
+        fprintf(file, "\n");
+    }
+
+    static void print_rule(
+            FILE     * file,
+            uint32_t   rule_id,
+            const std::vector<whisper_grammar_element> & rule,
+            const std::map<uint32_t, std::string>    & symbol_id_names) {
+        if (rule.empty() || rule.back().type != WHISPER_GRETYPE_END) {
+            throw std::runtime_error(
+                "malformed rule, does not end with WHISPER_GRETYPE_END: " + std::to_string(rule_id));
+        }
+        fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str());
+        for (size_t i = 0, end = rule.size() - 1; i < end; i++) {
+            whisper_grammar_element elem = rule[i];
+            switch (elem.type) {
+                case WHISPER_GRETYPE_END:
+                    throw std::runtime_error(
+                        "unexpected end of rule: " + std::to_string(rule_id) + "," +
+                        std::to_string(i));
+                case WHISPER_GRETYPE_ALT:
+                    fprintf(file, "| ");
+                    break;
+                case WHISPER_GRETYPE_RULE_REF:
+                    fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str());
+                    break;
+                case WHISPER_GRETYPE_CHAR:
+                    fprintf(file, "[");
+                    print_grammar_char(file, elem.value);
+                    break;
+                case WHISPER_GRETYPE_CHAR_NOT:
+                    fprintf(file, "[^");
+                    print_grammar_char(file, elem.value);
+                    break;
+                case WHISPER_GRETYPE_CHAR_RNG_UPPER:
+                    if (i == 0 || !is_char_element(rule[i - 1])) {
+                        throw std::runtime_error(
+                            "WHISPER_GRETYPE_CHAR_RNG_UPPER without preceding char: " +
+                            std::to_string(rule_id) + "," + std::to_string(i));
+                    }
+                    fprintf(file, "-");
+                    print_grammar_char(file, elem.value);
+                    break;
+                case WHISPER_GRETYPE_CHAR_ALT:
+                    if (i == 0 || !is_char_element(rule[i - 1])) {
+                        throw std::runtime_error(
+                            "WHISPER_GRETYPE_CHAR_ALT without preceding char: " +
+                            std::to_string(rule_id) + "," + std::to_string(i));
+                    }
+                    print_grammar_char(file, elem.value);
+                    break;
+            }
+            if (is_char_element(elem)) {
+                switch (rule[i + 1].type) {
+                    case WHISPER_GRETYPE_CHAR_ALT:
+                    case WHISPER_GRETYPE_CHAR_RNG_UPPER:
+                        break;
+                    default:
+                        fprintf(file, "] ");
+                }
+            }
+        }
+        fprintf(file, "\n");
+    }
+
+    void print_grammar(FILE * file, const parse_state & state) {
+        try {
+            std::map<uint32_t, std::string> symbol_id_names;
+            for (auto kv : state.symbol_ids) {
+                symbol_id_names[kv.second] = kv.first;
+            }
+            for (size_t i = 0, end = state.rules.size(); i < end; i++) {
+                // fprintf(file, "%zu: ", i);
+                // print_rule_binary(file, state.rules[i]);
+                print_rule(file, uint32_t(i), state.rules[i], symbol_id_names);
+                // fprintf(file, "\n");
+            }
+        } catch (const std::exception & err) {
+            fprintf(stderr, "\n%s: error printing grammar: %s\n", __func__, err.what());
+        }
+    }
+
+    std::vector<const whisper_grammar_element *> parse_state::c_rules() const {
+        std::vector<const whisper_grammar_element *> ret;
+        for (const auto & rule : rules) {
+            ret.push_back(rule.data());
+        }
+        return ret;
+    }
+}

examples/grammar-parser.h

@@ -0,0 +1,29 @@
+// Implements a parser for an extended Backus-Naur form (BNF), producing the
+// binary context-free grammar format specified by whisper.h. Supports character
+// ranges, grouping, and repetition operators. As an example, a grammar for
+// arithmetic might look like:
+//
+// root  ::= expr
+// expr  ::= term ([-+*/] term)*
+// term  ::= num | "(" space expr ")" space
+// num   ::= [0-9]+ space
+// space ::= [ \t\n]*
+
+#pragma once
+#include "whisper.h"
+#include <vector>
+#include <map>
+#include <cstdint>
+#include <string>
+
+namespace grammar_parser {
+    struct parse_state {
+        std::map<std::string, uint32_t>                   symbol_ids;
+        std::vector<std::vector<whisper_grammar_element>> rules;
+
+        std::vector<const whisper_grammar_element *>      c_rules() const;
+    };
+
+    parse_state parse(const char * src);
+    void print_grammar(FILE * file, const parse_state & state);
+}

examples/helpers.js

@@ -0,0 +1,191 @@
+// Common Javascript functions used by the examples
+
+function convertTypedArray(src, type) {
+    var buffer = new ArrayBuffer(src.byteLength);
+    var baseView = new src.constructor(buffer).set(src);
+    return new type(buffer);
+}
+
+var printTextarea = (function() {
+    var element = document.getElementById('output');
+    if (element) element.value = ''; // clear browser cache
+    return function(text) {
+        if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
+        console.log(text);
+        if (element) {
+            element.value += text + "\n";
+            element.scrollTop = element.scrollHeight; // focus on bottom
+        }
+    };
+})();
+
+async function clearCache() {
+    if (confirm('Are you sure you want to clear the cache?\nAll the models will be downloaded again.')) {
+        indexedDB.deleteDatabase(dbName);
+        location.reload();
+    }
+}
+
+// fetch a remote file from remote URL using the Fetch API
+async function fetchRemote(url, cbProgress, cbPrint) {
+    cbPrint('fetchRemote: downloading with fetch()...');
+
+    const response = await fetch(
+        url,
+        {
+            method: 'GET',
+        }
+    );
+
+    if (!response.ok) {
+        cbPrint('fetchRemote: failed to fetch ' + url);
+        return;
+    }
+
+    const contentLength = response.headers.get('content-length');
+    const total = parseInt(contentLength, 10);
+    const reader = response.body.getReader();
+
+    var chunks = [];
+    var receivedLength = 0;
+    var progressLast = -1;
+
+    while (true) {
+        const { done, value } = await reader.read();
+
+        if (done) {
+            break;
+        }
+
+        chunks.push(value);
+        receivedLength += value.length;
+
+        if (contentLength) {
+            cbProgress(receivedLength/total);
+
+            var progressCur = Math.round((receivedLength / total) * 10);
+            if (progressCur != progressLast) {
+                cbPrint('fetchRemote: fetching ' + 10*progressCur + '% ...');
+                progressLast = progressCur;
+            }
+        }
+    }
+
+    var position = 0;
+    var chunksAll = new Uint8Array(receivedLength);
+
+    for (var chunk of chunks) {
+        chunksAll.set(chunk, position);
+        position += chunk.length;
+    }
+
+    return chunksAll;
+}
+
+// load remote data
+// - check if the data is already in the IndexedDB
+// - if not, fetch it from the remote URL and store it in the IndexedDB
+function loadRemote(url, dst, size_mb, cbProgress, cbReady, cbCancel, cbPrint) {
+    if (!navigator.storage || !navigator.storage.estimate) {
+        cbPrint('loadRemote: navigator.storage.estimate() is not supported');
+    } else {
+        // query the storage quota and print it
+        navigator.storage.estimate().then(function (estimate) {
+            cbPrint('loadRemote: storage quota: ' + estimate.quota + ' bytes');
+            cbPrint('loadRemote: storage usage: ' + estimate.usage + ' bytes');
+        });
+    }
+
+    // check if the data is already in the IndexedDB
+    var rq = indexedDB.open(dbName, dbVersion);
+
+    rq.onupgradeneeded = function (event) {
+        var db = event.target.result;
+        if (db.version == 1) {
+            var os = db.createObjectStore('models', { autoIncrement: false });
+            cbPrint('loadRemote: created IndexedDB ' + db.name + ' version ' + db.version);
+        } else {
+            // clear the database
+            var os = event.currentTarget.transaction.objectStore('models');
+            os.clear();
+            cbPrint('loadRemote: cleared IndexedDB ' + db.name + ' version ' + db.version);
+        }
+    };
+
+    rq.onsuccess = function (event) {
+        var db = event.target.result;
+        var tx = db.transaction(['models'], 'readonly');
+        var os = tx.objectStore('models');
+        var rq = os.get(url);
+
+        rq.onsuccess = function (event) {
+            if (rq.result) {
+                cbPrint('loadRemote: "' + url + '" is already in the IndexedDB');
+                cbReady(dst, rq.result);
+            } else {
+                // data is not in the IndexedDB
+                cbPrint('loadRemote: "' + url + '" is not in the IndexedDB');
+
+                // alert and ask the user to confirm
+                if (!confirm(
+                    'You are about to download ' + size_mb + ' MB of data.\n' +
+                    'The model data will be cached in the browser for future use.\n\n' +
+                    'Press OK to continue.')) {
+                    cbCancel();
+                    return;
+                }
+
+                fetchRemote(url, cbProgress, cbPrint).then(function (data) {
+                    if (data) {
+                        // store the data in the IndexedDB
+                        var rq = indexedDB.open(dbName, dbVersion);
+                        rq.onsuccess = function (event) {
+                            var db = event.target.result;
+                            var tx = db.transaction(['models'], 'readwrite');
+                            var os = tx.objectStore('models');
+
+                            var rq = null;
+                            try {
+                                var rq = os.put(data, url);
+                            } catch (e) {
+                                cbPrint('loadRemote: failed to store "' + url + '" in the IndexedDB: \n' + e);
+                                cbCancel();
+                                return;
+                            }
+
+                            rq.onsuccess = function (event) {
+                                cbPrint('loadRemote: "' + url + '" stored in the IndexedDB');
+                                cbReady(dst, data);
+                            };
+
+                            rq.onerror = function (event) {
+                                cbPrint('loadRemote: failed to store "' + url + '" in the IndexedDB');
+                                cbCancel();
+                            };
+                        };
+                    }
+                });
+            }
+        };
+
+        rq.onerror = function (event) {
+            cbPrint('loadRemote: failed to get data from the IndexedDB');
+            cbCancel();
+        };
+    };
+
+    rq.onerror = function (event) {
+        cbPrint('loadRemote: failed to open IndexedDB');
+        cbCancel();
+    };
+
+    rq.onblocked = function (event) {
+        cbPrint('loadRemote: failed to open IndexedDB: blocked');
+        cbCancel();
+    };
+
+    rq.onabort = function (event) {
+        cbPrint('loadRemote: failed to open IndexedDB: abort');
+        cbCancel();
+    };
+}

examples/livestream.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# Transcribe audio livestream by feeding ffmpeg output to whisper.cpp at regular intervals
+# Idea by @semiformal-net
+# ref: https://github.com/ggml-org/whisper.cpp/issues/185
+#
+
+set -eo pipefail
+
+url="http://a.files.bbci.co.uk/media/live/manifesto/audio/simulcast/hls/nonuk/sbr_low/ak/bbc_world_service.m3u8"
+fmt=aac # the audio format extension of the stream (TODO: auto detect)
+step_s=30
+model="base.en"
+
+check_requirements()
+{
+    if ! command -v ./build/bin/whisper-cli &>/dev/null; then
+        echo "whisper.cpp main executable is required (make)"
+        exit 1
+    fi
+
+    if ! command -v ffmpeg &>/dev/null; then
+        echo "ffmpeg is required (https://ffmpeg.org)"
+        exit 1
+    fi
+}
+
+check_requirements
+
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 stream_url [step_s] [model]"
+    echo ""
+    echo "  Example:"
+    echo "    $0 $url $step_s $model"
+    echo ""
+    echo "No url specified, using default: $url"
+else
+    url="$1"
+fi
+
+if [ -n "$2" ]; then
+    step_s="$2"
+fi
+
+if [ -n "$3" ]; then
+    model="$3"
+fi
+
+# Whisper models
+models=( "tiny.en" "tiny" "base.en" "base" "small.en" "small" "medium.en" "medium" "large-v1" "large-v2" "large-v3" "large-v3-turbo" )
+
+# list available models
+function list_models {
+    printf "\n"
+    printf "  Available models:"
+    for model in "${models[@]}"; do
+        printf " $model"
+    done
+    printf "\n\n"
+}
+
+if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
+    printf "Invalid model: $model\n"
+    list_models
+
+    exit 1
+fi
+
+running=1
+
+trap "running=0" SIGINT SIGTERM
+
+printf "[+] Transcribing stream with model '$model', step_s $step_s (press Ctrl+C to stop):\n\n"
+
+# continuous stream in native fmt (this file will grow forever!)
+ffmpeg -loglevel quiet -y -re -probesize 32 -i $url -c copy /tmp/whisper-live0.${fmt} &
+if [ $? -ne 0 ]; then
+    printf "Error: ffmpeg failed to capture audio stream\n"
+    exit 1
+fi
+
+printf "Buffering audio. Please wait...\n\n"
+sleep $(($step_s))
+
+# do not stop script on error
+set +e
+
+i=0
+SECONDS=0
+while [ $running -eq 1 ]; do
+    # extract the next piece from the main file above and transcode to wav. -ss sets start time and nudges it by -0.5s to catch missing words (??)
+    err=1
+    while [ $err -ne 0 ]; do
+        if [ $i -gt 0 ]; then
+            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.${fmt} -y -ar 16000 -ac 1 -c:a pcm_s16le -ss $(($i*$step_s-1)).5 -t $step_s /tmp/whisper-live.wav 2> /tmp/whisper-live.err
+        else
+            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.${fmt} -y -ar 16000 -ac 1 -c:a pcm_s16le -ss $(($i*$step_s)) -t $step_s /tmp/whisper-live.wav 2> /tmp/whisper-live.err
+        fi
+        err=$(cat /tmp/whisper-live.err | wc -l)
+    done
+
+    ./build/bin/whisper-cli -t 8 -m ./models/ggml-${model}.bin -f /tmp/whisper-live.wav --no-timestamps -otxt 2> /tmp/whispererr | tail -n 1
+
+    while [ $SECONDS -lt $((($i+1)*$step_s)) ]; do
+        sleep 1
+    done
+    ((i=i+1))
+done
+
+killall -v ffmpeg
+killall -v whisper-cli

examples/lsp/CMakeLists.txt

@@ -0,0 +1,10 @@
+if (WHISPER_SDL2)
+    # stream
+    set(TARGET whisper-lsp)
+    add_executable(${TARGET} lsp.cpp)
+
+    include(DefaultTargetOptions)
+
+    target_link_libraries(${TARGET} PRIVATE common json_cpp common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})
+    install(TARGETS ${TARGET} RUNTIME)
+endif ()

examples/lsp/README.md

@@ -0,0 +1,104 @@
+# Language Server
+
+This example consists of a simple language server to expose both unguided
+and guided (command) transcriptions by sending json messages over stdout/stdin
+as well as a rather robust vim plugin that makes use of the language server.
+
+## Vim plugin quick start
+
+Compile the language server with
+
+```bash
+make lsp
+```
+Install the plugin itself by copying or symlinking whisper.vim into ~/.vim/autoload/
+
+In your vimrc, set the path of your whisper.cpp directory and optionally add some keybinds.
+
+```vim
+let g:whisper_dir = "~/whisper.cpp"
+" Start listening for commands when Ctrl - g is pressed in normal mode
+nnoremap <C-G> call whisper#requestCommands()<CR>
+" Start unguided transcription when Ctrl - g is pressed in insert mode
+inoremap <C-G> <Cmd>call whisper#doTranscription()<CR>
+```
+
+## Vim plugin usage
+
+The vim plugin was designed to closely follow the mnemonics of vim
+
+`s:spoken_dict` is used to translate keys to their spoken form.
+
+
+Keys corresponding to a string use that spoken value normally and when a motion is expected, but use the key itself when a character is expected.  
+Keys corresponding to a dict, like `i`, can have manual definitions given to each possible commandset.
+
+0 is normal (insert), 1 is motion (inside), 2 is it's usage as a single key ([till] i), and 3 is it's usage in an area selection (s -> [around] sentence)
+
+Some punctuation items, like `-` are explicitly given pronunciations to prevent them from being picked as punctuation instead of an actual command word.
+
+Not all commands will tokenize to a single token and this can interfere with interpretation. "yank" as an example, takes multiple tokens and correspondingly, will give more accurate detection when only the first "ya" is used. While it could be changed to something else that is a single token (copy), value was placed on maintaining vim mnemonics.
+
+Commands that would normally move the editor into insert mode (insert, append, open, change) will begin unguided transcription.
+Unguided transcription will end when a speech segment ends in exit.
+Presence of punctuation can be designated by whether or not you add a pause between the previous speech segment and exit.
+Exiting only occurs if exit is the last word, so "Take the first exit on your right" would not cause transcription to end.
+
+After a command is evaluated, the plugin will continue listening for the next command.
+
+While in command mode, "Exit" will end listening.
+
+A best effort approach is taken to keep track of audio that is recorded while a previous chunk is still processing and immediately interpret it afterwards, but the current voice detection still needs a fairly sizable gap to determine when a command has been spoken.
+
+Log information is sent to a special `whisper_log` buffer and can be accessed with
+```vim
+:e whisper_log
+```
+
+## Vim plugin configuration
+
+`g:whisper_dir`  
+A full path to the whisper.cpp repo. It can be expanded in the definition like so:
+```vim
+let g:whisper_dir = expand("~/whisper.cpp/")
+```
+(The WHISPER_CPP_HOME environment variable is also checked for users of the existing whisper.nvim script)
+
+`g:whisper_lsp_path`  
+Can be used to manually set the path to the language server.
+If not defined, it will be inferred from the above whisper_dir
+
+`g:whisper_model_path`  
+A full path to the model to load. If not defined, it will default to ggml-base.en.bin
+
+`g:whisper_user_commands`  
+A dictionary of spoken commands that correspond to either strings or funcrefs.
+This can be used to create connections with other user plugins, for example
+```vim
+let g:whisper_user_commands = {"gen": "llama#doLlamaGen"}
+```
+will trigger the llama.cpp plugin to begin generation when "gen" is spoken
+
+## Language server methods
+
+`registerCommandset`  
+`params` is a list of strings that should be checked for with this commandset. The server prepends a space to these strings before tokenizing.  
+Responds with  
+`result.index` an integer index for the commandset registered, which should be included when initiating a guided transcription to select this commandset.
+Will return an error if any of the commands in the commandset have duplicate tokenizations
+
+`guided`  
+`params.commandset_index` An index returned by a corresponding commandset registration. If not set, the most recently registered commandset is used.
+`params.timestamp` A positive unsigned integer which designates a point in time which audio should begin processing from. If left blank, the start point of audio processing will be the moment the message is received. This should be left blank unless you have a timestamp from a previous response.  
+Responds with  
+`result.command_index` The numerical index (starting from 0) of the detected command in the selected commandset
+`result.command_text` A string containing the command as provided in the commandset
+`result.timestamp` A positive unsigned integer that designates the point in time which audio stopped being processed at. Pass this timestamp back in a subsequent message to mask the latency of transcription.
+
+`unguided`  
+`params.no_context` Sets the corresponding whisper `no_context` param. Defaults to true. Might provide more accurate results for consecutive unguided transcriptions if those after the first are set to false.
+`params.prompt` If provided, sets the initial prompt used during transcription.
+`params.timestamp` A positive unsigned integer which designates a point in time which audio should begin processing from. If left blank, the start point of audio processing will be the moment the message is received. This should be left blank unless you have a timestamp from a previous response.  
+Responds with  
+`result.transcription` A string containing the transcribed text.  N.B. This will almost always start with a space due to how text is tokenized.
+`result.timestamp` A positive unsigned integer that designates the point in time which audio stopped being processed at. Pass this timestamp back in a subsequent message to mask the latency of transcription.

examples/lsp/lsp.cpp

@@ -0,0 +1,471 @@
+#include "common.h"
+#include "common-sdl.h"
+#include "whisper.h"
+#include "json.hpp"
+
+#include <cassert>
+#include <chrono>
+#include <cstdio>
+#include <deque>
+#include <iostream>
+#include <set>
+#include <string>
+#include <thread>
+#include <vector>
+
+using json = nlohmann::json;
+
+// command-line parameters
+struct whisper_params {
+    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t prompt_ms  = 5000;
+    int32_t command_ms = 8000;
+    int32_t capture_id = -1;
+    int32_t max_tokens = 32;
+    int32_t audio_ctx  = 0;
+
+    float vad_thold    = 0.6f;
+    float freq_thold   = 100.0f;
+
+    bool translate     = false;
+    bool print_special = false;
+    bool print_energy  = false;
+    bool use_gpu       = true;
+    bool flash_attn    = true;
+
+    std::string language  = "en";
+    std::string model     = "models/ggml-base.en.bin";
+};
+struct command {
+    std::vector<whisper_token> tokens;
+    std::string plaintext;
+};
+struct commandset {
+    std::vector<struct command> commands;
+    std::vector<whisper_token> prompt_tokens;
+    // TODO: Store longest command?
+    // Multi-token commands should have probabilities of subsequent logits
+    // given that the prior logit is correct.
+    // In this case, all commands must be iterated.
+    // This however, is likely highly involved as different tokens
+    // almost certainly have different spoken lengths
+    // It would also have performance implications equivalent to a beam search
+};
+
+void whisper_print_usage(int argc, char ** argv, const whisper_params & params);
+
+static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-h" || arg == "--help") {
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+        else if (arg == "-t"      || arg == "--threads")       { params.n_threads     = std::stoi(argv[++i]); }
+        else if (arg == "-pms"    || arg == "--prompt-ms")     { params.prompt_ms     = std::stoi(argv[++i]); }
+        else if (arg == "-cms"    || arg == "--command-ms")    { params.command_ms    = std::stoi(argv[++i]); }
+        else if (arg == "-c"      || arg == "--capture")       { params.capture_id    = std::stoi(argv[++i]); }
+        else if (arg == "-mt"     || arg == "--max-tokens")    { params.max_tokens    = std::stoi(argv[++i]); }
+        else if (arg == "-ac"     || arg == "--audio-ctx")     { params.audio_ctx     = std::stoi(argv[++i]); }
+        else if (arg == "-vth"    || arg == "--vad-thold")     { params.vad_thold     = std::stof(argv[++i]); }
+        else if (arg == "-fth"    || arg == "--freq-thold")    { params.freq_thold    = std::stof(argv[++i]); }
+        else if (arg == "-tr"     || arg == "--translate")     { params.translate     = true; }
+        else if (arg == "-ps"     || arg == "--print-special") { params.print_special = true; }
+        else if (arg == "-pe"     || arg == "--print-energy")  { params.print_energy  = true; }
+        else if (arg == "-ng"     || arg == "--no-gpu")        { params.use_gpu       = false; }
+        else if (arg == "-fa"     || arg == "--flash-attn")    { params.flash_attn    = true; }
+        else if (arg == "-nfa"    || arg == "--no-flash-attn") { params.flash_attn    = false; }
+        else if (arg == "-l"      || arg == "--language")      { params.language      = argv[++i]; }
+        else if (arg == "-m"      || arg == "--model")         { params.model         = argv[++i]; }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,         --help           [default] show this help message and exit\n");
+    fprintf(stderr, "  -t N,       --threads N      [%-7d] number of threads to use during computation\n", params.n_threads);
+    fprintf(stderr, "  -pms N,     --prompt-ms N    [%-7d] prompt duration in milliseconds\n",             params.prompt_ms);
+    fprintf(stderr, "  -cms N,     --command-ms N   [%-7d] command duration in milliseconds\n",            params.command_ms);
+    fprintf(stderr, "  -c ID,      --capture ID     [%-7d] capture device ID\n",                           params.capture_id);
+    fprintf(stderr, "  -mt N,      --max-tokens N   [%-7d] maximum number of tokens per audio chunk\n",    params.max_tokens);
+    fprintf(stderr, "  -ac N,      --audio-ctx N    [%-7d] audio context size (0 - all)\n",                params.audio_ctx);
+    fprintf(stderr, "  -vth N,     --vad-thold N    [%-7.2f] voice activity detection threshold\n",        params.vad_thold);
+    fprintf(stderr, "  -fth N,     --freq-thold N   [%-7.2f] high-pass frequency cutoff\n",                params.freq_thold);
+    fprintf(stderr, "  -tr,        --translate      [%-7s] translate from source language to english\n",   params.translate ? "true" : "false");
+    fprintf(stderr, "  -ps,        --print-special  [%-7s] print special tokens\n",                        params.print_special ? "true" : "false");
+    fprintf(stderr, "  -pe,        --print-energy   [%-7s] print sound energy (for debugging)\n",          params.print_energy ? "true" : "false");
+    fprintf(stderr, "  -ng,        --no-gpu         [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -fa,        --flash-attn     [%-7s] enable flash attention\n",                      params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -nfa,       --no-flash-attn  [%-7s] disable flash attention\n",                     params.flash_attn ? "false" : "true");
+    fprintf(stderr, "  -l LANG,    --language LANG  [%-7s] spoken language\n",                             params.language.c_str());
+    fprintf(stderr, "  -m FNAME,   --model FNAME    [%-7s] model path\n",                                  params.model.c_str());
+    fprintf(stderr, "\n");
+}
+static uint64_t wait_for_vad(audio_async & audio, json jparams, const whisper_params & params, uint64_t maxlength_ms, std::vector<float> & pcmf32) {
+    using namespace std::chrono;
+    uint64_t time_now = time_point_cast<milliseconds>(system_clock::now()).time_since_epoch().count();
+    uint64_t start_time = time_now;
+    if (jparams.contains("timestamp")) {
+        start_time = jparams.at("timestamp");
+    }
+    if(time_now - start_time < 500) {
+        //wait for a backlog of audio
+        std::this_thread::sleep_for(milliseconds(500 - (time_now - start_time)));
+        time_now = time_point_cast<milliseconds>(system_clock::now()).time_since_epoch().count();
+    } else if (time_now - start_time > 1000) {
+        audio.get(time_now-start_time, pcmf32);
+        size_t max_offset = pcmf32.size() - WHISPER_SAMPLE_RATE;
+        for(size_t offset=0;offset < max_offset;offset+=WHISPER_SAMPLE_RATE/10) {
+            std::vector<float> audio_chunk(&pcmf32[offset], &pcmf32[offset+WHISPER_SAMPLE_RATE]);
+            if(::vad_simple(audio_chunk, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {
+                pcmf32.resize(offset+WHISPER_SAMPLE_RATE);
+                if (offset*1000/WHISPER_SAMPLE_RATE+1000 > maxlength_ms) {
+                    //remove samples from the beginning
+                    pcmf32.erase(pcmf32.begin(),pcmf32.end()-(maxlength_ms*WHISPER_SAMPLE_RATE/1000));
+                    fprintf(stderr, "Shortened samples");
+                }
+                return start_time + offset*1000/WHISPER_SAMPLE_RATE+1000;
+            }
+        }
+    }
+    size_t window_duration = std::max((uint64_t)1000, time_now-start_time);
+    audio.get(window_duration, pcmf32);
+    while (!::vad_simple(pcmf32, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {
+        std::this_thread::sleep_for(milliseconds(100));
+        time_now = time_point_cast<milliseconds>(system_clock::now()).time_since_epoch().count();
+        window_duration = std::max((uint64_t)1000,time_now-start_time);
+        audio.get(window_duration, pcmf32);
+    }
+    if (time_now - start_time > maxlength_ms) {
+        audio.get(maxlength_ms, pcmf32);
+    } else {
+        audio.get(time_now - start_time, pcmf32);
+    }
+
+    return time_now;
+}
+
+static json unguided_transcription(struct whisper_context * ctx, audio_async &audio, json jparams, const whisper_params &params) {
+    std::vector<whisper_token> prompt_tokens;
+    std::vector<float> pcmf32;
+    uint64_t unprocessed_audio_timestamp = wait_for_vad(audio, jparams, params, 10000U, pcmf32);
+
+    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+    if (jparams.contains("prompt")) {
+        // unlikely to see much use. Under normal circumstances, no_context would be set to false
+        std::string prompt = jparams.at("prompt");
+        prompt_tokens.resize(1024);
+        int n = whisper_tokenize(ctx, prompt.c_str(), prompt_tokens.data(), 1024);
+        prompt_tokens.resize(n);
+
+        wparams.prompt_tokens    = prompt_tokens.data();
+        wparams.prompt_n_tokens  = prompt_tokens.size();
+    }
+    wparams.print_progress   = false;
+    wparams.print_special    = params.print_special;
+    wparams.print_realtime   = false;
+    wparams.print_timestamps = false;
+    wparams.translate        = params.translate;
+    wparams.no_context       = jparams.value("no_context", true);
+    wparams.single_segment   = true;
+    wparams.max_tokens       = params.max_tokens;
+    wparams.language         = params.language.c_str();
+    wparams.n_threads        = params.n_threads;
+
+    wparams.audio_ctx        = params.audio_ctx;
+    wparams.suppress_nst     = true;
+    // run the transformer and a single decoding pass
+    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
+        fprintf(stderr, "%s: ERROR: whisper_full() failed\n", __func__);
+        throw json{
+            {"code", -32803},
+            {"message", "ERROR: whisper_full() failed"}
+        };
+    }
+    std::string result = whisper_full_get_segment_text(ctx,0);
+    return json {
+        {"transcription", result},
+        {"timestamp", unprocessed_audio_timestamp}
+    };
+}
+
+// command-list mode
+// guide the transcription to match the most likely command from a provided list
+static json guided_transcription(struct whisper_context * ctx, audio_async &audio, const whisper_params &params, json jparams, std::vector<struct commandset> commandset_list) {
+    struct commandset cs = commandset_list[jparams.value("commandset_index", commandset_list.size()-1)];
+    std::vector<float> pcmf32;
+    uint64_t unprocessed_audio_timestamp = wait_for_vad(audio, jparams, params, 2000U, pcmf32);
+
+    fprintf(stderr, "%s: Speech detected! Processing ...\n", __func__);
+    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+
+    wparams.print_progress   = false;
+    wparams.print_special    = params.print_special;
+    wparams.print_realtime   = false;
+    wparams.print_timestamps = false;
+    wparams.translate        = params.translate;
+    wparams.no_context       = true;
+    wparams.single_segment   = true;
+    wparams.max_tokens       = 1;
+    wparams.language         = params.language.c_str();
+    wparams.n_threads        = params.n_threads;
+
+    wparams.audio_ctx        = params.audio_ctx;
+
+    // TODO: Do some time testing. Does an overly long prompt slow down processing?
+    // Set up command sets/precompute prompts
+    wparams.prompt_tokens    = cs.prompt_tokens.data();
+    wparams.prompt_n_tokens  = cs.prompt_tokens.size();
+    // TODO: properly expose as option
+    wparams.suppress_nst     = true;
+
+    // run the transformer and a single decoding pass
+    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
+        fprintf(stderr, "%s: ERROR: whisper_full() failed\n", __func__);
+        throw json{
+            {"code", -32803},
+            {"message", "ERROR: whisper_full() failed"}//TODO: format string (sprintf?)
+        };
+    }
+
+    // estimate command probability
+    // NOTE: not optimal
+    {
+        const auto * logits = whisper_get_logits(ctx);
+
+        std::vector<float> probs(whisper_n_vocab(ctx), 0.0f);
+
+        // compute probs from logits via softmax
+        {
+            float max = -1e9;
+            for (int i = 0; i < (int) probs.size(); ++i) {
+                max = std::max(max, logits[i]);
+            }
+
+            float sum = 0.0f;
+            for (int i = 0; i < (int) probs.size(); ++i) {
+                probs[i] = expf(logits[i] - max);
+                sum += probs[i];
+            }
+
+            for (int i = 0; i < (int) probs.size(); ++i) {
+                probs[i] /= sum;
+            }
+        }
+
+        std::vector<std::pair<float, int>> probs_id;
+
+        // In my testing, the most verbose token is always the desired.
+        // TODO: Trim commandset struct once efficacy has been verified
+        for (int i = 0; i < (int) cs.commands.size(); ++i) {
+            probs_id.emplace_back(probs[cs.commands[i].tokens[0]], i);
+        }
+
+        // sort descending
+        {
+            using pair_type = decltype(probs_id)::value_type;
+            std::sort(probs_id.begin(), probs_id.end(), [](const pair_type & a, const pair_type & b) {
+                    return a.first > b.first;
+                    });
+        }
+        int id = probs_id[0].second;
+        return json{
+            {"command_index", id},
+                {"command_text", cs.commands[id].plaintext},
+                {"timestamp", unprocessed_audio_timestamp},
+        };
+    }
+}
+
+static json register_commandset(struct whisper_context * ctx, json jparams, std::vector<struct commandset> &commandset_list) {
+    // TODO: check for token collision
+    struct commandset cs;
+
+    std::string  k_prompt = " select one from the available words: ";
+    std::set<whisper_token> token_set;
+    whisper_token tokens[32];
+    for (std::string s : jparams) {
+        std::vector<whisper_token> token_vec;
+        // The existing command implementation uses a nested for loop to tokenize single characters
+        // I fail to see the purpose of this when ' a' has a wholly different pronunciation than the start of ' apple'
+        const int n = whisper_tokenize(ctx, (" " + s).c_str(), tokens, 32);
+        if (n < 0) {
+            fprintf(stderr, "%s: error: failed to tokenize command '%s'\n", __func__, s.c_str());
+            return 3;
+        }
+        token_vec.push_back(tokens[0]);
+        if (!token_set.insert(tokens[0]).second) {
+            fprintf(stderr, "%s: warning: %s is a duplicate of an existing token\n", __func__, s.c_str());
+            throw json{
+                {"code",-31000},
+                {"message", "Duplicate token in token set: " + s}
+            };
+        }
+        if (n > 1) {// empty string if n=0? Should never occur
+            fprintf(stderr, "%s: error: command is more than a single token: %s\n", __func__, s.c_str());
+        }
+        struct command command = {token_vec, s};
+        cs.commands.push_back(command);
+        k_prompt += s;
+    }
+    k_prompt = k_prompt.substr(0,k_prompt.length()-2) + ". Selected word:";
+    cs.prompt_tokens.resize(1024);
+    int n = whisper_tokenize(ctx, k_prompt.c_str(), cs.prompt_tokens.data(), 1024);
+    cs.prompt_tokens.resize(n);
+    // prepare response
+    int index = commandset_list.size();
+    commandset_list.push_back(cs);
+    return json{{"index",index}};
+}
+
+static json seek(struct whisper_context * /*ctx*/, audio_async & /*audio*/, json /*params*/) {
+    // whisper_state has the pertinent offsets, but there also seem to be a large
+    // number of scratch buffers that would prevent rewinding context in a manner similar to llama
+    // I'll give this a another pass once everything else is implemented,
+    // but for now, it's unsupported
+    throw json {
+        {"code", -32601},
+            {"message", "Seeking is not yet supported."}
+    };
+}
+
+static json parse_job(const json &body, struct whisper_context * ctx, audio_async &audio, const whisper_params &params, std::vector<struct commandset> &commandset_list) {
+    // See: https://www.jsonrpc.org/specification
+    json id = body.at("id");
+    try {
+        std::string version = body.at("jsonrpc");
+        if (version != "2.0") {
+            // unsupported version
+            throw json{
+                {"code", -3260},
+                {"message", "invalid jsonrpc version"}
+            };
+        }
+        std::string method = body.at("method");
+        json jparams = json{{"dummy", "dummy"}};
+        if (body.contains("params"))
+            jparams = body.at("params");
+        json res;
+        // TODO: be consistent about argument order
+        fprintf(stderr, "Dispatching a job\n");
+        if (method == "unguided")                { res = unguided_transcription(ctx, audio, jparams, params); }
+        else if (method == "guided")             { res = guided_transcription(ctx, audio, params, jparams, commandset_list); }
+        else if (method == "seek")               { res = seek(ctx, audio, jparams); }
+        else if (method == "registerCommandset") { res = register_commandset(ctx, jparams, commandset_list); }
+        else if (method == "echo")               { res = jparams; }
+
+
+        return json{
+            {"jsonrpc", "2.0"},
+                {"result", res},
+                {"id", id}
+        };
+    } catch(json ex) {
+        return json {
+            {"jsonrpc", "2.0"},
+                {"error", ex},
+                {"id", id}
+        };
+    }
+}
+
+static void process_loop(struct whisper_context * ctx, audio_async &audio, const whisper_params &params) {
+    std::deque<json> jobqueue;
+    std::vector<struct commandset> commandset_list;
+    while (true) {
+        // For eventual cancellation support, shouldn't block if job exists
+        if (std::cin.rdbuf()->in_avail() > 22 || jobqueue.size() == 0) {
+            int content_length;
+            if (scanf("Content-Length: %d", &content_length) != 1) {
+                fprintf(stderr, "Could not read input: %d", std::cin.peek());
+                return;
+            }
+            // scanf leaves the new lines intact
+            std::cin.ignore(2);
+            if (std::cin.peek() != 13) {
+                // Content-Type. jsonrpc necessitates utf8.
+                std::cin.ignore(200,10);
+            }
+            std::cin.ignore(2);
+            // A message is being sent and blocking is acceptable
+            std::string content(content_length,'\0');
+            std::cin.read(&content[0], content_length);
+            json job = json::parse(content);
+            // TODO: Some messages(cancellation) should skip queue here
+            if (job.is_array()) {
+                // response must also be batched. Will implement later
+                // for (subjob : job.begin())
+                // TODO: At the very least respond with an unsupported error.
+            } else {
+                jobqueue.push_back(job);
+            }
+        }
+        assert(jobqueue.size() > 0);
+        json job = jobqueue.front();
+        json resp = parse_job(job, ctx, audio, params, commandset_list);
+        if (resp != "unfinished") {
+            jobqueue.pop_front();
+            // send response
+            std::string data = resp.dump(-1, ' ', false, json::error_handler_t::replace);
+            fprintf(stdout, "Content-Length: %d\r\n\r\n%s\n", (int)data.length()+1, data.c_str());
+            std::cout.flush();
+
+        }
+    }
+}
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    whisper_params params;
+    if (whisper_params_parse(argc, argv, params) == false) {
+        return 1;
+    }
+
+    if (whisper_lang_id(params.language.c_str()) == -1) {
+        fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+        whisper_print_usage(argc, argv, params);
+        exit(0);
+    }
+
+    // whisper init
+    struct whisper_context_params cparams = whisper_context_default_params();
+
+    cparams.use_gpu    = params.use_gpu;
+    cparams.flash_attn = params.flash_attn;
+
+    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
+    // init audio
+
+    audio_async audio(30*1000);
+    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {
+        fprintf(stderr, "%s: audio.init() failed!\n", __func__);
+        return 1;
+    }
+
+    audio.resume();
+    // TODO: Investigate why this is required. An extra second of startup latency is not great
+    // wait for 1 second to avoid any buffered noise
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    audio.clear();
+    // TODO: consider some sort of indicator to designate loading has finished?
+    // Potentially better for the client to just start with a non-blocking message (register commands)
+    process_loop(ctx, audio, params);
+
+    audio.pause();
+    whisper_print_timings(ctx);
+    whisper_free(ctx);
+
+    return 0;
+}

examples/lsp/whisper.vim

@@ -0,0 +1,362 @@
+if !exists("g:whisper_dir")
+    let g:whisper_dir = expand($WHISPER_CPP_HOME)
+    if g:whisper_dir == ""
+        echoerr "Please provide a path to the whisper.cpp repo in either the $WHISPER_CPP_HOME environment variable, or g:whisper_dir"
+    endif
+endif
+if !exists("g:whisper_lsp_path")
+    let g:whisper_lsp_path = g:whisper_dir .. "lsp"
+    if !filereadable(g:whisper_lsp_path)
+        echoerr "Was not able to locate a lsp executable at: " .. g:whisper_lsp_path
+        throw "Executable not found"
+    endif
+endif
+if !exists("g:whisper_model_path")
+    " TODO: allow custom paths relative to the repo dir
+    let g:whisper_model_path = g:whisper_dir .. "models/ggml-base.en.bin"
+    if !filereadable(g:whisper_model_path)
+        echoerr "Could not find model at: " .. g:whisper_model_path
+        throw "Model not found"
+    endif
+endif
+let s:output_buffer = bufnr("whisper_log", v:true)
+call setbufvar(s:output_buffer,"&buftype","nofile")
+let s:lsp_command = [g:whisper_lsp_path,"-m",g:whisper_model_path]
+" For faster execution. TODO: server load multiple models/run multiple servers?
+" let s:lsp_command = [g:whisper_lsp_path, "-m", g:whisper_dir .. "models/ggml-tiny.en.bin", "-ac", "128"]
+
+" requestCommands([params_dict])
+func whisper#requestCommands(...)
+    let l:req = {"method": "guided", "params": {"commandset_index": 0}}
+    if a:0 > 0
+        call extend(l:req.params, a:1)
+    endif
+    let resp = ch_sendexpr(g:lsp_job, l:req, {"callback": function("s:commandCallback", [l:req.params, 0])})
+endfunction
+
+" doTranscription([params_dict])
+func whisper#doTranscription(...)
+    let l:req = {"method": "unguided", "params": {}}
+    if a:0 > 0
+        call extend(l:req.params, a:1)
+    endif
+    let resp = ch_sendexpr(g:lsp_job, l:req, {"callback": function("s:transcriptionCallback", [function("s:insertText"),function("s:endTranscription")])})
+endfunction
+
+" For testing
+func whisper#uppertest(cha)
+    echo tr(a:cha, s:c_lowerkeys, s:c_upperkeys)
+endfunction
+
+
+" (upper, exit, count, motion, command, insert/append, save run) "base"
+" (upper, exit, count, motion, command, inside/around)           "motion/visual"
+" (upper, exit, count, motion, line,    inside/around)           "command already entered"
+" (upper, exit, key,                                 )           "from/till"
+
+" upper and lower keys is used to translate between cases with tr
+" Must be sunchronized
+let s:c_lowerkeys = "1234567890-=qwertyuiop[]\\asdfghjkl;'zxcvbnm,./\""
+let s:c_upperkeys = "!@#$%^&*()_+QWERTYUIOP{}|ASDFGHJKL:\"ZXCVBNM<>?'"
+let s:c_count = split("1234567890\"",'\zs')
+let s:c_command = split("ryuogpdxcv.iam", '\zs')
+let s:c_motion = split("wetf'hjklnb$^)",'\zs')
+" object words: Word, Sentence, Paragraph, [, (, <, Tag, {. ", '
+let s:c_area = split("wsp])>t}\"'",'\zs')
+"Special commands.
+let s:c_special_always = ["exit", "upper"]
+let s:c_special_normal = ["save", "run", "space"]
+
+" If not in dict, key is spoken word,
+" If key resolves to string, value is used for normal/motion, but key for chars
+" If key resolves to dict, {0: "normal",1: "motion",2:"single char",3: "area"}
+" Missing entries fall back as follows {0: "required", 1: 0, 2: "key", 3: 0}
+let s:spoken_dict = {"w": "word", "e": "end", "r": "replace", "t": {0: "till", 3: "tag"}, "y": "yank", "u": "undo", "i": {0: "insert", 1: "inside"}, "o": "open", "p": {0: "paste", 3: "paragraph"},  "a": {0: "append", 1: "around"}, "s": {0: "substitute", 3: "sentence"}, "d": "delete", "f": "from", "g": "go", "h": "left", "j": "down", "k": "up", "l": "right", "c": "change", "v": "visual", "b": "back", "n": "next", "m": "mark", ".": {0: "repeat", 2: "period"}, "]": {0: "bracket", 2: "bracket"}, "'": {0: "jump", 2: "apostrophe", 3:  "apostrophe"}, '"': {0: 'register', 2: "quotation", 3: "quotation"}, "-": {0: "minus", 2: "minus"}, "$": {0: "dollar", 2: "dollar"}, "^": {0: "carrot", 2: "carrot"}, ")": {0: "sentence", 2: "parenthesis",  3: "parenthesis"}, "}": {0: "paragraph", 2: "brace", 3: "brace"}, ">": {0: "indent", 2: "angle", 3: "angle"}}
+
+" Give this another pass. This seems overly hacky even if it's functional
+let s:sub_tran_msg = ""
+func s:subTranProg(msg)
+    if s:sub_tran_msg != ""
+        let s:sub_tran_msg = s:sub_tran_msg .. a:msg
+        if mode() !=? 'v'
+            exe "normal" "u" .. s:sub_tran_msg
+        endif
+    else
+        if s:command_backlog == ""
+            " this should not occur
+            call s:logCallback(0, "Warning: Encountered sub transcription without prior command")
+            let s:command_backlog = "a"
+        endif
+        if a:msg[0] == ' '
+            let s:sub_tran_msg = s:command_backlog .. a:msg[1:-1]
+        else
+            let s:sub_tran_msg = s:command_backlog  .. a:msg
+        endif
+        if mode() !=? 'v'
+            exe "normal" s:sub_tran_msg
+        endif
+    endif
+    call appendbufline(s:output_buffer, "$", s:sub_tran_msg ..  ":" .. string(a:msg ))
+endfunction
+
+func s:subTranFinish(params, timestamp)
+    let s:repeat_command = s:sub_tran_msg
+    " Visual selection is lot if used with streaming, so streaming of partial
+    " transcriptions is disabled in visual mode
+    if mode() ==? 'v'
+        exe "normal" s:sub_tran_msg
+    endif
+    let s:sub_tran_msg = ""
+    let s:command_backlog = ""
+    exe "normal a\<C-G>u"
+    let l:params = a:params
+    let l:params.timestamp = a:timestamp
+    if exists("l:params.commandset_index")
+        unlet l:params.commandset_index
+    endif
+    call whisper#requestCommands(a:params)
+endfunction
+
+func s:logCallback(channel, msg)
+    call appendbufline(s:output_buffer,"$",a:msg)
+endfunction
+
+
+func s:transcriptionCallback(progressCallback, finishedCallback, channel, msg)
+    let l:tr = a:msg.result.transcription
+
+    let l:ex_ind = match(tolower(l:tr),"exit", len(l:tr)-6)
+    " The worst case I've observed so far is " Exit.", which is 6 characters
+    if l:ex_ind != -1
+        call a:progressCallback(strpart(l:tr,0,l:ex_ind-1))
+        call a:finishedCallback(a:msg.result.timestamp)
+    else
+        call a:progressCallback(l:tr)
+        let req = {"method": "unguided", "params": {"timestamp": a:msg.result.timestamp, "no_context": v:true}}
+        let resp = ch_sendexpr(g:lsp_job, req, {"callback": function("s:transcriptionCallback", [a:progressCallback, a:finishedCallback])})
+    endif
+endfunc
+func s:insertText(msg)
+    exe "normal a" .. a:msg
+endfunction
+func s:endTranscription(timestamp)
+    call appendbufline(s:output_buffer, "$", "Ending unguided transcription")
+endfunction
+
+
+
+" If a command does not include a whole actionable step, attempting to execute
+" it discards the remainder of things. There is likely a simpler solution,
+" but it can be made functional now by storing a backbuffer until actionable
+let s:command_backlog = ""
+let s:repeat_command = ""
+let s:preceeding_upper = v:false
+func s:commandCallback(params, commandset_index, channel, msg)
+    let l:command_index = a:msg.result.command_index
+    let l:do_execute = v:false
+    let l:next_mode = a:commandset_index
+    let l:command = s:commandset_list[a:commandset_index][l:command_index]
+    call s:logCallback(0, string(a:msg) .. " " .. a:commandset_index .. " " .. l:command)
+    if l:command_index == 0
+        "exit
+        "if s:command_backlog == ""
+        call s:logCallback(0,"Stopping command mode")
+        echo "No longer listening"
+        let s:command_backlog = ""
+        return
+        "else
+        " Legacy code to clear an existing buffer with exit.
+        " Was found to be rarely desired and is better introduced as a
+        " standalone command (clear?)
+        "   call s:logCallback(0,"Clearing command_backlog" .. s:command_backlog)
+        "   let s:command_backlog = ""
+        "   let s:preceeding_upper = v:false
+        " endif
+    elseif l:command_index == 1
+        " upper
+        let s:preceeding_upper = !s:preceeding_upper
+    elseif l:command == "save"
+        " save and run can only happen in commandset 0,
+        exe "w"
+    elseif l:command == "run"
+        exe "make run"
+    elseif l:command == "space"
+        exe "normal i \<ESC>l"
+    elseif has_key(s:c_user, l:command)
+        let Userfunc = s:c_user[l:command]
+        if type(Userfunc) == v:t_string
+            let Userfunc = function(Userfunc)
+        endif
+        call Userfunc()
+    else
+        if s:preceeding_upper
+            " Upper should keep commandset
+            let s:preceeding_upper = v:false
+            let l:visual_command = tr(l:command, s:c_lowerkeys, s:c_upperkeys)
+        else
+            let l:visual_command = l:command
+        endif
+        echo s:command_backlog .. " - " .. l:visual_command
+        let s:command_backlog = s:command_backlog .. l:visual_command
+        if a:commandset_index == 2 || a:commandset_index == 3
+            " single key, either completes motion, replace, or register
+            " Should move to execute unless part of a register
+            " Change will be caught at execute
+            if s:command_backlog[-2:-2] !=# '"'
+                call s:logCallback(0,"not register")
+                let l:do_execute = v:true
+            end
+            let l:next_mode = 0
+            " commandset index only matters for a/i
+        elseif (l:command == "a" || l:command == "i") && a:commandset_index == 1
+            " inside/around. Is commandset 3
+            let l:next_mode = 3
+        elseif l:command ==# '"'
+            let l:next_mode = 2
+        elseif index(s:c_count, l:command) != -1
+            let l:next_mode = a:commandset_index
+        elseif index(s:c_motion, l:command) != -1
+            if l:command == 't' || l:command == 'f' || l:command == "'"
+                " prompt single key
+                let l:next_mode = 2
+            else
+                let l:do_execute = v:true
+                let l:next_mode = 0
+            endif
+        elseif index(s:c_command, l:command) != -1
+            if index(["y","g","d","c"], s:command_backlog[-1:-1]) != -1 && s:command_backlog[-1:-1] != s:command_backlog[-2:-2] && mode() !=? 'v'
+                " need motion or repeated command
+                " Potential for bad state here if disparaging command keys are
+                " entered (i.e. yd), but vim can handle checks for this at exe
+                " And checking for cases like y123d would complicate things
+                let l:next_mode = 1
+            elseif index(["i","a","c", "o", "s"], l:command) != -1 || s:command_backlog[-1:-1] ==# 'R'
+                "'Insert' mode, do general transcription
+                let l:req = {"method": "unguided", "params": a:params}
+                let l:req.params.timestamp = a:msg.result.timestamp
+                let l:req.params.no_context = v:true
+                let resp = ch_sendexpr(g:lsp_job, req, {"callback": function("s:transcriptionCallback", [function("s:subTranProg"), function("s:subTranFinish", [a:params])])})
+                return
+            elseif l:command == 'r' || l:command == 'm'
+                let l:next_mode = 2
+            elseif l:command == '.'
+                let l:next_mode = 0
+                let l:do_execute = v:true
+                let s:command_backlog = s:command_backlog[0:-2] .. s:repeat_command
+            else
+                if l:command ==? 'v'
+                    let l:next_mode = 1
+                else
+                    let l:next_mode = 0
+                endif
+                let l:do_execute = v:true
+            endif
+        else
+            throw "Invalid command state: " .. l:command .. " " .. a:commandset_index .. " " .. s:command_backlog
+        endif
+    endif
+    if l:do_execute
+        if mode() ==?'v' && l:next_mode == 0
+            let l:next_mode = 1
+        elseif match(s:command_backlog, 'c') != -1
+            let l:req = {"method": "unguided", "params": a:params}
+            let l:req.params.timestamp = a:msg.result.timestamp
+            let l:req.params.no_context = v:true
+            let resp = ch_sendexpr(g:lsp_job, req, {"callback": function("s:transcriptionCallback", [function("s:subTranProg"), function("s:subTranFinish", [a:params])])})
+            return
+        endif
+        exe "normal" s:command_backlog
+        if index(s:c_motion + ["u"],l:command) == -1
+            exe "normal a\<C-G>u"
+            let s:repeat_command = s:command_backlog
+            call s:logCallback(0, s:command_backlog)
+        endif
+        let s:command_backlog = ""
+    endif
+    let l:req = {"method": "guided", "params": a:params}
+    let l:req.params.timestamp = a:msg.result.timestamp
+    let l:req.params.commandset_index = l:next_mode
+    let resp = ch_sendexpr(g:lsp_job, l:req, {"callback": function("s:commandCallback",[a:params, l:next_mode])})
+endfunction
+
+func s:loadedCallback(channel, msg)
+    echo "Loading complete"
+    call s:logCallback(a:channel, a:msg)
+endfunction
+
+func s:registerCommandset(commandlist, is_final)
+    let req = {"method": "registerCommandset"}
+    let req.params = a:commandlist
+    call s:logCallback(0, join(a:commandlist))
+    call add(g:whisper_commandlist_spoken, a:commandlist)
+    if a:is_final
+        let resp = ch_sendexpr(g:lsp_job, req, {"callback": "s:loadedCallback"})
+    else
+        let resp = ch_sendexpr(g:lsp_job, req, {"callback": "s:logCallback"})
+    endif
+endfunction
+
+func s:registerAllCommands()
+    let l:normal = s:c_special_always + s:c_special_normal + s:c_count + s:c_command + s:c_motion + keys(s:c_user)
+    let l:visual = s:c_special_always + s:c_count + s:c_command + s:c_motion
+    " Currently the same as visual.
+    " let l:post_command = s:c_special_always + s:c_count + s:c_command + s:c_motion
+    let l:single_key = s:c_special_always + split(s:c_lowerkeys, '\zs')
+    let l:area = s:c_special_always + s:c_area
+
+    " Used only for compatibility with the testing script
+    let g:whisper_commandlist_spoken = []
+
+    let s:commandset_list = [l:normal, l:visual, l:single_key, l:area]
+    call s:registerCommandset(s:commandsetToSpoken(l:normal, 0), v:false)
+    call s:registerCommandset(s:commandsetToSpoken(l:visual, 1), v:false)
+    call s:registerCommandset(s:commandsetToSpoken(l:single_key, 2), v:false)
+    call s:registerCommandset(s:commandsetToSpoken(l:area, 3), v:true)
+endfunction
+
+func s:commandsetToSpoken(commandset, spoken_index)
+    let l:spoken_list = []
+    for l:command in a:commandset
+        if has_key(s:spoken_dict, l:command)
+            let l:spoken_value = s:spoken_dict[l:command]
+            if type(l:spoken_value) == v:t_dict
+                if has_key(l:spoken_value, a:spoken_index)
+                    let l:spoken_value = l:spoken_value[a:spoken_index]
+                else
+                    if a:spoken_index == 2
+                        let l:spoken_value = l:command
+                    else
+                        let l:spoken_value = l:spoken_value[0]
+                    endif
+                endif
+            else
+                if a:spoken_index == 2
+                    let l:spoken_value = l:command
+                endif
+            endif
+        else
+            let l:spoken_value = l:command
+        endif
+        call add(l:spoken_list, l:spoken_value)
+    endfor
+    return l:spoken_list
+endfunction
+
+" TODO: Check lifetime. If the script is resourced, is the existing
+" s:lsp_job dropped and therefore killed?
+" This seems to not be the case and I've had to deal with zombie processes
+" that survive exiting vim, even though said behavior conflicts with my
+" understanding of the provided documentation
+let s:lsp_opts = {"in_mode": "lsp", "out_mode": "lsp", "err_mode": "nl", "err_io": "buffer", "err_buf": s:output_buffer}
+if !exists("g:lsp_job")
+    if exists("g:whisper_user_commands")
+        let s:c_user = g:whisper_user_commands
+    else
+        let s:c_user = {}
+    endif
+    let g:lsp_job = job_start(s:lsp_command, s:lsp_opts)
+    if job_status(g:lsp_job) == "fail"
+        echoerr "Failed to start whisper job"
+    endif
+    call s:registerAllCommands()
+endif

examples/parakeet-cli/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(TARGET parakeet-cli)
+add_executable(${TARGET} parakeet-cli.cpp)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE common parakeet ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+
+install(TARGETS ${TARGET} RUNTIME)

examples/parakeet-cli/README.md

@@ -0,0 +1,106 @@
+# whisper.cpp/examples/parakeet-cli
+
+This is an example of using the [Parakeet] model in whisper.cpp.
+
+### Download converted model
+```console
+$ hf download ggml-org/parakeet-GGUF ggml-parakeet-tdt-0.6b-v3-f16.bin --local-dir models
+```
+
+### Building
+```console
+$ cmake -B build -S .
+$ cmake --build build --target parakeet-cli -j 12
+```
+
+### Usage
+```console
+$ ./build/bin/parakeet-cli --help
+
+usage: ./build/bin/parakeet-cli [options] file0 file1 ...
+supported audio formats: flac, mp3, ogg, wav
+
+options:
+  -h,     --help              [default] show this help message and exit
+  -t N,   --threads N         [4      ] number of threads to use during computation
+  -m,     --model FILE        [models/ggml-parakeet-tdt-0.6b-v3.bin] model path
+  -f,     --file FILE         [       ] input audio file
+  -ng,    --no-gpu            [false  ] disable GPU
+  -dev N, --device N          [0      ] GPU device to use
+  -ps,    --print-segments    [false  ] print segment information
+```
+
+### Example
+```console
+$ ./build/bin/parakeet-cli -m models/ggml-parakeet-tdt-0.6b-v3-f16.bin -f samples/jfk.wav
+Processing audio (176000 samples, 11.00 seconds)
+Processing audio: total_frames=1101, chunk_size=1101
+parakeet_decode: starting decode with n_frames=138
+And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
+```
+
+To print segment information:
+```console
+$ ./build/bin/parakeet-cli -m models/ggml-parakeet-tdt-0.6b-v3-f16.bin -f samples/jfk.wav --print-segments
+Processing audio (176000 samples, 11.00 seconds)
+Processing audio: total_frames=1101, chunk_size=1101
+parakeet_decode: starting decode with n_frames=138
+And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
+
+Segments (1):
+Segment 0: [0 -> 1101] "And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country."
+Tokens [38]:
+  [ 0] id= 1976 frame=  3 dur_idx= 4 dur_val= 4 p=0.9996 plog=-15.6206 t0=  24 t1=  56 word_start=true "▁And"
+  [ 1] id=  547 frame=  7 dur_idx= 4 dur_val= 4 p=0.9999 plog=-18.7922 t0=  56 t1=  88 word_start=true "▁so"
+  [ 2] id= 7877 frame= 11 dur_idx= 2 dur_val= 2 p=0.8451 plog=-14.5929 t0=  88 t1=  88 word_start=false ","
+  [ 3] id= 1103 frame= 13 dur_idx= 3 dur_val= 3 p=0.9996 plog=-15.6127 t0= 104 t1= 128 word_start=true "▁my"
+  [ 4] id=  309 frame= 16 dur_idx= 1 dur_val= 1 p=0.9912 plog=-11.9635 t0= 128 t1= 136 word_start=true "▁f"
+  [ 5] id=  530 frame= 17 dur_idx= 2 dur_val= 2 p=1.0000 plog=-13.5239 t0= 136 t1= 152 word_start=false "ell"
+  [ 6] id=  596 frame= 19 dur_idx= 3 dur_val= 3 p=1.0000 plog=-16.3120 t0= 152 t1= 176 word_start=false "ow"
+  [ 7] id= 3213 frame= 22 dur_idx= 4 dur_val= 4 p=0.9999 plog=-10.1462 t0= 176 t1= 208 word_start=true "▁Amer"
+  [ 8] id=  404 frame= 26 dur_idx= 4 dur_val= 4 p=1.0000 plog=-25.0910 t0= 208 t1= 240 word_start=false "ic"
+  [ 9] id=  667 frame= 30 dur_idx= 4 dur_val= 4 p=1.0000 plog=-27.1707 t0= 240 t1= 272 word_start=false "ans"
+  [10] id= 7877 frame= 37 dur_idx= 4 dur_val= 4 p=0.9094 plog=-16.3405 t0= 272 t1= 272 word_start=false ","
+  [11] id=  279 frame= 41 dur_idx= 4 dur_val= 4 p=0.9980 plog=-19.7244 t0= 328 t1= 360 word_start=true "▁a"
+  [12] id=  583 frame= 45 dur_idx= 4 dur_val= 4 p=1.0000 plog=-24.5312 t0= 360 t1= 392 word_start=false "sk"
+  [13] id= 1491 frame= 53 dur_idx= 4 dur_val= 4 p=1.0000 plog=-23.2991 t0= 424 t1= 456 word_start=true "▁not"
+  [14] id= 3470 frame= 65 dur_idx= 4 dur_val= 4 p=0.9995 plog=-16.7306 t0= 520 t1= 552 word_start=true "▁what"
+  [15] id= 3629 frame= 69 dur_idx= 2 dur_val= 2 p=0.8139 plog=-11.6486 t0= 552 t1= 568 word_start=true "▁your"
+  [16] id=  867 frame= 75 dur_idx= 1 dur_val= 1 p=0.9980 plog=-12.5265 t0= 600 t1= 608 word_start=true "▁co"
+  [17] id=  331 frame= 76 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.6697 t0= 608 t1= 624 word_start=false "un"
+  [18] id=  958 frame= 78 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.3621 t0= 624 t1= 640 word_start=false "tr"
+  [19] id= 7893 frame= 80 dur_idx= 2 dur_val= 2 p=1.0000 plog=-14.3245 t0= 640 t1= 656 word_start=false "y"
+  [20] id= 2059 frame= 82 dur_idx= 3 dur_val= 3 p=1.0000 plog=-17.7694 t0= 656 t1= 680 word_start=true "▁can"
+  [21] id=  458 frame= 85 dur_idx= 4 dur_val= 4 p=1.0000 plog=-23.2510 t0= 680 t1= 712 word_start=true "▁do"
+  [22] id=  509 frame= 89 dur_idx= 4 dur_val= 4 p=1.0000 plog=-23.0688 t0= 712 t1= 744 word_start=true "▁for"
+  [23] id= 1180 frame= 93 dur_idx= 4 dur_val= 4 p=0.9999 plog=-25.0567 t0= 744 t1= 776 word_start=true "▁you"
+  [24] id= 7877 frame= 98 dur_idx= 4 dur_val= 4 p=0.8820 plog=-14.2549 t0= 776 t1= 776 word_start=false ","
+  [25] id=  279 frame=102 dur_idx= 3 dur_val= 3 p=0.9992 plog=-16.8176 t0= 816 t1= 840 word_start=true "▁a"
+  [26] id=  583 frame=105 dur_idx= 4 dur_val= 4 p=1.0000 plog=-21.0352 t0= 840 t1= 872 word_start=false "sk"
+  [27] id= 3470 frame=109 dur_idx= 3 dur_val= 3 p=0.9999 plog=-15.4659 t0= 872 t1= 896 word_start=true "▁what"
+  [28] id= 1180 frame=112 dur_idx= 4 dur_val= 4 p=0.9997 plog=-17.6392 t0= 896 t1= 928 word_start=true "▁you"
+  [29] id= 2059 frame=116 dur_idx= 3 dur_val= 3 p=0.9999 plog=-15.5484 t0= 928 t1= 952 word_start=true "▁can"
+  [30] id=  458 frame=119 dur_idx= 2 dur_val= 2 p=1.0000 plog=-15.9953 t0= 952 t1= 968 word_start=true "▁do"
+  [31] id=  509 frame=121 dur_idx= 3 dur_val= 3 p=1.0000 plog=-15.9605 t0= 968 t1= 992 word_start=true "▁for"
+  [32] id= 3629 frame=124 dur_idx= 2 dur_val= 2 p=0.9994 plog=-12.2083 t0= 992 t1=1008 word_start=true "▁your"
+  [33] id=  867 frame=126 dur_idx= 2 dur_val= 2 p=0.9969 plog=-9.1252 t0=1008 t1=1024 word_start=true "▁co"
+  [34] id=  331 frame=128 dur_idx= 1 dur_val= 1 p=0.9999 plog=-12.6911 t0=1024 t1=1032 word_start=false "un"
+  [35] id=  958 frame=129 dur_idx= 1 dur_val= 1 p=1.0000 plog=-8.8885 t0=1032 t1=1040 word_start=false "tr"
+  [36] id= 7893 frame=130 dur_idx= 2 dur_val= 2 p=1.0000 plog=-14.1441 t0=1040 t1=1056 word_start=false "y"
+  [37] id= 7883 frame=132 dur_idx= 4 dur_val= 4 p=0.9567 plog=-11.5227 t0=1056 t1=1056 word_start=false "."
+```
+
+### Model conversion
+Clone the original model from Hugging Face:
+```console
+$ git clone https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3
+```
+Convert the model:
+```console
+(venv) $ python models/convert-parakeet-to-ggml.py \
+    --model <path to cloned model> \
+    --out-dir models \
+    --out-name ggml-parakeet-tdt-0.6b-v3-f16.bin
+```
+
+[Parakeet]: https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3

examples/parakeet-cli/parakeet-cli.cpp

@@ -0,0 +1,243 @@
+#include "parakeet.h"
+#include "common-whisper.h"
+
+#include <cstdio>
+#include <string>
+#include <thread>
+#include <vector>
+#include <cstring>
+#include <fstream>
+
+// command-line parameters
+struct parakeet_params {
+    int32_t n_threads         = std::min(4, (int32_t) std::thread::hardware_concurrency());
+
+    bool use_gpu       = true;
+    int32_t gpu_device = 0;
+
+    bool print_segments = false;
+    bool output_txt     = false;
+    bool no_prints      = false;
+
+    std::string model       = "models/ggml-parakeet-tdt-0.6b-v3.bin";
+    std::string output_file = "";
+    std::vector<std::string> fname_inp = {};
+};
+
+static void parakeet_print_usage(int argc, char ** argv, const parakeet_params & params);
+
+static char * requires_value_error(const std::string & arg) {
+    fprintf(stderr, "error: argument %s requires value\n", arg.c_str());
+    exit(1);
+}
+
+static bool parakeet_params_parse(int argc, char ** argv, parakeet_params & params) {
+    if (const char * env_device = std::getenv("PARAKEET_ARG_DEVICE")) {
+        params.gpu_device = std::stoi(env_device);
+    }
+
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-"){
+            params.fname_inp.push_back(arg);
+            continue;
+        }
+
+        if (arg[0] != '-') {
+            params.fname_inp.push_back(arg);
+            continue;
+        }
+
+        if (arg == "-h" || arg == "--help") {
+            parakeet_print_usage(argc, argv, params);
+            exit(0);
+        }
+        #define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))
+        else if (arg == "-t"    || arg == "--threads")         { params.n_threads         = std::stoi(ARGV_NEXT); }
+        else if (arg == "-m"    || arg == "--model")           { params.model             = ARGV_NEXT; }
+        else if (arg == "-f"    || arg == "--file")            { params.fname_inp.emplace_back(ARGV_NEXT); }
+        else if (arg == "-ng"   || arg == "--no-gpu")          { params.use_gpu           = false; }
+        else if (arg == "-dev"  || arg == "--device")          { params.gpu_device        = std::stoi(ARGV_NEXT); }
+        else if (arg == "-ps"   || arg == "--print-segments")  { params.print_segments    = true; }
+        else if (arg == "-otxt" || arg == "--output-txt")      { params.output_txt        = true; }
+        else if (arg == "-of"   || arg == "--output-file")     { params.output_file       = ARGV_NEXT; }
+        else if (arg == "-np"   || arg == "--no-prints")       { params.no_prints         = true; }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            parakeet_print_usage(argc, argv, params);
+            exit(1);
+        }
+    }
+
+    return true;
+}
+
+static void parakeet_print_usage(int /*argc*/, char ** argv, const parakeet_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options] file0 file1 ...\n", argv[0]);
+    fprintf(stderr, "supported audio formats: flac, mp3, ogg, wav\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,     --help              [default] show this help message and exit\n");
+    fprintf(stderr, "  -t N,   --threads N         [%-7d] number of threads to use during computation\n", params.n_threads);
+    fprintf(stderr, "  -m,     --model FILE        [%-7s] model path\n",                                  params.model.c_str());
+    fprintf(stderr, "  -f,     --file FILE         [%-7s] input audio file\n",                            "");
+    fprintf(stderr, "  -ng,    --no-gpu            [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -dev N, --device N          [%-7d] GPU device to use\n",                           params.gpu_device);
+    fprintf(stderr, "  -ps,    --print-segments    [%-7s] print segment information\n",                   params.print_segments ? "true" : "false");
+    fprintf(stderr, "  -otxt,  --output-txt        [%-7s] output result in a text file\n",                params.output_txt ? "true" : "false");
+    fprintf(stderr, "  -of,    --output-file FILE  [%-7s] output file path (without file extension)\n",   "");
+    fprintf(stderr, "  -np,    --no-prints         [%-7s] do not print anything other than the results\n", params.no_prints ? "true" : "false");
+    fprintf(stderr, "\n");
+}
+
+void token_callback(parakeet_context * ctx, parakeet_state * state, const parakeet_token_data * token_data, void * user_data) {
+    bool * is_first = (bool *) user_data;
+
+    const char * token_str = parakeet_token_to_str(ctx, token_data->id);
+    char text_buf[256];
+    parakeet_token_to_text(token_str, *is_first, text_buf, sizeof(text_buf));
+    printf("%s", text_buf);
+    fflush(stdout);
+
+    *is_first = false;
+}
+
+static void cb_log_disable(enum ggml_log_level , const char * , void * ) { }
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    parakeet_params params;
+
+    if (parakeet_params_parse(argc, argv, params) == false) {
+        return 1;
+    }
+
+    if (params.no_prints) {
+        parakeet_log_set(cb_log_disable, NULL);
+    }
+
+    if (params.fname_inp.empty()) {
+        fprintf(stderr, "error: no input files specified\n");
+        parakeet_print_usage(argc, argv, params);
+        return 1;
+    }
+
+    struct parakeet_context_params ctx_params = parakeet_context_default_params();
+    ctx_params.use_gpu     = params.use_gpu;
+    ctx_params.gpu_device  = params.gpu_device;
+
+    if (!params.no_prints) {
+        fprintf(stderr, "Loading Parakeet model from: %s\n", params.model.c_str());
+    }
+
+
+    struct parakeet_context * pctx = parakeet_init_from_file_with_params(params.model.c_str(), ctx_params);
+    if (pctx == nullptr) {
+        fprintf(stderr, "error: failed to load Parakeet model from '%s'\n", params.model.c_str());
+        return 1;
+    }
+
+    if (!params.no_prints) {
+        fprintf(stderr, "Successfully loaded Parakeet model\n");
+        fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
+                params.n_threads, (int32_t) std::thread::hardware_concurrency(), parakeet_print_system_info());
+    }
+
+    // Process each input file
+    for (const auto & fname : params.fname_inp) {
+        if (!params.no_prints) {
+            fprintf(stderr, "\nProcessing file: %s\n", fname.c_str());
+        }
+
+        std::vector<float> pcmf32;
+        std::vector<std::vector<float>> pcmf32s;
+        if (!read_audio_data(fname.c_str(), pcmf32, pcmf32s, false)) {
+            fprintf(stderr, "error: failed to read audio file '%s'\n", fname.c_str());
+            continue;
+        }
+
+        if (pcmf32.empty()) {
+            fprintf(stderr, "error: no audio data in file '%s'\n", fname.c_str());
+            continue;
+        }
+
+        bool is_first = true;
+        struct parakeet_full_params full_params = parakeet_full_default_params(PARAKEET_SAMPLING_GREEDY);
+        full_params.n_threads           = params.n_threads;
+        full_params.new_token_callback  = token_callback;
+        full_params.new_token_callback_user_data = &is_first;
+
+        const int mel_frames = (int)(pcmf32.size() / PARAKEET_HOP_LENGTH);
+        int ret = parakeet_full(pctx, full_params, pcmf32.data(), pcmf32.size());
+
+        if (ret != 0) {
+            fprintf(stderr, "error: failed to process audio file '%s'\n", fname.c_str());
+            continue;
+        }
+
+        printf("\n");
+
+        if (params.output_txt) {
+            const std::string fname_out = (!params.output_file.empty() ? params.output_file : fname) + ".txt";
+
+            std::ofstream fout(fname_out);
+            if (fout.is_open()) {
+                const int n_segments = parakeet_full_n_segments(pctx);
+                for (int i = 0; i < n_segments; ++i) {
+                    const char * text = parakeet_full_get_segment_text(pctx, i);
+                    fout << text << "\n";
+                }
+                fout.close();
+                if (!params.no_prints) {
+                    fprintf(stderr, "Output written to: %s\n", fname_out.c_str());
+                }
+            } else {
+                fprintf(stderr, "error: failed to open '%s' for writing\n", fname_out.c_str());
+            }
+        }
+
+        if (!params.no_prints) {
+            parakeet_print_timings(pctx);
+        }
+
+        if (params.print_segments) {
+            const int n_segments = parakeet_full_n_segments(pctx);
+            fprintf(stderr, "\nSegments (%d):\n", n_segments);
+
+            for (int i = 0; i < n_segments; i++) {
+                const char * text = parakeet_full_get_segment_text(pctx, i);
+                const int64_t t0 = parakeet_full_get_segment_t0(pctx, i);
+                const int64_t t1 = parakeet_full_get_segment_t1(pctx, i);
+                const int n_tokens = parakeet_full_n_tokens(pctx, i);
+
+                fprintf(stderr, "Segment %d: [%lld -> %lld] \"%s\"\n", i, (long long)t0, (long long)t1, text);
+                fprintf(stderr, "Tokens [%d]:\n", n_tokens);
+
+                for (int j = 0; j < n_tokens; j++) {
+                    parakeet_token_data token_data = parakeet_full_get_token_data(pctx, i, j);
+                    const char * token_str = parakeet_token_to_str(pctx, token_data.id);
+
+                    fprintf(stderr, "  [%2d] id=%5d frame=%3d dur_idx=%2d dur_val=%2d p=%.4f plog=%.4f t0=%4lld t1=%4lld word_start=%s \"%s\"\n",
+                           j,
+                           token_data.id,
+                           token_data.frame_index,
+                           token_data.duration_idx,
+                           token_data.duration_value,
+                           token_data.p,
+                           token_data.plog,
+                           (long long)token_data.t0,
+                           (long long)token_data.t1,
+                           token_data.is_word_start ? "true": "false",
+                           token_str);
+                }
+            }
+        }
+    }
+
+    parakeet_free(pctx);
+
+    return 0;
+}

examples/parakeet-quantize/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(TARGET parakeet-quantize)
+add_executable(${TARGET} parakeet-quantize.cpp)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE common parakeet ${CMAKE_THREAD_LIBS_INIT})
+install(TARGETS ${TARGET} RUNTIME)

examples/parakeet-quantize/parakeet-quantize.cpp

@@ -0,0 +1,230 @@
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include "common-ggml.h"
+
+#include <cassert>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <string>
+#include <vector>
+
+struct parakeet_hparams {
+    int32_t n_vocab                = 0;
+    int32_t n_audio_ctx            = 0;
+    int32_t n_audio_state          = 0;
+    int32_t n_audio_head           = 0;
+    int32_t n_audio_layer          = 0;
+    int32_t n_mels                 = 0;
+    int32_t ftype                  = 0;
+    int32_t n_fft                  = 0;
+    int32_t subsampling_factor     = 0;
+    int32_t n_subsampling_channels = 0;
+    int32_t n_conv_kernel          = 0;
+    int32_t n_pred_dim             = 0;
+    int32_t n_pred_layers          = 0;
+    int32_t n_tdt_durations        = 0;
+    int32_t n_max_tokens           = 0;
+};
+
+static bool parakeet_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
+    printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
+
+    auto finp = std::ifstream(fname_inp, std::ios::binary);
+    if (!finp) {
+        fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
+        return false;
+    }
+
+    auto fout = std::ofstream(fname_out, std::ios::binary);
+    if (!fout) {
+        fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
+        return false;
+    }
+
+    // magic
+    {
+        uint32_t magic;
+        finp.read((char *) &magic, sizeof(magic));
+        if (magic != GGML_FILE_MAGIC) {
+            fprintf(stderr, "%s: invalid model file (bad magic)\n", __func__);
+            return false;
+        }
+        fout.write((char *) &magic, sizeof(magic));
+    }
+
+    // hparams
+    parakeet_hparams hparams;
+    {
+        finp.read((char *) &hparams.n_vocab,                sizeof(hparams.n_vocab));
+        finp.read((char *) &hparams.n_audio_ctx,            sizeof(hparams.n_audio_ctx));
+        finp.read((char *) &hparams.n_audio_state,          sizeof(hparams.n_audio_state));
+        finp.read((char *) &hparams.n_audio_head,           sizeof(hparams.n_audio_head));
+        finp.read((char *) &hparams.n_audio_layer,          sizeof(hparams.n_audio_layer));
+        finp.read((char *) &hparams.n_mels,                 sizeof(hparams.n_mels));
+        finp.read((char *) &hparams.ftype,                  sizeof(hparams.ftype));
+        finp.read((char *) &hparams.n_fft,                  sizeof(hparams.n_fft));
+        finp.read((char *) &hparams.subsampling_factor,     sizeof(hparams.subsampling_factor));
+        finp.read((char *) &hparams.n_subsampling_channels, sizeof(hparams.n_subsampling_channels));
+        finp.read((char *) &hparams.n_conv_kernel,          sizeof(hparams.n_conv_kernel));
+        finp.read((char *) &hparams.n_pred_dim,             sizeof(hparams.n_pred_dim));
+        finp.read((char *) &hparams.n_pred_layers,          sizeof(hparams.n_pred_layers));
+        finp.read((char *) &hparams.n_tdt_durations,        sizeof(hparams.n_tdt_durations));
+        finp.read((char *) &hparams.n_max_tokens,           sizeof(hparams.n_max_tokens));
+
+        const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
+        const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
+
+        fprintf(stderr, "%s: n_vocab              = %d\n",  __func__, hparams.n_vocab);
+        fprintf(stderr, "%s: n_audio_state        = %d\n",  __func__, hparams.n_audio_state);
+        fprintf(stderr, "%s: n_audio_layer        = %d\n",  __func__, hparams.n_audio_layer);
+        fprintf(stderr, "%s: n_mels               = %d\n",  __func__, hparams.n_mels);
+        fprintf(stderr, "%s: ftype (src)          = %d\n",  __func__, hparams.ftype);
+        fprintf(stderr, "%s: qntvr (src)          = %d\n",  __func__, qntvr_src);
+        fprintf(stderr, "%s: ftype (dst)          = %d\n",  __func__, ftype_dst);
+        fprintf(stderr, "%s: qntvr (dst)          = %d\n",  __func__, GGML_QNT_VERSION);
+
+        fout.write((char *) &hparams.n_vocab,                sizeof(hparams.n_vocab));
+        fout.write((char *) &hparams.n_audio_ctx,            sizeof(hparams.n_audio_ctx));
+        fout.write((char *) &hparams.n_audio_state,          sizeof(hparams.n_audio_state));
+        fout.write((char *) &hparams.n_audio_head,           sizeof(hparams.n_audio_head));
+        fout.write((char *) &hparams.n_audio_layer,          sizeof(hparams.n_audio_layer));
+        fout.write((char *) &hparams.n_mels,                 sizeof(hparams.n_mels));
+        fout.write((char *) &ftype_dst,                      sizeof(ftype_dst));
+        fout.write((char *) &hparams.n_fft,                  sizeof(hparams.n_fft));
+        fout.write((char *) &hparams.subsampling_factor,     sizeof(hparams.subsampling_factor));
+        fout.write((char *) &hparams.n_subsampling_channels, sizeof(hparams.n_subsampling_channels));
+        fout.write((char *) &hparams.n_conv_kernel,          sizeof(hparams.n_conv_kernel));
+        fout.write((char *) &hparams.n_pred_dim,             sizeof(hparams.n_pred_dim));
+        fout.write((char *) &hparams.n_pred_layers,          sizeof(hparams.n_pred_layers));
+        fout.write((char *) &hparams.n_tdt_durations,        sizeof(hparams.n_tdt_durations));
+        fout.write((char *) &hparams.n_max_tokens,           sizeof(hparams.n_max_tokens));
+    }
+
+    // mel filterbank
+    {
+        int32_t n_mel, n_fb;
+        finp.read((char *) &n_mel, sizeof(n_mel));
+        fout.write((char *) &n_mel, sizeof(n_mel));
+        finp.read((char *) &n_fb,  sizeof(n_fb));
+        fout.write((char *) &n_fb,  sizeof(n_fb));
+
+        const size_t n = (size_t) n_mel * n_fb;
+        std::vector<float> buf(n);
+        finp.read((char *) buf.data(), n * sizeof(float));
+        fout.write((char *) buf.data(), n * sizeof(float));
+    }
+
+    // window function
+    {
+        int32_t n_window;
+        finp.read((char *) &n_window, sizeof(n_window));
+        fout.write((char *) &n_window, sizeof(n_window));
+
+        std::vector<float> buf(n_window);
+        finp.read((char *) buf.data(), n_window * sizeof(float));
+        fout.write((char *) buf.data(), n_window * sizeof(float));
+    }
+
+    // TDT durations
+    {
+        std::vector<uint32_t> buf(hparams.n_tdt_durations);
+        finp.read((char *) buf.data(), hparams.n_tdt_durations * sizeof(uint32_t));
+        fout.write((char *) buf.data(), hparams.n_tdt_durations * sizeof(uint32_t));
+    }
+
+    // vocab
+    {
+        int32_t n_tokens;
+        finp.read((char *) &n_tokens, sizeof(n_tokens));
+        fout.write((char *) &n_tokens, sizeof(n_tokens));
+
+        for (int i = 0; i < n_tokens; ++i) {
+            int32_t len;
+            finp.read((char *) &len, sizeof(len));
+            fout.write((char *) &len, sizeof(len));
+
+            std::string token(len, '\0');
+            finp.read(&token[0], len);
+            fout.write(&token[0], len);
+        }
+    }
+
+    // tensors — quantize 2D weights skipping tensors that must stay F32:
+    // ggml_ssm_conv / ggml_conv2d_dw CUDA kernels require F32 weights.
+    // pos_bias_u / pos_bias_v are declared F32 in the loader.
+    const std::vector<std::string> to_quant = { ".*" };
+    std::vector<std::string> to_skip = {
+        // CUDA kernel constraints (ggml_ssm_conv / ggml_conv2d_dw require F32 weights)
+        "encoder\\.layers\\..+\\.conv\\.depthwise_conv\\.weight",
+        // Declared F32 in loader (pos_bias tensors)
+        "encoder\\.layers\\..+\\.self_attn\\.pos_bias_u",
+        "encoder\\.layers\\..+\\.self_attn\\.pos_bias_v",
+    };
+
+    // Prediction/joint tensors use n_pred_dim as their inner dimension. K-quant
+    // types (block size 256) cannot quantize 640 evenly, so keep them F32. For
+    // other types (Q8_0, Q4_0, block size 32) 640 is divisible and they can be
+    // quantized normally. The loader mirrors this logic at load time.
+    {
+        const ggml_type qtype = ggml_ftype_to_ggml_type(ftype);
+        const int32_t   blck  = ggml_blck_size(qtype);
+        if (blck > 1 && hparams.n_pred_dim % blck != 0) {
+            to_skip.push_back("decoder\\.prediction\\.embed\\.weight");
+            to_skip.push_back("decoder\\.prediction\\.dec_rnn\\.lstm\\.weight_ih_l.*");
+            to_skip.push_back("decoder\\.prediction\\.dec_rnn\\.lstm\\.weight_hh_l.*");
+            to_skip.push_back("joint\\.pred\\.weight");
+            to_skip.push_back("joint\\.joint_net\\.2\\.weight");
+        }
+    }
+
+    if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, to_skip)) {
+        fprintf(stderr, "%s: failed to quantize tensors\n", __func__);
+        return false;
+    }
+
+    finp.close();
+    fout.close();
+
+    return true;
+}
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    if (argc != 4) {
+        fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
+        ggml_print_ftypes(stderr);
+        return 1;
+    }
+
+    // initialise F16 lookup tables
+    {
+        struct ggml_init_params params = { 0, NULL, false };
+        struct ggml_context * ctx = ggml_init(params);
+        ggml_free(ctx);
+    }
+
+    const std::string fname_inp = argv[1];
+    const std::string fname_out = argv[2];
+    const ggml_ftype  ftype     = ggml_parse_ftype(argv[3]);
+
+    if (ftype == GGML_FTYPE_UNKNOWN) {
+        fprintf(stderr, "%s: invalid quantization type\n", argv[0]);
+        ggml_print_ftypes(stderr);
+        return 1;
+    }
+
+    const int64_t t_start_us = ggml_time_us();
+
+    if (!parakeet_model_quantize(fname_inp, fname_out, ftype)) {
+        fprintf(stderr, "%s: failed to quantize model from '%s'\n", argv[0], fname_inp.c_str());
+        return 1;
+    }
+
+    printf("\n%s: quantize time = %8.2f ms\n", argv[0], (ggml_time_us() - t_start_us) / 1000.0f);
+    printf("%s: output model  = %s\n",         argv[0], fname_out.c_str());
+
+    return 0;
+}

examples/python/test_whisper_processor.py

@@ -0,0 +1,7 @@
+import whisper_processor
+
+try:
+    result = whisper_processor.process_audio("./audio/wake_word_detected16k.wav", "base.en")
+    print(result)
+except Exception as e:
+    print(f"Error: {e}")

examples/python/whisper_processor.py

@@ -0,0 +1,54 @@
+import subprocess
+import sys
+import os
+
+def process_audio(wav_file, model_name="base.en"):
+    """
+    Processes an audio file using a specified model and returns the processed string.
+
+    :param wav_file: Path to the WAV file
+    :param model_name: Name of the model to use
+    :return: Processed string output from the audio processing
+    :raises: Exception if an error occurs during processing
+    """
+
+    model = f"./models/ggml-{model_name}.bin"
+
+    # Check if the file exists
+    if not os.path.exists(model):
+        raise FileNotFoundError(f"Model file not found: {model} \n\nDownload a model with this command:\n\n> bash ./models/download-ggml-model.sh {model_name}\n\n")
+
+    if not os.path.exists(wav_file):
+        raise FileNotFoundError(f"WAV file not found: {wav_file}")
+
+    full_command = f"./main -m {model} -f {wav_file} -nt"
+
+    # Execute the command
+    process = subprocess.Popen(full_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    # Get the output and error (if any)
+    output, error = process.communicate()
+
+    if error:
+        raise Exception(f"Error processing audio: {error.decode('utf-8')}")
+
+    # Process and return the output string
+    decoded_str = output.decode('utf-8').strip()
+    processed_str = decoded_str.replace('[BLANK_AUDIO]', '').strip()
+
+    return processed_str
+
+def main():
+    if len(sys.argv) >= 2:
+        wav_file = sys.argv[1]
+        model_name = sys.argv[2] if len(sys.argv) == 3 else "base.en"
+        try:
+            result = process_audio(wav_file, model_name)
+            print(result)
+        except Exception as e:
+            print(f"Error: {e}")
+    else:
+        print("Usage: python whisper_processor.py <wav_file> [<model_name>]")
+
+if __name__ == "__main__":
+    main()

examples/quantize/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(TARGET whisper-quantize)
+add_executable(${TARGET} quantize.cpp)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE common whisper ${CMAKE_THREAD_LIBS_INIT})
+install(TARGETS ${TARGET} RUNTIME)

examples/quantize/README.md

@@ -0,0 +1,3 @@
+# quantize
+
+Tool for integer quantization of Whisper `ggml` model files

examples/quantize/quantize.cpp

@@ -0,0 +1,226 @@
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include "common.h"
+#include "common-ggml.h"
+
+#include <cassert>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+#include <regex>
+
+// default hparams (Whisper tiny)
+struct whisper_hparams {
+    int32_t n_vocab       = 51864;
+    int32_t n_audio_ctx   = 1500;
+    int32_t n_audio_state = 384;
+    int32_t n_audio_head  = 6;
+    int32_t n_audio_layer = 4;
+    int32_t n_text_ctx    = 448;
+    int32_t n_text_state  = 384;
+    int32_t n_text_head   = 6;
+    int32_t n_text_layer  = 4;
+    int32_t n_mels        = 80;
+    int32_t ftype         = 1;
+};
+
+struct whisper_filters {
+    int32_t n_mel;
+    int32_t n_fft;
+
+    std::vector<float> data;
+};
+
+// quantize a model
+static bool whisper_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
+    gpt_vocab vocab;
+
+    printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
+
+    auto finp = std::ifstream(fname_inp, std::ios::binary);
+    if (!finp) {
+        fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
+        return false;
+    }
+
+    auto fout = std::ofstream(fname_out, std::ios::binary);
+    if (!fout) {
+        fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
+        return false;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        finp.read((char *) &magic, sizeof(magic));
+        if (magic != GGML_FILE_MAGIC) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
+            return false;
+        }
+
+        fout.write((char *) &magic, sizeof(magic));
+    }
+
+    whisper_hparams hparams;
+
+    // load hparams
+    {
+        finp.read((char *) &hparams.n_vocab,       sizeof(hparams.n_vocab));
+        finp.read((char *) &hparams.n_audio_ctx,   sizeof(hparams.n_audio_ctx));
+        finp.read((char *) &hparams.n_audio_state, sizeof(hparams.n_audio_state));
+        finp.read((char *) &hparams.n_audio_head,  sizeof(hparams.n_audio_head));
+        finp.read((char *) &hparams.n_audio_layer, sizeof(hparams.n_audio_layer));
+        finp.read((char *) &hparams.n_text_ctx,    sizeof(hparams.n_text_ctx));
+        finp.read((char *) &hparams.n_text_state,  sizeof(hparams.n_text_state));
+        finp.read((char *) &hparams.n_text_head,   sizeof(hparams.n_text_head));
+        finp.read((char *) &hparams.n_text_layer,  sizeof(hparams.n_text_layer));
+        finp.read((char *) &hparams.n_mels,        sizeof(hparams.n_mels));
+        finp.read((char *) &hparams.ftype,         sizeof(hparams.ftype));
+
+        const int32_t qntvr_src =    hparams.ftype / GGML_QNT_VERSION_FACTOR;
+        const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
+
+        fprintf(stderr, "%s: n_vocab       = %d\n", __func__, hparams.n_vocab);
+        fprintf(stderr, "%s: n_audio_ctx   = %d\n", __func__, hparams.n_audio_ctx);
+        fprintf(stderr, "%s: n_audio_state = %d\n", __func__, hparams.n_audio_state);
+        fprintf(stderr, "%s: n_audio_head  = %d\n", __func__, hparams.n_audio_head);
+        fprintf(stderr, "%s: n_audio_layer = %d\n", __func__, hparams.n_audio_layer);
+        fprintf(stderr, "%s: n_text_ctx    = %d\n", __func__, hparams.n_text_ctx);
+        fprintf(stderr, "%s: n_text_state  = %d\n", __func__, hparams.n_text_state);
+        fprintf(stderr, "%s: n_text_head   = %d\n", __func__, hparams.n_text_head);
+        fprintf(stderr, "%s: n_text_layer  = %d\n", __func__, hparams.n_text_layer);
+        fprintf(stderr, "%s: n_mels        = %d\n", __func__, hparams.n_mels);
+        fprintf(stderr, "%s: ftype (src)   = %d\n", __func__, hparams.ftype);
+        fprintf(stderr, "%s: qntvr (src)   = %d\n", __func__, qntvr_src);
+        fprintf(stderr, "%s: ftype (dst)   = %d\n", __func__, ftype_dst);
+        fprintf(stderr, "%s: qntvr (dst)   = %d\n", __func__, GGML_QNT_VERSION);
+
+        fout.write((const char *) &hparams.n_vocab,       sizeof(hparams.n_vocab));
+        fout.write((const char *) &hparams.n_audio_ctx,   sizeof(hparams.n_audio_ctx));
+        fout.write((const char *) &hparams.n_audio_state, sizeof(hparams.n_audio_state));
+        fout.write((const char *) &hparams.n_audio_head,  sizeof(hparams.n_audio_head));
+        fout.write((const char *) &hparams.n_audio_layer, sizeof(hparams.n_audio_layer));
+        fout.write((const char *) &hparams.n_text_ctx,    sizeof(hparams.n_text_ctx));
+        fout.write((const char *) &hparams.n_text_state,  sizeof(hparams.n_text_state));
+        fout.write((const char *) &hparams.n_text_head,   sizeof(hparams.n_text_head));
+        fout.write((const char *) &hparams.n_text_layer,  sizeof(hparams.n_text_layer));
+        fout.write((const char *) &hparams.n_mels,        sizeof(hparams.n_mels));
+        fout.write((const char *) &ftype_dst,             sizeof(hparams.ftype));
+    }
+
+    // load mel filters
+    {
+        whisper_filters filters;
+
+        finp.read ((char *) &filters.n_mel, sizeof(filters.n_mel));
+        fout.write((char *) &filters.n_mel, sizeof(filters.n_mel));
+        finp.read ((char *) &filters.n_fft, sizeof(filters.n_fft));
+        fout.write((char *) &filters.n_fft, sizeof(filters.n_fft));
+
+        filters.data.resize(filters.n_mel * filters.n_fft);
+        finp.read ((char *) filters.data.data(), filters.data.size() * sizeof(float));
+        fout.write((char *) filters.data.data(), filters.data.size() * sizeof(float));
+    }
+
+    // load vocab
+    {
+        int32_t n_vocab = 0;
+        finp.read ((char *) &n_vocab, sizeof(n_vocab));
+        fout.write((char *) &n_vocab, sizeof(n_vocab));
+
+        //if (n_vocab != hparams.n_vocab) {
+        //    fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+        //            __func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
+        //    return false;
+        //}
+
+        char word[129];
+
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            finp.read ((char *) &len, sizeof(len));
+            fout.write((char *) &len, sizeof(len));
+
+            word[len] = '\0';
+
+            finp.read ((char *) word, len);
+            fout.write((char *) word, len);
+
+            vocab.token_to_id[word] = i;
+            vocab.id_to_token[i] = word;
+        }
+    }
+
+    // regexes of tensor names to not be quantized
+    const std::vector<std::string> to_skip = {
+        //"encoder.*",
+        "encoder.conv1.bias",
+        "encoder.conv2.bias",
+        "encoder.positional_embedding",
+        "decoder.positional_embedding",
+    };
+
+    if (!ggml_common_quantize_0(finp, fout, ftype, { ".*" }, to_skip)) {
+        fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
+        return false;
+    }
+
+    finp.close();
+    fout.close();
+
+    return true;
+}
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    if (argc != 4) {
+        fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
+        ggml_print_ftypes(stderr);
+        return 1;
+    }
+
+    // needed to initialize f16 tables
+    {
+        struct ggml_init_params params = { 0, NULL, false };
+        struct ggml_context * ctx = ggml_init(params);
+        ggml_free(ctx);
+    }
+
+    const std::string fname_inp = argv[1];
+    const std::string fname_out = argv[2];
+
+    const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
+
+    const int64_t t_main_start_us = ggml_time_us();
+
+    int64_t t_quantize_us = 0;
+
+    // load the model
+    {
+        const int64_t t_start_us = ggml_time_us();
+
+        if (!whisper_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
+            fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
+            return 1;
+        }
+
+        t_quantize_us = ggml_time_us() - t_start_us;
+    }
+
+    // report timing
+    {
+        const int64_t t_main_end_us = ggml_time_us();
+
+        printf("\n");
+        printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
+        printf("%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
+    }
+
+    return 0;
+}

examples/server.py

@@ -0,0 +1,120 @@
+import http.server
+import socketserver
+import os
+import sys
+from pathlib import Path
+import urllib.parse
+
+SCRIPT_DIR = Path(__file__).parent.absolute()
+DIRECTORY = os.path.join(SCRIPT_DIR, "../build-em/bin")
+DIRECTORY = os.path.abspath(DIRECTORY)
+
+# The context root we want for all applications
+CONTEXT_ROOT = "/whisper.cpp"
+
+class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, directory=DIRECTORY, **kwargs)
+
+    def do_GET(self):
+        # Redirect root to the context root
+        if self.path == '/':
+            self.send_response(302)
+            self.send_header('Location', CONTEXT_ROOT + '/')
+            self.end_headers()
+            return
+
+        # Handle requests under the context root
+        if self.path.startswith(CONTEXT_ROOT):
+            # Remove the context root prefix to get the actual path
+            actual_path = self.path[len(CONTEXT_ROOT):]
+
+            if not actual_path:
+                self.send_response(302)
+                self.send_header('Location', CONTEXT_ROOT + '/')
+                self.end_headers()
+                return
+
+            if '.worker.js' in actual_path:
+                worker_file = os.path.basename(actual_path)
+                worker_path = os.path.join(DIRECTORY, worker_file)
+
+                if os.path.exists(worker_path):
+                    print(f"Found worker file: {worker_path}")
+                    self.path = '/' + worker_file
+                else:
+                    print(f"Worker file not found: {worker_path}")
+
+            elif actual_path == '/':
+                self.path = '/whisper.wasm/index.html'
+            elif any(actual_path.startswith(prefix) for prefix in (
+                '/bench.wasm/',
+                '/command.wasm/',
+                '/stream.wasm/',
+                '/wchess.wasm/'
+            )):
+                # Keep the path as is, just remove the context root
+                self.path = actual_path
+            # For all other paths under the context root
+            else:
+                # Check if this is a request to a file in whisper.wasm
+                potential_file = os.path.join(DIRECTORY, 'whisper.wasm', actual_path.lstrip('/'))
+                if os.path.exists(potential_file) and not os.path.isdir(potential_file):
+                    self.path = '/whisper.wasm' + actual_path
+                else:
+                    # Try to resolve the file from the base directory
+                    potential_file = os.path.join(DIRECTORY, actual_path.lstrip('/'))
+                    if os.path.exists(potential_file):
+                        self.path = actual_path
+
+        # For direct requests to worker files (without context root as these
+        # are in the build-em/bin directory
+        elif '.worker.js' in self.path:
+            worker_file = os.path.basename(self.path)
+            worker_path = os.path.join(DIRECTORY, worker_file)
+
+            if os.path.exists(worker_path):
+                self.path = '/' + worker_file
+
+        # Handle coi-serviceworker.js separately
+        if 'coi-serviceworker.js' in self.path:
+            worker_file = "coi-serviceworker.js"
+            worker_path = os.path.join(SCRIPT_DIR, worker_file)
+            if os.path.exists(worker_path):
+                self.send_response(200)
+                self.send_header('Content-type', 'application/javascript')
+                self.end_headers()
+                with open(worker_path, 'rb') as file:
+                    self.wfile.write(file.read())
+                return
+            else:
+                print(f"Warning: Could not find {worker_path}")
+
+        return super().do_GET()
+
+    def end_headers(self):
+        # Add required headers for SharedArrayBuffer
+        self.send_header("Cross-Origin-Opener-Policy", "same-origin")
+        self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
+        self.send_header("Access-Control-Allow-Origin", "*")
+        super().end_headers()
+
+PORT = 8000
+
+# Enable address reuse
+class CustomServer(socketserver.TCPServer):
+    allow_reuse_address = True
+
+try:
+    with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd:
+        print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}")
+        print(f"Application context root: http://localhost:{PORT}{CONTEXT_ROOT}/")
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            print("\nServer stopped.")
+            # Force complete exit
+            sys.exit(0)
+except OSError as e:
+    print(f"Error: {e}")
+    sys.exit(1)

examples/server/CMakeLists.txt

@@ -0,0 +1,15 @@
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(TARGET whisper-server)
+add_executable(${TARGET} server.cpp httplib.h)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE common json_cpp whisper ${CMAKE_THREAD_LIBS_INIT})
+
+if (WIN32)
+    target_link_libraries(${TARGET} PRIVATE ws2_32)
+endif()
+
+install(TARGETS ${TARGET} RUNTIME)

examples/server/README.md

@@ -0,0 +1,124 @@
+# whisper.cpp/examples/server
+
+Simple http server. WAV Files are passed to the inference model via http requests.
+
+https://github.com/ggerganov/whisper.cpp/assets/1991296/e983ee53-8741-4eb5-9048-afe5e4594b8f
+
+## Usage
+
+```
+./build/bin/whisper-server -h
+
+usage: ./build/bin/whisper-server [options]
+
+options:
+  -h,        --help              [default] show this help message and exit
+  -t N,      --threads N         [4      ] number of threads to use during computation
+  -p N,      --processors N      [1      ] number of processors to use during computation
+  -ot N,     --offset-t N        [0      ] time offset in milliseconds
+  -on N,     --offset-n N        [0      ] segment index offset
+  -d  N,     --duration N        [0      ] duration of audio to process in milliseconds
+  -mc N,     --max-context N     [-1     ] maximum number of text context tokens to store
+  -ml N,     --max-len N         [0      ] maximum segment length in characters
+  -sow,      --split-on-word     [false  ] split on word rather than on token
+  -bo N,     --best-of N         [2      ] number of best candidates to keep
+  -bs N,     --beam-size N       [-1     ] beam size for beam search
+  -ac N,     --audio-ctx N       [0      ] audio context size (0 - all)
+  -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold
+  -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail
+  -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail
+  -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)
+  -tr,       --translate         [false  ] translate from source language to english
+  -di,       --diarize           [false  ] stereo audio diarization
+  -tdrz,     --tinydiarize       [false  ] enable tinydiarize (requires a tdrz model)
+  -nf,       --no-fallback       [false  ] do not use temperature fallback while decoding
+  -ps,       --print-special     [false  ] print special tokens
+  -pc,       --print-colors      [false  ] print colors
+  -pr,       --print-realtime    [false  ] print output in realtime
+  -pp,       --print-progress    [false  ] print progress
+  -nt,       --no-timestamps     [false  ] do not print timestamps
+  -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
+  -dl,       --detect-language   [false  ] exit after automatically detecting language
+             --prompt PROMPT     [       ] initial prompt
+             --carry-initial-prompt [false  ] always prepend initial prompt
+  -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
+  -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference
+  -dtw MODEL --dtw MODEL         [       ] compute token-level timestamps
+  --host HOST,                   [127.0.0.1] Hostname/ip-adress for the server
+  --port PORT,                   [8080   ] Port number for the server
+  --public PATH,                 [examples/server/public] Path to the public folder
+  --request-path PATH,           [       ] Request path for all requests
+  --inference-path PATH,         [/inference] Inference path for all requests
+  --convert,                     [false  ] Convert audio to WAV, requires ffmpeg on the server
+  -sns,      --suppress-nst      [false  ] suppress non-speech tokens
+  -nth N,    --no-speech-thold N [0.60   ] no speech threshold
+  -nc,       --no-context        [false  ] do not use previous audio context
+  -ng,       --no-gpu            [false  ] do not use gpu
+  -fa,       --flash-attn        [false  ] flash attention
+
+Voice Activity Detection (VAD) options:
+             --vad                           [false  ] enable Voice Activity Detection (VAD)
+  -vm FNAME, --vad-model FNAME               [       ] VAD model path
+  -vt N,     --vad-threshold N               [0.50   ] VAD threshold for speech recognition
+  -vspd N,   --vad-min-speech-duration-ms  N [250    ] VAD min speech duration (0.0-1.0)
+  -vsd N,    --vad-min-silence-duration-ms N [100    ] VAD min silence duration (to split segments)
+  -vmsd N,   --vad-max-speech-duration-s   N [FLT_MAX] VAD max speech duration (auto-split longer)
+  -vp N,     --vad-speech-pad-ms           N [30     ] VAD speech padding (extend segments)
+  -vo N,     --vad-samples-overlap         N [0.10   ] VAD samples overlap (seconds between segments)
+```
+
+> [!WARNING]
+> **Do not run the server example with administrative privileges and ensure it's operated in a sandbox environment, especially since it involves risky operations like accepting user file uploads and using ffmpeg for format conversions. Always validate and sanitize inputs to guard against potential security threats.**
+
+## request examples
+
+**/inference**
+```
+curl 127.0.0.1:8080/inference \
+-H "Content-Type: multipart/form-data" \
+-F file="@<file-path>" \
+-F temperature="0.0" \
+-F temperature_inc="0.2" \
+-F prompt="<prompt>" \
+-F carry_initial_prompt="true" \
+-F response_format="json"
+```
+
+**/load**
+```
+curl 127.0.0.1:8080/load \
+-H "Content-Type: multipart/form-data" \
+-F model="<path-to-model-file>"
+```
+
+## Load testing with k6
+
+> **Note:** Install [k6](https://k6.io/docs/get-started/installation/) before running the benchmark script.
+
+You can benchmark the Whisper server using the provided bench.js script with [k6](https://k6.io/). This script sends concurrent multipart requests to the /inference endpoint and is fully configurable via environment variables.
+
+**Example usage:**
+
+```
+k6 run bench.js \
+  --env FILE_PATH=/absolute/path/to/samples/jfk.wav \
+  --env BASE_URL=http://127.0.0.1:8080 \
+  --env ENDPOINT=/inference \
+  --env CONCURRENCY=4 \
+  --env TEMPERATURE=0.0 \
+  --env TEMPERATURE_INC=0.2 \
+  --env RESPONSE_FORMAT=json
+```
+
+**Environment variables:**
+- `FILE_PATH`: Path to the audio file to send (must be absolute or relative to the k6 working directory)
+- `BASE_URL`: Server base URL (default: `http://127.0.0.1:8080`)
+- `ENDPOINT`: API endpoint (default: `/inference`)
+- `CONCURRENCY`: Number of concurrent requests (default: 4)
+- `TEMPERATURE`: Decoding temperature (default: 0.0)
+- `TEMPERATURE_INC`: Temperature increment (default: 0.2)
+- `RESPONSE_FORMAT`: Response format (default: `json`)
+
+**Note:**
+- The server must be running and accessible at the specified `BASE_URL` and `ENDPOINT`.
+- The script is located in the same directory as this README: `bench.js`.

examples/server/bench.js

@@ -0,0 +1,29 @@
+import http from 'k6/http'
+import { check } from 'k6'
+
+export let options = {
+  vus: parseInt(__ENV.CONCURRENCY) || 4,
+  iterations: parseInt(__ENV.CONCURRENCY) || 4,
+}
+
+const filePath        = __ENV.FILE_PATH
+const baseURL         = __ENV.BASE_URL        || 'http://127.0.0.1:8080'
+const endpoint        = __ENV.ENDPOINT        || '/inference'
+const temperature     = __ENV.TEMPERATURE     || '0.0'
+const temperatureInc  = __ENV.TEMPERATURE_INC || '0.2'
+const responseFormat  = __ENV.RESPONSE_FORMAT || 'json'
+
+// Read the file ONCE at init time
+const fileBin = open(filePath, 'b')
+
+export default function () {
+  const payload = {
+    file:           http.file(fileBin, filePath),
+    temperature:    temperature,
+    temperature_inc: temperatureInc,
+    response_format: responseFormat,
+  }
+
+  const res = http.post(`${baseURL}${endpoint}`, payload)
+  check(res, { 'status is 200': r => r.status === 200 })
+} 

examples/stream.wasm/CMakeLists.txt

@@ -0,0 +1,49 @@
+#
+# libstream
+#
+
+set(TARGET libstream)
+
+add_executable(${TARGET}
+    emscripten.cpp
+    )
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE
+    whisper
+    )
+
+unset(EXTRA_FLAGS)
+
+if (WHISPER_WASM_SINGLE_FILE)
+    set(EXTRA_FLAGS "-s SINGLE_FILE=1")
+    message(STATUS "Embedding WASM inside stream.js")
+
+    add_custom_command(
+        TARGET ${TARGET} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${CMAKE_BINARY_DIR}/bin/libstream.js
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/stream.wasm/stream.js
+        )
+endif()
+
+set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
+    --bind \
+    -s USE_PTHREADS=1 \
+    -s PTHREAD_POOL_SIZE=8 \
+    -s INITIAL_MEMORY=1024MB \
+    -s TOTAL_MEMORY=1024MB \
+    -s FORCE_FILESYSTEM=1 \
+    -s EXPORTED_RUNTIME_METHODS=\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\" \
+    ${EXTRA_FLAGS} \
+    ")
+
+#
+# stream.wasm
+#
+
+set(TARGET stream.wasm)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)

examples/stream.wasm/README.md

@@ -0,0 +1,45 @@
+# stream.wasm
+
+Real-time transcription in the browser using WebAssembly
+
+Online demo: https://ggml.ai/whisper.cpp/stream.wasm/
+
+## Build instructions
+
+```bash
+# build using Emscripten (v3.1.2)
+git clone https://github.com/ggerganov/whisper.cpp
+cd whisper.cpp
+mkdir build-em && cd build-em
+emcmake cmake ..
+make -j
+```
+The example can then be started by running a local HTTP server:
+```console
+python3 examples/server.py
+```
+And then opening a browser to the following URL:
+http://localhost:8000/stream.wasm
+
+To run the example in a different server, you need to copy the following files
+to the server's HTTP path:
+```
+# copy the produced page to your HTTP path
+cp bin/stream.wasm/*       /path/to/html/
+cp bin/libstream.js        /path/to/html/
+cp bin/libstream.worker.js /path/to/html/
+```
+
+> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`
+> which means that that a separate .wasm file will not be generated. Instead, the
+> WASM module is embedded in the main JS file as a base64 encoded string. To
+> generate a separate .wasm file, you need to disable this option by passing
+> `-DWHISPER_WASM_SINGLE_FILE=OFF`:
+> ```console
+> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF
+> ```
+> This will generate a `libstream.wasm` file in the build/bin directory.
+
+> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no
+> longer generated and the worker is embedded in the main JS file. So the worker
+> file will not be generated for versions later than `3.1.58`.

examples/stream.wasm/emscripten.cpp

@@ -0,0 +1,216 @@
+#include "ggml.h"
+#include "whisper.h"
+
+#include <emscripten.h>
+#include <emscripten/bind.h>
+
+#include <atomic>
+#include <cmath>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+constexpr int N_THREAD = 8;
+
+std::vector<struct whisper_context *> g_contexts(4, nullptr);
+
+std::mutex g_mutex;
+std::thread g_worker;
+
+std::atomic<bool> g_running(false);
+
+std::string g_status        = "";
+std::string g_status_forced = "";
+std::string g_transcribed   = "";
+
+std::vector<float> g_pcmf32;
+
+void stream_set_status(const std::string & status) {
+    std::lock_guard<std::mutex> lock(g_mutex);
+    g_status = status;
+}
+
+void stream_main(size_t index, const std::string & lang) {
+    stream_set_status("loading data ...");
+
+    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);
+    bool is_multilingual = whisper_is_multilingual(g_contexts[index]);
+
+    wparams.n_threads        = std::min(N_THREAD, (int) std::thread::hardware_concurrency());
+    wparams.offset_ms        = 0;
+    wparams.translate        = false;
+    wparams.no_context       = true;
+    wparams.single_segment   = true;
+    wparams.print_realtime   = false;
+    wparams.print_progress   = false;
+    wparams.print_timestamps = true;
+    wparams.print_special    = false;
+
+    wparams.max_tokens       = 32;
+    wparams.audio_ctx        = 768; // partial encoder context for better performance
+
+    // disable temperature fallback
+    wparams.temperature_inc  = -1.0f;
+
+    wparams.language         = is_multilingual ? lang.c_str() : "en";
+
+    printf("stream: using %d threads\n", wparams.n_threads);
+
+    std::vector<float> pcmf32;
+
+    // whisper context
+    auto & ctx = g_contexts[index];
+
+    // 5 seconds interval
+    const int64_t window_samples = 5*WHISPER_SAMPLE_RATE;
+
+    while (g_running) {
+        stream_set_status("waiting for audio ...");
+
+        {
+            std::unique_lock<std::mutex> lock(g_mutex);
+
+            if (g_pcmf32.size() < 1024) {
+                lock.unlock();
+
+                std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+                continue;
+            }
+
+            pcmf32 = std::vector<float>(g_pcmf32.end() - std::min((int64_t) g_pcmf32.size(), window_samples), g_pcmf32.end());
+            g_pcmf32.clear();
+        }
+
+        {
+            const auto t_start = std::chrono::high_resolution_clock::now();
+
+            stream_set_status("running whisper ...");
+
+            int ret = whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size());
+            if (ret != 0) {
+                printf("whisper_full() failed: %d\n", ret);
+                break;
+            }
+
+            const auto t_end = std::chrono::high_resolution_clock::now();
+
+            printf("stream: whisper_full() returned %d in %f seconds\n", ret, std::chrono::duration<double>(t_end - t_start).count());
+        }
+
+        {
+            std::string text_heard;
+
+            {
+                const int n_segments = whisper_full_n_segments(ctx);
+                if (n_segments > 0) {
+                    const char * text = whisper_full_get_segment_text(ctx, n_segments - 1);
+
+                    const int64_t t0 = whisper_full_get_segment_t0(ctx, n_segments - 1);
+                    const int64_t t1 = whisper_full_get_segment_t1(ctx, n_segments - 1);
+
+                    printf("transcribed: %s\n", text);
+
+                    text_heard += text;
+                }
+            }
+
+            {
+                std::lock_guard<std::mutex> lock(g_mutex);
+                g_transcribed = text_heard;
+            }
+        }
+    }
+
+    if (index < g_contexts.size()) {
+        whisper_free(g_contexts[index]);
+        g_contexts[index] = nullptr;
+    }
+}
+EMSCRIPTEN_BINDINGS(stream) {
+    emscripten::function("init", emscripten::optional_override([](const std::string & path_model, const std::string & lang) {
+        for (size_t i = 0; i < g_contexts.size(); ++i) {
+            if (g_contexts[i] == nullptr) {
+                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());
+                if (g_contexts[i] != nullptr) {
+                    g_running = true;
+                    if (g_worker.joinable()) {
+                        g_worker.join();
+                    }
+                    g_worker = std::thread([i, lang]() {
+                        stream_main(i, lang);
+                    });
+
+                    return i + 1;
+                } else {
+                    return (size_t) 0;
+                }
+            }
+        }
+
+        return (size_t) 0;
+    }));
+
+    emscripten::function("free", emscripten::optional_override([](size_t index) {
+        if (g_running) {
+            g_running = false;
+        }
+    }));
+
+    emscripten::function("set_audio", emscripten::optional_override([](size_t index, const emscripten::val & audio) {
+        --index;
+
+        if (index >= g_contexts.size()) {
+            return -1;
+        }
+
+        if (g_contexts[index] == nullptr) {
+            return -2;
+        }
+
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            const int n = audio["length"].as<int>();
+
+            emscripten::val heap = emscripten::val::module_property("HEAPU8");
+            emscripten::val memory = heap["buffer"];
+
+            g_pcmf32.resize(n);
+
+            emscripten::val memoryView = audio["constructor"].new_(memory, reinterpret_cast<uintptr_t>(g_pcmf32.data()), n);
+            memoryView.call<void>("set", audio);
+        }
+
+        return 0;
+    }));
+
+    emscripten::function("get_transcribed", emscripten::optional_override([]() {
+        std::string transcribed;
+
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            transcribed = std::move(g_transcribed);
+        }
+
+        return transcribed;
+    }));
+
+    emscripten::function("get_status", emscripten::optional_override([]() {
+        std::string status;
+
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            status = g_status_forced.empty() ? g_status : g_status_forced;
+        }
+
+        return status;
+    }));
+
+    emscripten::function("set_status", emscripten::optional_override([](const std::string & status) {
+        {
+            std::lock_guard<std::mutex> lock(g_mutex);
+            g_status_forced = status;
+        }
+    }));
+}

examples/stream.wasm/index-tmpl.html

@@ -0,0 +1,491 @@
+<!doctype html>
+<html lang="en-us">
+    <head>
+        <title>stream : Real-time Whisper transcription in WebAssembly</title>
+
+        <style>
+            #output {
+                width: 100%;
+                height: 100%;
+                margin: 0 auto;
+                margin-top: 10px;
+                border-left: 0px;
+                border-right: 0px;
+                padding-left: 0px;
+                padding-right: 0px;
+                display: block;
+                background-color: black;
+                color: white;
+                font-size: 10px;
+                font-family: 'Lucida Console', Monaco, monospace;
+                outline: none;
+                white-space: pre;
+                overflow-wrap: normal;
+                overflow-x: scroll;
+            }
+        </style>
+        <script src="../coi-serviceworker.js"></script>
+        <link rel="icon" href="data:,">
+    </head>
+    <body>
+        <div id="main-container">
+            <b>stream : Real-time Whisper transcription in WebAssembly</b>
+
+            <br><br>
+
+            You can find more about this project on <a href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/stream.wasm">GitHub</a>.
+
+            <br><br>
+
+            <b>More examples:</b>
+                <a href="../">main</a> |
+                <a href="../bench.wasm/">bench</a> |
+                <a href="../stream.wasm">stream</a> |
+                <a href="../command.wasm/">command</a> |
+                <a href="../wchess.wasm/">wchess</a> |
+
+            <br><br>
+
+            <hr>
+
+            Select the model you would like to use, click the "Start" button and start speaking
+
+            <br><br>
+
+            <div id="model-whisper">
+                Whisper model: <span id="model-whisper-status"></span>
+                <button id="fetch-whisper-tiny-en" onclick="loadWhisper('tiny.en')">tiny.en (75 MB)</button>
+                <button id="fetch-whisper-base-en" onclick="loadWhisper('base.en')">base.en (142 MB)</button>
+                <button id="fetch-whisper-base" onclick="loadWhisper('base')">base (142 MB)</button>
+                <br><br>
+                Quantized models:<br><br>
+                <button id="fetch-whisper-tiny-en-q5_1"   onclick="loadWhisper('tiny-en-q5_1')">tiny.en (Q5_1, 31 MB)</button>
+                <button id="fetch-whisper-base-en-q5_1"   onclick="loadWhisper('base-en-q5_1')">base.en (Q5_1, 57 MB)</button>
+                <span id="fetch-whisper-progress"></span>
+
+                <!--
+                    <input type="file" id="file" name="file" onchange="loadFile(event, 'whisper.bin')" />
+                -->
+            </div>
+
+            <table>
+                <tr>
+                    <td>
+                        Language:
+                        <select id="language" name="language">
+                            <option value="en">English</option>
+                            <option value="ar">Arabic</option>
+                            <option value="hy">Armenian</option>
+                            <option value="az">Azerbaijani</option>
+                            <option value="eu">Basque</option>
+                            <option value="be">Belarusian</option>
+                            <option value="bn">Bengali</option>
+                            <option value="bg">Bulgarian</option>
+                            <option value="ca">Catalan</option>
+                            <option value="zh">Chinese</option>
+                            <option value="hr">Croatian</option>
+                            <option value="cs">Czech</option>
+                            <option value="da">Danish</option>
+                            <option value="nl">Dutch</option>
+                            <option value="en">English</option>
+                            <option value="et">Estonian</option>
+                            <option value="tl">Filipino</option>
+                            <option value="fi">Finnish</option>
+                            <option value="fr">French</option>
+                            <option value="gl">Galician</option>
+                            <option value="ka">Georgian</option>
+                            <option value="de">German</option>
+                            <option value="el">Greek</option>
+                            <option value="gu">Gujarati</option>
+                            <option value="iw">Hebrew</option>
+                            <option value="hi">Hindi</option>
+                            <option value="hu">Hungarian</option>
+                            <option value="is">Icelandic</option>
+                            <option value="id">Indonesian</option>
+                            <option value="ga">Irish</option>
+                            <option value="it">Italian</option>
+                            <option value="ja">Japanese</option>
+                            <option value="kn">Kannada</option>
+                            <option value="ko">Korean</option>
+                            <option value="la">Latin</option>
+                            <option value="lv">Latvian</option>
+                            <option value="lt">Lithuanian</option>
+                            <option value="mk">Macedonian</option>
+                            <option value="ms">Malay</option>
+                            <option value="mt">Maltese</option>
+                            <option value="no">Norwegian</option>
+                            <option value="fa">Persian</option>
+                            <option value="pl">Polish</option>
+                            <option value="pt">Portuguese</option>
+                            <option value="ro">Romanian</option>
+                            <option value="ru">Russian</option>
+                            <option value="sr">Serbian</option>
+                            <option value="sk">Slovak</option>
+                            <option value="sl">Slovenian</option>
+                            <option value="es">Spanish</option>
+                            <option value="sw">Swahili</option>
+                            <option value="sv">Swedish</option>
+                            <option value="ta">Tamil</option>
+                            <option value="te">Telugu</option>
+                            <option value="th">Thai</option>
+                            <option value="tr">Turkish</option>
+                            <option value="uk">Ukrainian</option>
+                            <option value="ur">Urdu</option>
+                            <option value="vi">Vietnamese</option>
+                            <option value="cy">Welsh</option>
+                            <option value="yi">Yiddish</option>
+                        </select>
+                    </td>
+                </tr>
+            </table>
+
+            <br>
+
+            <div id="input">
+                <button id="start"  onclick="onStart()" disabled>Start</button>
+                <button id="stop"   onclick="onStop()" disabled>Stop</button>
+                <button id="clear"  onclick="clearCache()">Clear Cache</button>
+            </div>
+
+            <br>
+
+            <div id="state">
+                Status: <b><span id="state-status">not started</span></b>
+
+                <pre id="state-transcribed">[The transcribed text will be displayed here]</pre>
+            </div>
+
+            <hr>
+
+            Debug output:
+            <textarea id="output" rows="20"></textarea>
+
+            <br>
+
+            <b>Troubleshooting</b>
+
+            <br><br>
+
+            The page does some heavy computations, so make sure:
+
+            <ul>
+                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>
+                <li>To use a fast desktop or laptop computer (i.e. not a mobile phone)</li>
+                <li>Your browser supports WASM <a href="https://webassembly.org/roadmap/">Fixed-width SIMD</a></li>
+            </ul>
+
+            <div class="cell-version">
+                <span>
+                    |
+                    Build time: <span class="nav-link">@GIT_DATE@</span> |
+                    Commit hash: <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@">@GIT_SHA1@</a> |
+                    Commit subject: <span class="nav-link">@GIT_COMMIT_SUBJECT@</span> |
+                    <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/stream.wasm">Source Code</a> |
+                </span>
+            </div>
+        </div>
+
+        <script type="text/javascript" src="helpers.js"></script>
+        <script type='text/javascript'>
+            // web audio context
+            var context = null;
+
+            // audio data
+            var audio = null;
+            var audio0 = null;
+
+            // the stream instance
+            var instance = null;
+
+            // model name
+            var model_whisper = null;
+
+            var Module = {
+                print: printTextarea,
+                printErr: printTextarea,
+                setStatus: function(text) {
+                    printTextarea('js: ' + text);
+                },
+                monitorRunDependencies: function(left) {
+                },
+                preRun: function() {
+                    printTextarea('js: Preparing ...');
+                },
+                postRun: function() {
+                    printTextarea('js: Initialized successfully!');
+                }
+            };
+
+            //
+            // fetch models
+            //
+
+            let dbVersion = 1
+            let dbName    = 'whisper.ggerganov.com';
+            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB
+
+            function storeFS(fname, buf) {
+                // write to WASM file using FS_createDataFile
+                // if the file exists, delete it
+                try {
+                    Module.FS_unlink(fname);
+                } catch (e) {
+                    // ignore
+                }
+
+                Module.FS_createDataFile("/", fname, buf, true, true);
+
+                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);
+
+                document.getElementById('model-whisper-status').innerHTML = 'loaded "' + model_whisper + '"!';
+
+                if (model_whisper != null) {
+                    document.getElementById('start').disabled = false;
+                    document.getElementById('stop' ).disabled = true;
+                }
+            }
+
+            function loadWhisper(model) {
+                let urls = {
+                    'tiny.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',
+                    'base.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',
+                    'base'   : 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin',
+
+                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',
+                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',
+                };
+
+                let sizes = {
+                    'tiny.en': 75,
+                    'base.en': 142,
+                    'base':     142,
+
+                    'tiny-en-q5_1':   31,
+                    'base-en-q5_1':   57,
+                };
+
+                let url     = urls[model];
+                let dst     = 'whisper.bin';
+                let size_mb = sizes[model];
+
+                model_whisper = model;
+
+                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';
+                document.getElementById('fetch-whisper-base-en').style.display = 'none';
+                document.getElementById('fetch-whisper-base').style.display = 'none';
+
+                document.getElementById('fetch-whisper-tiny-en-q5_1').style.display = 'none';
+                document.getElementById('fetch-whisper-base-en-q5_1').style.display = 'none';
+
+                document.getElementById('model-whisper-status').innerHTML = 'loading "' + model + '" ... ';
+
+                cbProgress = function(p) {
+                    let el = document.getElementById('fetch-whisper-progress');
+                    el.innerHTML = Math.round(100*p) + '%';
+                };
+
+                cbCancel = function() {
+                    var el;
+                    el = document.getElementById('fetch-whisper-tiny-en'); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base-en'); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base'); if (el) el.style.display = 'inline-block';
+
+                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'); if (el) el.style.display = 'inline-block';
+                    el = document.getElementById('fetch-whisper-base-en-q5_1'); if (el) el.style.display = 'inline-block';
+
+                    el = document.getElementById('model-whisper-status');  if (el) el.innerHTML = '';
+                };
+
+                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);
+            }
+
+            //
+            // microphone
+            //
+
+            const kSampleRate = 16000;
+            const kRestartRecording_s = 120;
+            const kIntervalAudio_ms = 5000; // pass the recorded audio to the C++ instance at this rate
+
+            var mediaRecorder = null;
+            var doRecording = false;
+            var startTime = 0;
+
+            window.AudioContext = window.AudioContext || window.webkitAudioContext;
+            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;
+
+            function stopRecording() {
+                Module.set_status("paused");
+                doRecording = false;
+                audio0 = null;
+                audio = null;
+                context = null;
+            }
+
+            function startRecording() {
+                if (!context) {
+                    context = new AudioContext({
+                        sampleRate: kSampleRate,
+                        channelCount: 1,
+                        echoCancellation: false,
+                        autoGainControl:  true,
+                        noiseSuppression: true,
+                    });
+                }
+
+                Module.set_status("");
+
+                document.getElementById('start').disabled = true;
+                document.getElementById('stop').disabled = false;
+
+                doRecording = true;
+                startTime = Date.now();
+
+                var chunks = [];
+                var stream = null;
+
+                navigator.mediaDevices.getUserMedia({audio: true, video: false})
+                    .then(function(s) {
+                        stream = s;
+                        mediaRecorder = new MediaRecorder(stream);
+                        mediaRecorder.ondataavailable = function(e) {
+                            chunks.push(e.data);
+
+                            var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
+                            var reader = new FileReader();
+
+                            reader.onload = function(event) {
+                                var buf = new Uint8Array(reader.result);
+
+                                if (!context) {
+                                    return;
+                                }
+                                context.decodeAudioData(buf.buffer, function(audioBuffer) {
+                                    var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);
+                                    var source = offlineContext.createBufferSource();
+                                    source.buffer = audioBuffer;
+                                    source.connect(offlineContext.destination);
+                                    source.start(0);
+
+                                    offlineContext.startRendering().then(function(renderedBuffer) {
+                                        audio = renderedBuffer.getChannelData(0);
+
+                                        //printTextarea('js: audio recorded, size: ' + audio.length + ', old size: ' + (audio0 == null ? 0 : audio0.length));
+
+                                        var audioAll = new Float32Array(audio0 == null ? audio.length : audio0.length + audio.length);
+                                        if (audio0 != null) {
+                                            audioAll.set(audio0, 0);
+                                        }
+                                        audioAll.set(audio, audio0 == null ? 0 : audio0.length);
+
+                                        if (instance) {
+                                            Module.set_audio(instance, audioAll);
+                                        }
+                                    });
+                                }, function(e) {
+                                    audio = null;
+                                });
+                            }
+
+                            reader.readAsArrayBuffer(blob);
+                        };
+
+                        mediaRecorder.onstop = function(e) {
+                            if (doRecording) {
+                                setTimeout(function() {
+                                    startRecording();
+                                });
+                            }
+                        };
+
+                        mediaRecorder.start(kIntervalAudio_ms);
+                    })
+                    .catch(function(err) {
+                        printTextarea('js: error getting audio stream: ' + err);
+                    });
+
+                var interval = setInterval(function() {
+                    if (!doRecording) {
+                        clearInterval(interval);
+                        mediaRecorder.stop();
+                        stream.getTracks().forEach(function(track) {
+                            track.stop();
+                        });
+
+                        document.getElementById('start').disabled = false;
+                        document.getElementById('stop').disabled  = true;
+
+                        mediaRecorder = null;
+                    }
+
+                    // if audio length is more than kRestartRecording_s seconds, restart recording
+                    if (audio != null && audio.length > kSampleRate*kRestartRecording_s) {
+                        if (doRecording) {
+                            //printTextarea('js: restarting recording');
+
+                            clearInterval(interval);
+                            audio0 = audio;
+                            audio = null;
+                            mediaRecorder.stop();
+                            stream.getTracks().forEach(function(track) {
+                                track.stop();
+                            });
+                        }
+                    }
+                }, 100);
+            }
+
+            //
+            // main
+            //
+
+            var nLines = 0;
+            var intervalUpdate = null;
+            var transcribedAll = '';
+
+            function onStart() {
+                if (!instance) {
+                    instance = Module.init('whisper.bin', document.getElementById('language').value);
+
+                    if (instance) {
+                        printTextarea("js: whisper initialized, instance: " + instance);
+                    }
+                }
+
+                if (!instance) {
+                    printTextarea("js: failed to initialize whisper");
+                    return;
+                }
+
+                startRecording();
+
+                intervalUpdate = setInterval(function() {
+                    var transcribed = Module.get_transcribed();
+
+                    if (transcribed != null && transcribed.length > 1) {
+                        transcribedAll += transcribed + '<br>';
+                        nLines++;
+
+                        // if more than 10 lines, remove the first line
+                        if (nLines > 10) {
+                            var i = transcribedAll.indexOf('<br>');
+                            if (i > 0) {
+                                transcribedAll = transcribedAll.substring(i + 4);
+                                nLines--;
+                            }
+                        }
+                    }
+
+                    document.getElementById('state-status').innerHTML = Module.get_status();
+                    document.getElementById('state-transcribed').innerHTML = transcribedAll;
+                }, 100);
+            }
+
+            function onStop() {
+                stopRecording();
+            }
+
+        </script>
+        <script type="text/javascript" src="stream.js"></script>
+    </body>
+</html>

examples/stream/CMakeLists.txt

@@ -0,0 +1,10 @@
+if (WHISPER_SDL2)
+    set(TARGET whisper-stream)
+    add_executable(${TARGET} stream.cpp)
+
+    include(DefaultTargetOptions)
+
+    target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})
+
+    install(TARGETS ${TARGET} RUNTIME)
+endif ()

examples/stream/README.md

@@ -0,0 +1,51 @@
+# whisper.cpp/examples/stream
+
+This is a naive example of performing real-time inference on audio from your microphone.
+The `whisper-stream` tool samples the audio every half a second and runs the transcription continuously.
+More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
+
+```bash
+./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
+```
+
+https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4
+
+## Sliding window mode with VAD
+
+Setting the `--step` argument to `0` enables the sliding window mode:
+
+```bash
+ ./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 6 --step 0 --length 30000 -vth 0.6
+```
+
+In this mode, the tool will transcribe only after some speech activity is detected. A very
+basic VAD detector is used, but in theory a more sophisticated approach can be added. The
+`-vth` argument determines the VAD threshold - higher values will make it detect silence more often.
+It's best to tune it to the specific use case, but a value around `0.6` should be OK in general.
+When silence is detected, it will transcribe the last `--length` milliseconds of audio and output
+a transcription block that is suitable for parsing.
+
+## Building
+
+The `whisper-stream` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
+
+```bash
+# Install SDL2
+# On Debian based linux distributions:
+sudo apt-get install libsdl2-dev
+
+# On Fedora Linux:
+sudo dnf install SDL2 SDL2-devel
+
+# Install SDL2 on Mac OS
+brew install sdl2
+
+cmake -B build -DWHISPER_SDL2=ON
+cmake --build build --config Release
+
+./build/bin/whisper-stream
+```
+
+## Web version
+
+This tool can also run in the browser: [examples/stream.wasm](/examples/stream.wasm)

examples/stream/stream.cpp

@@ -0,0 +1,437 @@
+// Real-time speech recognition of input from a microphone
+//
+// A very quick-n-dirty implementation serving mainly as a proof of concept.
+//
+#include "common-sdl.h"
+#include "common.h"
+#include "common-whisper.h"
+#include "whisper.h"
+
+#include <chrono>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+// command-line parameters
+struct whisper_params {
+    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t step_ms    = 3000;
+    int32_t length_ms  = 10000;
+    int32_t keep_ms    = 200;
+    int32_t capture_id = -1;
+    int32_t max_tokens = 32;
+    int32_t audio_ctx  = 0;
+    int32_t beam_size  = -1;
+
+    float vad_thold    = 0.6f;
+    float freq_thold   = 100.0f;
+
+    bool translate     = false;
+    bool no_fallback   = false;
+    bool print_special = false;
+    bool no_context    = true;
+    bool no_timestamps = false;
+    bool tinydiarize   = false;
+    bool save_audio    = false; // save audio to wav file
+    bool use_gpu       = true;
+    bool flash_attn    = true;
+
+    std::string language  = "en";
+    std::string model     = "models/ggml-base.en.bin";
+    std::string fname_out;
+};
+
+void whisper_print_usage(int argc, char ** argv, const whisper_params & params);
+
+static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-h" || arg == "--help") {
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+        else if (arg == "-t"    || arg == "--threads")       { params.n_threads     = std::stoi(argv[++i]); }
+        else if (                  arg == "--step")          { params.step_ms       = std::stoi(argv[++i]); }
+        else if (                  arg == "--length")        { params.length_ms     = std::stoi(argv[++i]); }
+        else if (                  arg == "--keep")          { params.keep_ms       = std::stoi(argv[++i]); }
+        else if (arg == "-c"    || arg == "--capture")       { params.capture_id    = std::stoi(argv[++i]); }
+        else if (arg == "-mt"   || arg == "--max-tokens")    { params.max_tokens    = std::stoi(argv[++i]); }
+        else if (arg == "-ac"   || arg == "--audio-ctx")     { params.audio_ctx     = std::stoi(argv[++i]); }
+        else if (arg == "-bs"   || arg == "--beam-size")     { params.beam_size     = std::stoi(argv[++i]); }
+        else if (arg == "-vth"  || arg == "--vad-thold")     { params.vad_thold     = std::stof(argv[++i]); }
+        else if (arg == "-fth"  || arg == "--freq-thold")    { params.freq_thold    = std::stof(argv[++i]); }
+        else if (arg == "-tr"   || arg == "--translate")     { params.translate     = true; }
+        else if (arg == "-nf"   || arg == "--no-fallback")   { params.no_fallback   = true; }
+        else if (arg == "-ps"   || arg == "--print-special") { params.print_special = true; }
+        else if (arg == "-kc"   || arg == "--keep-context")  { params.no_context    = false; }
+        else if (arg == "-l"    || arg == "--language")      { params.language      = argv[++i]; }
+        else if (arg == "-m"    || arg == "--model")         { params.model         = argv[++i]; }
+        else if (arg == "-f"    || arg == "--file")          { params.fname_out     = argv[++i]; }
+        else if (arg == "-tdrz" || arg == "--tinydiarize")   { params.tinydiarize   = true; }
+        else if (arg == "-sa"   || arg == "--save-audio")    { params.save_audio    = true; }
+        else if (arg == "-ng"   || arg == "--no-gpu")        { params.use_gpu       = false; }
+        else if (arg == "-fa"   || arg == "--flash-attn")    { params.flash_attn    = true; }
+        else if (arg == "-nfa"  || arg == "--no-flash-attn") { params.flash_attn    = false; }
+
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,       --help          [default] show this help message and exit\n");
+    fprintf(stderr, "  -t N,     --threads N     [%-7d] number of threads to use during computation\n",    params.n_threads);
+    fprintf(stderr, "            --step N        [%-7d] audio step size in milliseconds\n",                params.step_ms);
+    fprintf(stderr, "            --length N      [%-7d] audio length in milliseconds\n",                   params.length_ms);
+    fprintf(stderr, "            --keep N        [%-7d] audio to keep from previous step in ms\n",         params.keep_ms);
+    fprintf(stderr, "  -c ID,    --capture ID    [%-7d] capture device ID\n",                              params.capture_id);
+    fprintf(stderr, "  -mt N,    --max-tokens N  [%-7d] maximum number of tokens per audio chunk\n",       params.max_tokens);
+    fprintf(stderr, "  -ac N,    --audio-ctx N   [%-7d] audio context size (0 - all)\n",                   params.audio_ctx);
+    fprintf(stderr, "  -bs N,    --beam-size N   [%-7d] beam size for beam search\n",                      params.beam_size);
+    fprintf(stderr, "  -vth N,   --vad-thold N   [%-7.2f] voice activity detection threshold\n",           params.vad_thold);
+    fprintf(stderr, "  -fth N,   --freq-thold N  [%-7.2f] high-pass frequency cutoff\n",                   params.freq_thold);
+    fprintf(stderr, "  -tr,      --translate     [%-7s] translate from source language to english\n",      params.translate ? "true" : "false");
+    fprintf(stderr, "  -nf,      --no-fallback   [%-7s] do not use temperature fallback while decoding\n", params.no_fallback ? "true" : "false");
+    fprintf(stderr, "  -ps,      --print-special [%-7s] print special tokens\n",                           params.print_special ? "true" : "false");
+    fprintf(stderr, "  -kc,      --keep-context  [%-7s] keep context between audio chunks\n",              params.no_context ? "false" : "true");
+    fprintf(stderr, "  -l LANG,  --language LANG [%-7s] spoken language\n",                                params.language.c_str());
+    fprintf(stderr, "  -m FNAME, --model FNAME   [%-7s] model path\n",                                     params.model.c_str());
+    fprintf(stderr, "  -f FNAME, --file FNAME    [%-7s] text output file name\n",                          params.fname_out.c_str());
+    fprintf(stderr, "  -tdrz,    --tinydiarize   [%-7s] enable tinydiarize (requires a tdrz model)\n",     params.tinydiarize ? "true" : "false");
+    fprintf(stderr, "  -sa,      --save-audio    [%-7s] save the recorded audio to a file\n",              params.save_audio ? "true" : "false");
+    fprintf(stderr, "  -ng,      --no-gpu        [%-7s] disable GPU inference\n",                          params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -fa,      --flash-attn    [%-7s] enable flash attention during inference\n",        params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -nfa,     --no-flash-attn [%-7s] disable flash attention during inference\n",       params.flash_attn ? "false" : "true");
+    fprintf(stderr, "\n");
+}
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    whisper_params params;
+
+    if (whisper_params_parse(argc, argv, params) == false) {
+        return 1;
+    }
+
+    params.keep_ms   = std::min(params.keep_ms,   params.step_ms);
+    params.length_ms = std::max(params.length_ms, params.step_ms);
+
+    const int n_samples_step = (1e-3*params.step_ms  )*WHISPER_SAMPLE_RATE;
+    const int n_samples_len  = (1e-3*params.length_ms)*WHISPER_SAMPLE_RATE;
+    const int n_samples_keep = (1e-3*params.keep_ms  )*WHISPER_SAMPLE_RATE;
+    const int n_samples_30s  = (1e-3*30000.0         )*WHISPER_SAMPLE_RATE;
+
+    const bool use_vad = n_samples_step <= 0; // sliding window mode uses VAD
+
+    const int n_new_line = !use_vad ? std::max(1, params.length_ms / params.step_ms - 1) : 1; // number of steps to print new line
+
+    params.no_timestamps  = !use_vad;
+    params.no_context    |= use_vad;
+    params.max_tokens     = 0;
+
+    // init audio
+
+    audio_async audio(params.length_ms);
+    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {
+        fprintf(stderr, "%s: audio.init() failed!\n", __func__);
+        return 1;
+    }
+
+    audio.resume();
+
+    // whisper init
+    if (params.language != "auto" && whisper_lang_id(params.language.c_str()) == -1){
+        fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+        whisper_print_usage(argc, argv, params);
+        exit(0);
+    }
+
+    struct whisper_context_params cparams = whisper_context_default_params();
+
+    cparams.use_gpu    = params.use_gpu;
+    cparams.flash_attn = params.flash_attn;
+
+    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
+    if (ctx == nullptr) {
+        fprintf(stderr, "error: failed to initialize whisper context\n");
+        return 2;
+    }
+
+    std::vector<float> pcmf32    (n_samples_30s, 0.0f);
+    std::vector<float> pcmf32_old;
+    std::vector<float> pcmf32_new(n_samples_30s, 0.0f);
+
+    std::vector<whisper_token> prompt_tokens;
+
+    // print some info about the processing
+    {
+        fprintf(stderr, "\n");
+        if (!whisper_is_multilingual(ctx)) {
+            if (params.language != "en" || params.translate) {
+                params.language = "en";
+                params.translate = false;
+                fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
+            }
+        }
+        fprintf(stderr, "%s: processing %d samples (step = %.1f sec / len = %.1f sec / keep = %.1f sec), %d threads, lang = %s, task = %s, timestamps = %d ...\n",
+                __func__,
+                n_samples_step,
+                float(n_samples_step)/WHISPER_SAMPLE_RATE,
+                float(n_samples_len )/WHISPER_SAMPLE_RATE,
+                float(n_samples_keep)/WHISPER_SAMPLE_RATE,
+                params.n_threads,
+                params.language.c_str(),
+                params.translate ? "translate" : "transcribe",
+                params.no_timestamps ? 0 : 1);
+
+        if (!use_vad) {
+            fprintf(stderr, "%s: n_new_line = %d, no_context = %d\n", __func__, n_new_line, params.no_context);
+        } else {
+            fprintf(stderr, "%s: using VAD, will transcribe on speech activity\n", __func__);
+        }
+
+        fprintf(stderr, "\n");
+    }
+
+    int n_iter = 0;
+
+    bool is_running = true;
+
+    std::ofstream fout;
+    if (params.fname_out.length() > 0) {
+        fout.open(params.fname_out);
+        if (!fout.is_open()) {
+            fprintf(stderr, "%s: failed to open output file '%s'!\n", __func__, params.fname_out.c_str());
+            return 1;
+        }
+    }
+
+    wav_writer wavWriter;
+    // save wav file
+    if (params.save_audio) {
+        // Get current date/time for filename
+        time_t now = time(0);
+        char buffer[80];
+        strftime(buffer, sizeof(buffer), "%Y%m%d%H%M%S", localtime(&now));
+        std::string filename = std::string(buffer) + ".wav";
+
+        wavWriter.open(filename, WHISPER_SAMPLE_RATE, 16, 1);
+    }
+    printf("[Start speaking]\n");
+    fflush(stdout);
+
+    auto t_last  = std::chrono::high_resolution_clock::now();
+    const auto t_start = t_last;
+
+    // main audio loop
+    while (is_running) {
+        if (params.save_audio) {
+            wavWriter.write(pcmf32_new.data(), pcmf32_new.size());
+        }
+        // handle Ctrl + C
+        is_running = sdl_poll_events();
+
+        if (!is_running) {
+            break;
+        }
+
+        // process new audio
+
+        if (!use_vad) {
+            while (true) {
+                // handle Ctrl + C
+                is_running = sdl_poll_events();
+                if (!is_running) {
+                    break;
+                }
+                audio.get(params.step_ms, pcmf32_new);
+
+                if ((int) pcmf32_new.size() > 2*n_samples_step) {
+                    fprintf(stderr, "\n\n%s: WARNING: cannot process audio fast enough, dropping audio ...\n\n", __func__);
+                    audio.clear();
+                    continue;
+                }
+
+                if ((int) pcmf32_new.size() >= n_samples_step) {
+                    audio.clear();
+                    break;
+                }
+
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
+
+            const int n_samples_new = pcmf32_new.size();
+
+            // take up to params.length_ms audio from previous iteration
+            const int n_samples_take = std::min((int) pcmf32_old.size(), std::max(0, n_samples_keep + n_samples_len - n_samples_new));
+
+            //printf("processing: take = %d, new = %d, old = %d\n", n_samples_take, n_samples_new, (int) pcmf32_old.size());
+
+            pcmf32.resize(n_samples_new + n_samples_take);
+
+            for (int i = 0; i < n_samples_take; i++) {
+                pcmf32[i] = pcmf32_old[pcmf32_old.size() - n_samples_take + i];
+            }
+
+            memcpy(pcmf32.data() + n_samples_take, pcmf32_new.data(), n_samples_new*sizeof(float));
+
+            pcmf32_old = pcmf32;
+        } else {
+            const auto t_now  = std::chrono::high_resolution_clock::now();
+            const auto t_diff = std::chrono::duration_cast<std::chrono::milliseconds>(t_now - t_last).count();
+
+            if (t_diff < 2000) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+                continue;
+            }
+
+            audio.get(2000, pcmf32_new);
+
+            if (::vad_simple(pcmf32_new, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, false)) {
+                audio.get(params.length_ms, pcmf32);
+            } else {
+                std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+                continue;
+            }
+
+            t_last = t_now;
+        }
+
+        // run the inference
+        {
+            whisper_full_params wparams = whisper_full_default_params(params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY);
+
+            wparams.print_progress   = false;
+            wparams.print_special    = params.print_special;
+            wparams.print_realtime   = false;
+            wparams.print_timestamps = !params.no_timestamps;
+            wparams.translate        = params.translate;
+            wparams.single_segment   = !use_vad;
+            wparams.max_tokens       = params.max_tokens;
+            wparams.language         = params.language.c_str();
+            wparams.n_threads        = params.n_threads;
+            wparams.beam_search.beam_size = params.beam_size;
+
+            wparams.audio_ctx        = params.audio_ctx;
+
+            wparams.tdrz_enable      = params.tinydiarize; // [TDRZ]
+
+            // disable temperature fallback
+            //wparams.temperature_inc  = -1.0f;
+            wparams.temperature_inc  = params.no_fallback ? 0.0f : wparams.temperature_inc;
+
+            wparams.prompt_tokens    = params.no_context ? nullptr : prompt_tokens.data();
+            wparams.prompt_n_tokens  = params.no_context ? 0       : prompt_tokens.size();
+
+            if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
+                fprintf(stderr, "%s: failed to process audio\n", argv[0]);
+                return 6;
+            }
+
+            // print result;
+            {
+                if (!use_vad) {
+                    printf("\33[2K\r");
+
+                    // print long empty line to clear the previous line
+                    printf("%s", std::string(100, ' ').c_str());
+
+                    printf("\33[2K\r");
+                } else {
+                    const int64_t t1 = (t_last - t_start).count()/1000000;
+                    const int64_t t0 = std::max(0.0, t1 - pcmf32.size()*1000.0/WHISPER_SAMPLE_RATE);
+
+                    printf("\n");
+                    printf("### Transcription %d START | t0 = %d ms | t1 = %d ms\n", n_iter, (int) t0, (int) t1);
+                    printf("\n");
+                }
+
+                const int n_segments = whisper_full_n_segments(ctx);
+                for (int i = 0; i < n_segments; ++i) {
+                    const char * text = whisper_full_get_segment_text(ctx, i);
+
+                    if (params.no_timestamps) {
+                        printf("%s", text);
+                        fflush(stdout);
+
+                        if (params.fname_out.length() > 0) {
+                            fout << text;
+                        }
+                    } else {
+                        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);
+                        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);
+
+                        std::string output = "[" + to_timestamp(t0, false) + " --> " + to_timestamp(t1, false) + "]  " + text;
+
+                        if (whisper_full_get_segment_speaker_turn_next(ctx, i)) {
+                            output += " [SPEAKER_TURN]";
+                        }
+
+                        output += "\n";
+
+                        printf("%s", output.c_str());
+                        fflush(stdout);
+
+                        if (params.fname_out.length() > 0) {
+                            fout << output;
+                        }
+                    }
+                }
+
+                if (params.fname_out.length() > 0) {
+                    fout << std::endl;
+                }
+
+                if (use_vad) {
+                    printf("\n");
+                    printf("### Transcription %d END\n", n_iter);
+                }
+            }
+
+            ++n_iter;
+
+            if (!use_vad && (n_iter % n_new_line) == 0) {
+                printf("\n");
+
+                // keep part of the audio for next iteration to try to mitigate word boundary issues
+                pcmf32_old = std::vector<float>(pcmf32.end() - n_samples_keep, pcmf32.end());
+
+                // Add tokens of the last full length segment as the prompt
+                if (!params.no_context) {
+                    prompt_tokens.clear();
+
+                    const int n_segments = whisper_full_n_segments(ctx);
+                    for (int i = 0; i < n_segments; ++i) {
+                        const int token_count = whisper_full_n_tokens(ctx, i);
+                        for (int j = 0; j < token_count; ++j) {
+                            prompt_tokens.push_back(whisper_full_get_token_id(ctx, i, j));
+                        }
+                    }
+                }
+            }
+            fflush(stdout);
+        }
+    }
+
+    audio.pause();
+
+    whisper_print_timings(ctx);
+    whisper_free(ctx);
+
+    return 0;
+}

examples/sycl/CMakeLists.txt

@@ -0,0 +1,9 @@
+#  MIT license
+#  Copyright (C) 2024 Intel Corporation
+#  SPDX-License-Identifier: MIT
+
+set(TARGET ls-sycl-device)
+add_executable(${TARGET} ls-sycl-device.cpp)
+install(TARGETS ${TARGET} RUNTIME)
+target_link_libraries(${TARGET} PRIVATE common whisper ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/sycl/README.md

@@ -0,0 +1,47 @@
+# llama.cpp/example/sycl
+
+This example program provide the tools for llama.cpp for SYCL on Intel GPU.
+
+## Tool
+
+|Tool Name| Function|Status|
+|-|-|-|
+|ls-sycl-device| List all SYCL devices with ID, compute capability, max work group size, etc.|Support|
+
+### ls-sycl-device
+
+List all SYCL devices with ID, compute capability, max work group size, etc.
+
+1. Build the llama.cpp for SYCL for all targets.
+
+2. Enable oneAPI running environment
+
+```
+source /opt/intel/oneapi/setvars.sh
+```
+
+3. Execute
+
+```
+./build/bin/ls-sycl-device
+```
+
+Check the ID in startup log, like:
+
+```
+found 4 SYCL devices:
+  Device 0: Intel(R) Arc(TM) A770 Graphics,	compute capability 1.3,
+    max compute_units 512,	max work group size 1024,	max sub group size 32,	global mem size 16225243136
+  Device 1: Intel(R) FPGA Emulation Device,	compute capability 1.2,
+    max compute_units 24,	max work group size 67108864,	max sub group size 64,	global mem size 67065057280
+  Device 2: 13th Gen Intel(R) Core(TM) i7-13700K,	compute capability 3.0,
+    max compute_units 24,	max work group size 8192,	max sub group size 64,	global mem size 67065057280
+  Device 3: Intel(R) Arc(TM) A770 Graphics,	compute capability 3.0,
+    max compute_units 512,	max work group size 1024,	max sub group size 32,	global mem size 16225243136
+
+```
+
+|Attribute|Note|
+|-|-|
+|compute capability 1.3|Level-zero running time, recommended |
+|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|

examples/sycl/build.sh

@@ -0,0 +1,22 @@
+#  MIT license
+#  Copyright (C) 2024 Intel Corporation
+#  SPDX-License-Identifier: MIT
+
+mkdir -p build
+cd build
+source /opt/intel/oneapi/setvars.sh
+
+#for FP16
+#cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DWHISPER_SYCL_F16=ON # faster for long-prompt inference
+
+#for FP32
+cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
+
+#for other features from the examples, e.g. stream and talk link with SDL2:
+#cmake .. -DGGML_SYCL=ON -DWHISPER_SDL2=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
+
+#build example/main only
+#cmake --build . --config Release --target main
+
+#build all binary
+cmake --build . --config Release -v

examples/sycl/ls-sycl-device.cpp

@@ -0,0 +1,11 @@
+/*MIT license
+  Copyright (C) 2024 Intel Corporation
+  SPDX-License-Identifier: MIT
+*/
+
+#include "ggml-sycl.h"
+
+int main(int argc, char ** argv) {
+    ggml_backend_sycl_print_sycl_devices();
+    return 0;
+}

examples/sycl/run-whisper.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#  MIT license
+#  Copyright (C) 2024 Intel Corporation
+#  SPDX-License-Identifier: MIT
+
+INPUT2="Building a website can be done in 10 simple steps:\nStep 1:"
+source /opt/intel/oneapi/setvars.sh
+
+if [ $# -gt 0 ]; then
+    export GGML_SYCL_DEVICE=$1
+else
+    export GGML_SYCL_DEVICE=0
+fi
+echo GGML_SYCL_DEVICE=$GGML_SYCL_DEVICE
+#export GGML_SYCL_DEBUG=1
+./build/bin/main -m models/ggml-base.en.bin -f samples/jfk.wav

examples/talk-llama/.gitignore

@@ -0,0 +1,2 @@
+audio.mp3
+to_speak.txt

examples/talk-llama/CMakeLists.txt

@@ -0,0 +1,43 @@
+if (WHISPER_SDL2)
+    set(CMAKE_CXX_STANDARD 17)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+    if (WHISPER_USE_SYSTEM_LLAMA)
+        message(STATUS "Using system-provided llama.cpp, skipping llama.cpp fetch")
+        find_package(llama 0.3.0 REQUIRED)
+    else()
+        include(FetchContent)
+
+        set(LLAMA_TAG "v0.3.0")
+
+        set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+        set(LLAMA_BUILD_TESTS    OFF CACHE BOOL "" FORCE)
+        set(LLAMA_BUILD_COMMON   OFF CACHE BOOL "" FORCE)
+        set(LLAMA_BUILD_SERVER   OFF CACHE BOOL "" FORCE)
+        set(LLAMA_BUILD_TOOLS    OFF CACHE BOOL "" FORCE)
+
+        FetchContent_Declare(
+            llama
+            GIT_REPOSITORY https://github.com/ggml-org/llama.cpp.git
+            GIT_TAG        ${LLAMA_TAG}
+        )
+
+        message(STATUS "Fetching llama.cpp dependency (${LLAMA_TAG})...")
+        FetchContent_MakeAvailable(llama)
+    endif()
+
+    set(TARGET whisper-talk-llama)
+    add_executable(${TARGET} talk-llama.cpp)
+    target_link_libraries(${TARGET} PRIVATE llama)
+    target_include_directories(${TARGET} PRIVATE . ${SDL2_INCLUDE_DIRS})
+
+    target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${SDL2_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+    install(TARGETS ${TARGET} RUNTIME)
+
+    if(WIN32)
+        # It requires Windows 8.1 or later for PrefetchVirtualMemory
+        target_compile_definitions(${TARGET} PRIVATE -D_WIN32_WINNT=0x0602)
+    endif()
+
+    include(DefaultTargetOptions)
+endif ()

examples/talk-llama/README.md

@@ -0,0 +1,61 @@
+# whisper.cpp/examples/talk-llama
+
+Talk with an LLaMA AI in your terminal
+
+*Latest perf as of 2 Nov 2023 using Whisper Medium + LLaMA v2 13B Q8_0 on M2 Ultra:*
+
+https://github.com/ggerganov/whisper.cpp/assets/1991296/d97a3788-bf2a-4756-9a43-60c6b391649e
+
+*Previous demo running on CPUs*
+
+[Demo Talk](https://user-images.githubusercontent.com/1991296/228024237-848f998c-c334-46a6-bef8-3271590da83b.mp4)
+
+## Building
+
+The `whisper-talk-llama` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
+
+```bash
+# Install SDL2
+# On Debian based linux distributions:
+sudo apt-get install libsdl2-dev
+
+# On Fedora Linux:
+sudo dnf install SDL2 SDL2-devel
+
+# Install SDL2 on Mac OS
+brew install sdl2
+
+# Build the "whisper-talk-llama" executable
+cmake -B build -S . -DWHISPER_SDL2=ON
+cmake --build build --config Release
+
+# Run it
+./build/bin/whisper-talk-llama -mw ./models/ggml-small.en.bin -ml ../llama.cpp/models/llama-13b/ggml-model-q4_0.gguf -p "Georgi" -t 8
+```
+
+- The `-mw` argument specifies the Whisper model that you would like to use. Recommended `base` or `small` for real-time experience
+- The `-ml` argument specifies the LLaMA model that you would like to use. Read the instructions in https://github.com/ggerganov/llama.cpp for information about how to obtain a `ggml` compatible LLaMA model
+
+## Session
+
+The `whisper-talk-llama` tool supports session management to enable more coherent and continuous conversations. By maintaining context from previous interactions, it can better understand and respond to user requests in a more natural way.
+
+To enable session support, use the `--session FILE` command line option when running the program. The `whisper-talk-llama` model state will be saved to the specified file after each interaction. If the file does not exist, it will be created. If the file exists, the model state will be loaded from it, allowing you to resume a previous session.
+
+This feature is especially helpful for maintaining context in long conversations or when interacting with the AI assistant across multiple sessions. It ensures that the assistant remembers the previous interactions and can provide more relevant and contextual responses.
+
+Example usage:
+
+```bash
+./build/bin/whisper-talk-llama --session ./my-session-file -mw ./models/ggml-small.en.bin -ml ../llama.cpp/models/llama-13b/ggml-model-q4_0.gguf -p "Georgi" -t 8
+```
+
+## TTS
+
+For best experience, this example needs a TTS tool to convert the generated text responses to voice.
+You can use any TTS engine that you would like - simply edit the [speak](speak) script to your needs.
+By default, it is configured to use MacOS's `say` or Windows SpeechSynthesizer, but you can use whatever you wish.
+
+## Discussion
+
+If you have any feedback, please let "us" know in the following discussion: https://github.com/ggerganov/whisper.cpp/discussions/672?converting=1

examples/talk-llama/eleven-labs.py

@@ -0,0 +1,80 @@
+import sys
+import argparse
+import textwrap
+
+parser = argparse.ArgumentParser(add_help=False,
+    formatter_class=argparse.RawTextHelpFormatter)
+parser.add_argument("-q", "--quick", action="store_true",
+    help="skip checking the required library")
+
+modes = parser.add_argument_group("action")
+modes.add_argument("inputfile", metavar="TEXTFILE",
+    nargs='?', type=argparse.FileType(), default=sys.stdin,
+    help="read the text file (default: stdin)")
+modes.add_argument("-l", "--list", action="store_true",
+    help="show the list of voices and exit")
+modes.add_argument("-h", "--help", action="help",
+    help="show this help and exit")
+
+selopts = parser.add_argument_group("voice selection")
+selmodes = selopts.add_mutually_exclusive_group()
+selmodes.add_argument("-n", "--name",
+    default="Arnold",
+    help="get a voice object by name (default: Arnold)")
+selmodes.add_argument("-v", "--voice", type=int, metavar="NUMBER",
+    help="get a voice object by number (see --list)")
+selopts.add_argument("-f", "--filter", action="append", metavar="KEY=VAL",
+    default=["use case=narration"],
+    help=textwrap.dedent('''\
+        filter voices by labels (default: "use case=narration")
+        this option can be used multiple times
+        filtering will be disabled if the first -f has no "=" (e.g. -f "any")
+        '''))
+
+outmodes = parser.add_argument_group("output")
+outgroup = outmodes.add_mutually_exclusive_group()
+outgroup.add_argument("-s", "--save", metavar="FILE",
+    default="audio.mp3",
+    help="save the TTS to a file (default: audio.mp3)")
+outgroup.add_argument("-p", "--play", action="store_true",
+    help="play the TTS with ffplay")
+
+args = parser.parse_args()
+
+if not args.quick:
+    import importlib.util
+    if importlib.util.find_spec("elevenlabs") is None:
+        print("elevenlabs library is not installed, you can install it to your enviroment using 'pip install elevenlabs'")
+        sys.exit()
+
+from elevenlabs import voices, generate, play, save
+
+if args.filter and "=" in args.filter[0]:
+    voicelist = voices()
+    for f in args.filter:
+        label, value = f.split("=")
+        voicelist = filter(lambda x: x.labels.get(label) == value, voicelist)
+    voicelist = list(voicelist)
+else:
+    voicelist = list(voices())
+
+if args.list:
+    for i, v in enumerate(voicelist):
+        print(str(i) + ": " + v.name + " " + str(v.labels))
+    sys.exit()
+
+if args.voice:
+    voice = voicelist[args.voice % len(voicelist)]
+else:
+    voice = args.name
+    # if -n should consult -f, use the following
+    #voice = next(x for x in voicelist if x.name == args.name)
+
+audio = generate(
+    text=str(args.inputfile.read()),
+    voice=voice
+)
+if args.play:
+    play(audio)
+else:
+    save(audio, args.save) 

examples/talk-llama/prompts/talk-alpaca.txt

@@ -0,0 +1,23 @@
+Below is an instruction that describes a task. Write a response that appropriately completes the request.
+
+### Instruction:
+
+Write a text transcript of a never ending dialog, where {0} interacts with an AI assistant named {1}.
+{1} is helpful, kind, honest, friendly, good at writing and never fails to answer {0}’s requests immediately and with details and precision.
+There are no annotations like (30 seconds passed...) or (to himself), just what {0} and {1} say aloud to each other.
+The transcript only includes text, it does not include markup like HTML and Markdown.
+{1} responds with short and concise answers.
+
+### Response:
+
+{0}{4} Hello, {1}!
+{1}{4} Hello {0}! How may I help you today?
+{0}{4} What time is it?
+{1}{4} It is {2} o'clock.
+{0}{4} What year is it?
+{1}{4} We are in {3}.
+{0}{4} What is a cat?
+{1}{4} A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae.
+{0}{4} Name a color.
+{1}{4} Blue
+{0}{4}

examples/talk-llama/speak

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Usage:
+#  speak <voice_id> <textfile>
+
+function installed() { command -v $1 >/dev/null 2>&1; }
+
+if installed espeak; then
+  espeak -v en-us+m$1 -s 225 -p 50 -a 200 -g 5 -k 5 -f $2
+
+elif installed piper && installed aplay; then
+  cat $2 | piper --model ~/en_US-lessac-medium.onnx --output-raw | aplay -q -r 22050 -f S16_LE -t raw -
+
+# for Mac
+elif installed say; then
+  say -f $2
+
+# Eleven Labs
+elif installed python3 && \
+  python3 -c 'import importlib.util; exit(not importlib.util.find_spec("elevenlabs"))' && \
+  installed ffplay; then
+    # It's possible to use the API for free with limited number of characters.
+    # To increase this limit register to https://beta.elevenlabs.io to get an api key
+    # and paste it after 'ELEVEN_API_KEY='
+    # Keep the line commented to use the free version without api key
+    #export ELEVEN_API_KEY=your_api_key
+    wd=$(dirname $0)
+    script=$wd/eleven-labs.py
+    python3 $script -q -p -v $1 $2 >/dev/null 2>&1
+
+    # Uncomment to keep the audio file
+    #python3 $script -q -s ./audio.mp3 -v $1 $2 >/dev/null 2>&1
+    #ffplay -autoexit -nodisp -loglevel quiet -hide_banner -i ./audio.mp3 >/dev/null 2>&1
+
+else
+  echo 'Install espeak ("brew install espeak" or "apt-get install espeak"),'
+  echo 'piper ("pip install piper-tts" or https://github.com/rhasspy/piper) with aplay,'
+  echo 'or elevenlabs ("pip install elevenlabs") with ffplay.'
+  echo '(export ELEVEN_API_KEY if you have an api key from https://beta.elevenlabs.io)'
+fi

examples/talk-llama/speak.bat

@@ -0,0 +1 @@
+@powershell -ExecutionPolicy Bypass -F examples\talk-llama\speak.ps1 %1 %2

examples/talk-llama/speak.ps1

@@ -0,0 +1,14 @@
+# Set-ExecutionPolicy -ExecutionPolicy Bypass -Scope CurrentUser
+param(
+  [Parameter(Mandatory=$true)][int]$voicenum,
+  [Parameter(Mandatory=$true)][string]$textfile
+)
+
+Add-Type -AssemblyName System.Speech;
+$speak = New-Object System.Speech.Synthesis.SpeechSynthesizer;
+$voiceoptions = $speak.GetInstalledVoices("en-US");
+$voice = $voiceoptions[$voicenum % $voiceoptions.count];
+$speak.SelectVoice($voice.VoiceInfo.Name);
+$speak.Rate="0";
+$text = Get-Content -Path $textfile;
+$speak.Speak($text);

examples/talk-llama/talk-llama.cpp

@@ -0,0 +1,813 @@
+// Talk with AI
+//
+
+#include "common-sdl.h"
+#include "common.h"
+#include "common-whisper.h"
+#include "whisper.h"
+#include "llama.h"
+
+#include <chrono>
+#include <cstdio>
+#include <fstream>
+#include <regex>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+static std::vector<llama_token> llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) {
+    const llama_model * model = llama_get_model(ctx);
+    const llama_vocab * vocab = llama_model_get_vocab(model);
+
+    // upper limit for the number of tokens
+    int n_tokens = text.length() + add_bos;
+    std::vector<llama_token> result(n_tokens);
+    n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_bos, false);
+    if (n_tokens < 0) {
+        result.resize(-n_tokens);
+        int check = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_bos, false);
+        GGML_ASSERT(check == -n_tokens);
+    } else {
+        result.resize(n_tokens);
+    }
+    return result;
+}
+
+static std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token) {
+    const llama_model * model = llama_get_model(ctx);
+    const llama_vocab * vocab = llama_model_get_vocab(model);
+
+    std::vector<char> result(8, 0);
+    const int n_tokens = llama_token_to_piece(vocab, token, result.data(), result.size(), 0, false);
+    if (n_tokens < 0) {
+        result.resize(-n_tokens);
+        int check = llama_token_to_piece(vocab, token, result.data(), result.size(), 0, false);
+        GGML_ASSERT(check == -n_tokens);
+    } else {
+        result.resize(n_tokens);
+    }
+
+    return std::string(result.data(), result.size());
+}
+
+// command-line parameters
+struct whisper_params {
+    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t voice_ms   = 10000;
+    int32_t capture_id = -1;
+    int32_t max_tokens = 32;
+    int32_t audio_ctx  = 0;
+    int32_t n_gpu_layers = 999;
+    int32_t seed = 0;
+    int32_t top_k = 5;
+    int32_t min_keep = 1;
+    float top_p = 0.80f;
+    float min_p = 0.01f;
+    float temp  = 0.30f;
+
+    float vad_thold  = 0.6f;
+    float freq_thold = 100.0f;
+
+    bool translate      = false;
+    bool print_special  = false;
+    bool print_energy   = false;
+    bool no_timestamps  = true;
+    bool verbose_prompt = false;
+    bool use_gpu        = true;
+    bool flash_attn     = true;
+
+    std::string person      = "Georgi";
+    std::string bot_name    = "LLaMA";
+    std::string wake_cmd    = "";
+    std::string heard_ok    = "";
+    std::string language    = "en";
+    std::string model_wsp   = "models/ggml-base.en.bin";
+    std::string model_llama = "models/ggml-llama-7B.bin";
+    std::string speak       = "./examples/talk-llama/speak";
+    std::string speak_file  = "./examples/talk-llama/to_speak.txt";
+    std::string prompt      = "";
+    std::string fname_out;
+    std::string path_session = "";       // path to file for saving/loading model eval state
+};
+
+void whisper_print_usage(int argc, char ** argv, const whisper_params & params);
+
+static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-h" || arg == "--help") {
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+        else if (arg == "-t"   || arg == "--threads")        { params.n_threads      = std::stoi(argv[++i]); }
+        else if (arg == "-vms" || arg == "--voice-ms")       { params.voice_ms       = std::stoi(argv[++i]); }
+        else if (arg == "-c"   || arg == "--capture")        { params.capture_id     = std::stoi(argv[++i]); }
+        else if (arg == "-mt"  || arg == "--max-tokens")     { params.max_tokens     = std::stoi(argv[++i]); }
+        else if (arg == "-ac"  || arg == "--audio-ctx")      { params.audio_ctx      = std::stoi(argv[++i]); }
+        else if (arg == "-ngl" || arg == "--n-gpu-layers")   { params.n_gpu_layers   = std::stoi(argv[++i]); }
+        else if (arg == "--seed")                            { params.seed           = std::stoi(argv[++i]); }
+        else if (arg == "--top-k")                           { params.top_k          = std::stoi(argv[++i]); }
+        else if (arg == "--min-keep")                        { params.min_keep       = std::stoul(argv[++i]);}
+        else if (arg == "--top-p")                           { params.top_p          = std::stof(argv[++i]); }
+        else if (arg == "--min-p")                           { params.min_p          = std::stof(argv[++i]); }
+        else if (arg == "--temp")                            { params.temp           = std::stof(argv[++i]); }
+        else if (arg == "-vth" || arg == "--vad-thold")      { params.vad_thold      = std::stof(argv[++i]); }
+        else if (arg == "-fth" || arg == "--freq-thold")     { params.freq_thold     = std::stof(argv[++i]); }
+        else if (arg == "-tr"  || arg == "--translate")      { params.translate      = true; }
+        else if (arg == "-ps"  || arg == "--print-special")  { params.print_special  = true; }
+        else if (arg == "-pe"  || arg == "--print-energy")   { params.print_energy   = true; }
+        else if (arg == "-vp"  || arg == "--verbose-prompt") { params.verbose_prompt = true; }
+        else if (arg == "-ng"  || arg == "--no-gpu")         { params.use_gpu        = false; }
+        else if (arg == "-fa"  || arg == "--flash-attn")     { params.flash_attn     = true; }
+        else if (arg == "-nfa" || arg == "--no-flash-attn")  { params.flash_attn     = false; }
+        else if (arg == "-p"   || arg == "--person")         { params.person         = argv[++i]; }
+        else if (arg == "-bn"   || arg == "--bot-name")      { params.bot_name       = argv[++i]; }
+        else if (arg == "--session")                         { params.path_session   = argv[++i]; }
+        else if (arg == "-w"   || arg == "--wake-command")   { params.wake_cmd       = argv[++i]; }
+        else if (arg == "-ho"  || arg == "--heard-ok")       { params.heard_ok       = argv[++i]; }
+        else if (arg == "-l"   || arg == "--language")       { params.language       = argv[++i]; }
+        else if (arg == "-mw"  || arg == "--model-whisper")  { params.model_wsp      = argv[++i]; }
+        else if (arg == "-ml"  || arg == "--model-llama")    { params.model_llama    = argv[++i]; }
+        else if (arg == "-s"   || arg == "--speak")          { params.speak          = argv[++i]; }
+        else if (arg == "-sf"  || arg == "--speak-file")     { params.speak_file     = argv[++i]; }
+        else if (arg == "--prompt-file")                     {
+            std::ifstream file(argv[++i]);
+            std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
+            if (params.prompt.back() == '\n') {
+                params.prompt.pop_back();
+            }
+        }
+        else if (arg == "-f"   || arg == "--file")          { params.fname_out     = argv[++i]; }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,       --help           [default] show this help message and exit\n");
+    fprintf(stderr, "  -t N,     --threads N      [%-7d] number of threads to use during computation\n", params.n_threads);
+    fprintf(stderr, "  -vms N,   --voice-ms N     [%-7d] voice duration in milliseconds\n",              params.voice_ms);
+    fprintf(stderr, "  -c ID,    --capture ID     [%-7d] capture device ID\n",                           params.capture_id);
+    fprintf(stderr, "  -mt N,    --max-tokens N   [%-7d] maximum number of tokens per audio chunk\n",    params.max_tokens);
+    fprintf(stderr, "  -ac N,    --audio-ctx N    [%-7d] audio context size (0 - all)\n",                params.audio_ctx);
+    fprintf(stderr, "  -ngl N,   --n-gpu-layers N [%-7d] number of layers to store in VRAM\n",           params.n_gpu_layers);
+    fprintf(stderr, "  --seed N                   [%-7d] seed sampling\n",                               params.seed);
+    fprintf(stderr, "  --top-k N                  [%-7d] top-k sampling (0 = disabled)\n",               params.top_k);
+    fprintf(stderr, "  --min-keep N               [%-7d] minimum number of tokens to keep\n",            params.min_keep);
+    fprintf(stderr, "  --top-p N                  [%-7.2f] top-p sampling\n",                            params.top_p);
+    fprintf(stderr, "  --min-p N                  [%-7.2f] min-p sampling\n",                            params.min_p);
+    fprintf(stderr, "  --temp N                   [%-7.2f] temperature\n",                               params.temp);
+    fprintf(stderr, "  -vth N,   --vad-thold N    [%-7.2f] voice activity detection threshold\n",        params.vad_thold);
+    fprintf(stderr, "  -fth N,   --freq-thold N   [%-7.2f] high-pass frequency cutoff\n",                params.freq_thold);
+    fprintf(stderr, "  -tr,      --translate      [%-7s] translate from source language to english\n",   params.translate ? "true" : "false");
+    fprintf(stderr, "  -ps,      --print-special  [%-7s] print special tokens\n",                        params.print_special ? "true" : "false");
+    fprintf(stderr, "  -pe,      --print-energy   [%-7s] print sound energy (for debugging)\n",          params.print_energy ? "true" : "false");
+    fprintf(stderr, "  -vp,      --verbose-prompt [%-7s] print prompt at start\n",                       params.verbose_prompt ? "true" : "false");
+    fprintf(stderr, "  -ng,      --no-gpu         [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -fa,      --flash-attn     [%-7s] enable flash attention\n",                      params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -nfa,     --no-flash-attn  [%-7s] disable flash attention\n",                     params.flash_attn ? "false" : "true");
+    fprintf(stderr, "  -p NAME,  --person NAME    [%-7s] person name (for prompt selection)\n",          params.person.c_str());
+    fprintf(stderr, "  -bn NAME, --bot-name NAME  [%-7s] bot name (to display)\n",                       params.bot_name.c_str());
+    fprintf(stderr, "  -w TEXT,  --wake-command T [%-7s] wake-up command to listen for\n",               params.wake_cmd.c_str());
+    fprintf(stderr, "  -ho TEXT, --heard-ok TEXT  [%-7s] said by TTS before generating reply\n",         params.heard_ok.c_str());
+    fprintf(stderr, "  -l LANG,  --language LANG  [%-7s] spoken language\n",                             params.language.c_str());
+    fprintf(stderr, "  -mw FILE, --model-whisper  [%-7s] whisper model file\n",                          params.model_wsp.c_str());
+    fprintf(stderr, "  -ml FILE, --model-llama    [%-7s] llama model file\n",                            params.model_llama.c_str());
+    fprintf(stderr, "  -s FILE,  --speak TEXT     [%-7s] command for TTS\n",                             params.speak.c_str());
+    fprintf(stderr, "  -sf FILE, --speak-file     [%-7s] file to pass to TTS\n",                         params.speak_file.c_str());
+    fprintf(stderr, "  --prompt-file FNAME        [%-7s] file with custom prompt to start dialog\n",     "");
+    fprintf(stderr, "  --session FNAME                   file to cache model state in (may be large!) (default: none)\n");
+    fprintf(stderr, "  -f FNAME, --file FNAME     [%-7s] text output file name\n",                       params.fname_out.c_str());
+    fprintf(stderr, "\n");
+}
+
+static std::string transcribe(
+        whisper_context * ctx,
+        const whisper_params & params,
+        const std::vector<float> & pcmf32,
+        const std::string prompt_text,
+        float & prob,
+        int64_t & t_ms) {
+    const auto t_start = std::chrono::high_resolution_clock::now();
+
+    prob = 0.0f;
+    t_ms = 0;
+
+    std::vector<whisper_token> prompt_tokens;
+
+    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+
+    prompt_tokens.resize(1024);
+    prompt_tokens.resize(whisper_tokenize(ctx, prompt_text.c_str(), prompt_tokens.data(), prompt_tokens.size()));
+
+    wparams.print_progress   = false;
+    wparams.print_special    = params.print_special;
+    wparams.print_realtime   = false;
+    wparams.print_timestamps = !params.no_timestamps;
+    wparams.translate        = params.translate;
+    wparams.no_context       = true;
+    wparams.single_segment   = true;
+    wparams.max_tokens       = params.max_tokens;
+    wparams.language         = params.language.c_str();
+    wparams.n_threads        = params.n_threads;
+
+    wparams.prompt_tokens    = prompt_tokens.empty() ? nullptr : prompt_tokens.data();
+    wparams.prompt_n_tokens  = prompt_tokens.empty() ? 0       : prompt_tokens.size();
+
+    wparams.audio_ctx        = params.audio_ctx;
+
+    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
+        return "";
+    }
+
+    int prob_n = 0;
+    std::string result;
+
+    const int n_segments = whisper_full_n_segments(ctx);
+    for (int i = 0; i < n_segments; ++i) {
+        const char * text = whisper_full_get_segment_text(ctx, i);
+
+        result += text;
+
+        const int n_tokens = whisper_full_n_tokens(ctx, i);
+        for (int j = 0; j < n_tokens; ++j) {
+            const auto token = whisper_full_get_token_data(ctx, i, j);
+
+            prob += token.p;
+            ++prob_n;
+        }
+    }
+
+    if (prob_n > 0) {
+        prob /= prob_n;
+    }
+
+    const auto t_end = std::chrono::high_resolution_clock::now();
+    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();
+
+    return result;
+}
+
+static std::vector<std::string> get_words(const std::string &txt) {
+    std::vector<std::string> words;
+
+    std::istringstream iss(txt);
+    std::string word;
+    while (iss >> word) {
+        words.push_back(word);
+    }
+
+    return words;
+}
+
+const std::string k_prompt_whisper = R"(A conversation with a person called {1}.)";
+
+const std::string k_prompt_llama = R"(Text transcript of a never ending dialog, where {0} interacts with an AI assistant named {1}.
+{1} is helpful, kind, honest, friendly, good at writing and never fails to answer {0}’s requests immediately and with details and precision.
+There are no annotations like (30 seconds passed...) or (to himself), just what {0} and {1} say aloud to each other.
+The transcript only includes text, it does not include markup like HTML and Markdown.
+{1} responds with short and concise answers.
+
+{0}{4} Hello, {1}!
+{1}{4} Hello {0}! How may I help you today?
+{0}{4} What time is it?
+{1}{4} It is {2} o'clock.
+{0}{4} What year is it?
+{1}{4} We are in {3}.
+{0}{4} What is a cat?
+{1}{4} A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae.
+{0}{4} Name a color.
+{1}{4} Blue
+{0}{4})";
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    whisper_params params;
+
+    if (whisper_params_parse(argc, argv, params) == false) {
+        return 1;
+    }
+
+    if (params.language != "auto" && whisper_lang_id(params.language.c_str()) == -1) {
+        fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+        whisper_print_usage(argc, argv, params);
+        exit(0);
+    }
+
+    // whisper init
+
+    struct whisper_context_params cparams = whisper_context_default_params();
+
+    cparams.use_gpu    = params.use_gpu;
+    cparams.flash_attn = params.flash_attn;
+
+    struct whisper_context * ctx_wsp = whisper_init_from_file_with_params(params.model_wsp.c_str(), cparams);
+    if (!ctx_wsp) {
+        fprintf(stderr, "No whisper.cpp model specified. Please provide using -mw <modelfile>\n");
+        return 1;
+    }
+
+    // llama init
+
+    llama_backend_init();
+
+    auto lmparams = llama_model_default_params();
+    if (!params.use_gpu) {
+        lmparams.n_gpu_layers = 0;
+    } else {
+        lmparams.n_gpu_layers = params.n_gpu_layers;
+    }
+
+    struct llama_model * model_llama = llama_model_load_from_file(params.model_llama.c_str(), lmparams);
+    if (!model_llama) {
+        fprintf(stderr, "No llama.cpp model specified. Please provide using -ml <modelfile>\n");
+        return 1;
+    }
+
+    const llama_vocab * vocab_llama = llama_model_get_vocab(model_llama);
+
+    llama_context_params lcparams = llama_context_default_params();
+
+    // tune these to your liking
+    lcparams.n_ctx     = 2048;
+    lcparams.n_threads = params.n_threads;
+
+    lcparams.flash_attn_type = params.flash_attn ? LLAMA_FLASH_ATTN_TYPE_AUTO : LLAMA_FLASH_ATTN_TYPE_DISABLED;
+
+    struct llama_context * ctx_llama = llama_init_from_model(model_llama, lcparams);
+
+    // print some info about the processing
+    {
+        fprintf(stderr, "\n");
+
+        if (!whisper_is_multilingual(ctx_wsp)) {
+            if (params.language != "en" || params.translate) {
+                params.language = "en";
+                params.translate = false;
+                fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
+            }
+        }
+        fprintf(stderr, "%s: processing, %d threads, lang = %s, task = %s, timestamps = %d ...\n",
+                __func__,
+                params.n_threads,
+                params.language.c_str(),
+                params.translate ? "translate" : "transcribe",
+                params.no_timestamps ? 0 : 1);
+
+        fprintf(stderr, "\n");
+    }
+
+    // init audio
+
+    audio_async audio(30*1000);
+    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {
+        fprintf(stderr, "%s: audio.init() failed!\n", __func__);
+        return 1;
+    }
+
+    audio.resume();
+
+    bool is_running  = true;
+    bool force_speak = false;
+
+    float prob0 = 0.0f;
+
+    const std::string chat_symb = ":";
+
+    std::vector<float> pcmf32_cur;
+    std::vector<float> pcmf32_prompt;
+
+    const std::string prompt_whisper = ::replace(k_prompt_whisper, "{1}", params.bot_name);
+
+    // construct the initial prompt for LLaMA inference
+    std::string prompt_llama = params.prompt.empty() ? k_prompt_llama : params.prompt;
+
+    // need to have leading ' '
+    prompt_llama.insert(0, 1, ' ');
+
+    prompt_llama = ::replace(prompt_llama, "{0}", params.person);
+    prompt_llama = ::replace(prompt_llama, "{1}", params.bot_name);
+
+    {
+        // get time string
+        std::string time_str;
+        {
+            time_t t = time(0);
+            struct tm * now = localtime(&t);
+            char buf[128];
+            strftime(buf, sizeof(buf), "%H:%M", now);
+            time_str = buf;
+        }
+        prompt_llama = ::replace(prompt_llama, "{2}", time_str);
+    }
+
+    {
+        // get year string
+        std::string year_str;
+        {
+            time_t t = time(0);
+            struct tm * now = localtime(&t);
+            char buf[128];
+            strftime(buf, sizeof(buf), "%Y", now);
+            year_str = buf;
+        }
+        prompt_llama = ::replace(prompt_llama, "{3}", year_str);
+    }
+
+    prompt_llama = ::replace(prompt_llama, "{4}", chat_symb);
+
+    llama_batch batch = llama_batch_init(llama_n_ctx(ctx_llama), 0, 1);
+
+    // init sampler
+    auto sparams = llama_sampler_chain_default_params();
+
+    llama_sampler * smpl = llama_sampler_chain_init(sparams);
+
+    if (params.temp > 0.0f) {
+        llama_sampler_chain_add(smpl, llama_sampler_init_top_k(params.top_k));
+        llama_sampler_chain_add(smpl, llama_sampler_init_top_p(params.top_p, params.min_keep));
+        llama_sampler_chain_add(smpl, llama_sampler_init_temp (params.temp));
+        llama_sampler_chain_add(smpl, llama_sampler_init_dist (params.seed));
+        llama_sampler_chain_add(smpl, llama_sampler_init_min_p (params.min_p, params.min_keep));
+    } else {
+        llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
+    }
+
+    // init session
+    std::string path_session = params.path_session;
+    std::vector<llama_token> session_tokens;
+    auto embd_inp = ::llama_tokenize(ctx_llama, prompt_llama, true);
+
+    if (!path_session.empty()) {
+        fprintf(stderr, "%s: attempting to load saved session from %s\n", __func__, path_session.c_str());
+
+        // fopen to check for existing session
+        FILE * fp = std::fopen(path_session.c_str(), "rb");
+        if (fp != NULL) {
+            std::fclose(fp);
+
+            session_tokens.resize(llama_n_ctx(ctx_llama));
+            size_t n_token_count_out = 0;
+            if (!llama_state_load_file(ctx_llama, path_session.c_str(), session_tokens.data(), session_tokens.capacity(), &n_token_count_out)) {
+                fprintf(stderr, "%s: error: failed to load session file '%s'\n", __func__, path_session.c_str());
+                return 1;
+            }
+            session_tokens.resize(n_token_count_out);
+            for (size_t i = 0; i < session_tokens.size(); i++) {
+                embd_inp[i] = session_tokens[i];
+            }
+
+            fprintf(stderr, "%s: loaded a session with prompt size of %d tokens\n", __func__, (int) session_tokens.size());
+        } else {
+            fprintf(stderr, "%s: session file does not exist, will create\n", __func__);
+        }
+    }
+
+    // evaluate the initial prompt
+
+    printf("\n");
+    printf("%s : initializing - please wait ...\n", __func__);
+
+    // prepare batch
+    {
+        batch.n_tokens = embd_inp.size();
+
+        for (int i = 0; i < batch.n_tokens; i++) {
+            batch.token[i]     = embd_inp[i];
+            batch.pos[i]       = i;
+            batch.n_seq_id[i]  = 1;
+            batch.seq_id[i][0] = 0;
+            batch.logits[i]    = i == batch.n_tokens - 1;
+        }
+    }
+
+    if (llama_decode(ctx_llama, batch)) {
+        fprintf(stderr, "%s : failed to decode\n", __func__);
+        return 1;
+    }
+
+    if (params.verbose_prompt) {
+        fprintf(stdout, "\n");
+        fprintf(stdout, "%s", prompt_llama.c_str());
+        fflush(stdout);
+    }
+
+     // debug message about similarity of saved session, if applicable
+    size_t n_matching_session_tokens = 0;
+    if (session_tokens.size()) {
+        for (llama_token id : session_tokens) {
+            if (n_matching_session_tokens >= embd_inp.size() || id != embd_inp[n_matching_session_tokens]) {
+                break;
+            }
+            n_matching_session_tokens++;
+        }
+        if (n_matching_session_tokens >= embd_inp.size()) {
+            fprintf(stderr, "%s: session file has exact match for prompt!\n", __func__);
+        } else if (n_matching_session_tokens < (embd_inp.size() / 2)) {
+            fprintf(stderr, "%s: warning: session file has low similarity to prompt (%zu / %zu tokens); will mostly be reevaluated\n",
+                __func__, n_matching_session_tokens, embd_inp.size());
+        } else {
+            fprintf(stderr, "%s: session file matches %zu / %zu tokens of prompt\n",
+                __func__, n_matching_session_tokens, embd_inp.size());
+        }
+    }
+
+    // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session
+    // if we loaded a session with at least 75% similarity. It's currently just used to speed up the
+    // initial prompt so it doesn't need to be an exact match.
+    bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < (embd_inp.size() * 3 / 4);
+
+    printf("%s : done! start speaking in the microphone\n", __func__);
+
+    // show wake command if enabled
+    const std::string wake_cmd = params.wake_cmd;
+    const int wake_cmd_length = get_words(wake_cmd).size();
+    const bool use_wake_cmd = wake_cmd_length > 0;
+
+    if (use_wake_cmd) {
+        printf("%s : the wake-up command is: '%s%s%s'\n", __func__, "\033[1m", wake_cmd.c_str(), "\033[0m");
+    }
+
+    printf("\n");
+    printf("%s%s", params.person.c_str(), chat_symb.c_str());
+    fflush(stdout);
+
+    // clear audio buffer
+    audio.clear();
+
+    // text inference variables
+    const int voice_id = 2;
+    const int n_keep   = embd_inp.size();
+    const int n_ctx    = llama_n_ctx(ctx_llama);
+
+    int n_past = n_keep;
+    int n_prev = 64; // TODO arg
+    int n_session_consumed = !path_session.empty() && session_tokens.size() > 0 ? session_tokens.size() : 0;
+
+    std::vector<llama_token> embd;
+
+    // reverse prompts for detecting when it's time to stop speaking
+    std::vector<std::string> antiprompts = {
+        params.person + chat_symb,
+    };
+
+    // main loop
+    while (is_running) {
+        // handle Ctrl + C
+        is_running = sdl_poll_events();
+
+        if (!is_running) {
+            break;
+        }
+
+        // delay
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+        int64_t t_ms = 0;
+
+        {
+            audio.get(2000, pcmf32_cur);
+
+            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1250, params.vad_thold, params.freq_thold, params.print_energy) || force_speak) {
+                //fprintf(stdout, "%s: Speech detected! Processing ...\n", __func__);
+
+                audio.get(params.voice_ms, pcmf32_cur);
+
+                std::string all_heard;
+
+                if (!force_speak) {
+                    all_heard = ::trim(::transcribe(ctx_wsp, params, pcmf32_cur, prompt_whisper, prob0, t_ms));
+                }
+
+                const auto words = get_words(all_heard);
+
+                std::string wake_cmd_heard;
+                std::string text_heard;
+
+                for (int i = 0; i < (int) words.size(); ++i) {
+                    if (i < wake_cmd_length) {
+                        wake_cmd_heard += words[i] + " ";
+                    } else {
+                        text_heard += words[i] + " ";
+                    }
+                }
+
+                // check if audio starts with the wake-up command if enabled
+                if (use_wake_cmd) {
+                    const float sim = similarity(wake_cmd_heard, wake_cmd);
+
+                    if ((sim < 0.7f) || (text_heard.empty())) {
+                        audio.clear();
+                        continue;
+                    }
+                }
+
+                // optionally give audio feedback that the current text is being processed
+                if (!params.heard_ok.empty()) {
+                    speak_with_file(params.speak, params.heard_ok, params.speak_file, voice_id);
+                }
+
+                // remove text between brackets using regex
+                {
+                    std::regex re("\\[.*?\\]");
+                    text_heard = std::regex_replace(text_heard, re, "");
+                }
+
+                // remove text between brackets using regex
+                {
+                    std::regex re("\\(.*?\\)");
+                    text_heard = std::regex_replace(text_heard, re, "");
+                }
+
+                // remove all characters, except for letters, numbers, punctuation and ':', '\'', '-', ' '
+                text_heard = std::regex_replace(text_heard, std::regex("[^a-zA-Z0-9åäöÅÄÖ\\.,\\?!\\s\\:\\'\\-]"), "");
+
+                // take first line
+                text_heard = text_heard.substr(0, text_heard.find_first_of('\n'));
+
+                // remove leading and trailing whitespace
+                text_heard = std::regex_replace(text_heard, std::regex("^\\s+"), "");
+                text_heard = std::regex_replace(text_heard, std::regex("\\s+$"), "");
+
+                const std::vector<llama_token> tokens = llama_tokenize(ctx_llama, text_heard.c_str(), false);
+
+                if (text_heard.empty() || tokens.empty() || force_speak) {
+                    //fprintf(stdout, "%s: Heard nothing, skipping ...\n", __func__);
+                    audio.clear();
+
+                    continue;
+                }
+
+                force_speak = false;
+
+                text_heard.insert(0, 1, ' ');
+                text_heard += "\n" + params.bot_name + chat_symb;
+                fprintf(stdout, "%s%s%s", "\033[1m", text_heard.c_str(), "\033[0m");
+                fflush(stdout);
+
+                embd = ::llama_tokenize(ctx_llama, text_heard, false);
+
+                // Append the new input tokens to the session_tokens vector
+                if (!path_session.empty()) {
+                    session_tokens.insert(session_tokens.end(), tokens.begin(), tokens.end());
+                }
+
+                // text inference
+                bool done = false;
+                std::string text_to_speak;
+                while (true) {
+                    // predict
+                    if (embd.size() > 0) {
+                        if (n_past + (int) embd.size() > n_ctx) {
+                            n_past = n_keep;
+
+                            // insert n_left/2 tokens at the start of embd from last_n_tokens
+                            embd.insert(embd.begin(), embd_inp.begin() + embd_inp.size() - n_prev, embd_inp.end());
+                            // stop saving session if we run out of context
+                            path_session = "";
+                            //printf("\n---\n");
+                            //printf("resetting: '");
+                            //for (int i = 0; i < (int) embd.size(); i++) {
+                            //    printf("%s", llama_token_to_piece(ctx_llama, embd[i]));
+                            //}
+                            //printf("'\n");
+                            //printf("\n---\n");
+                        }
+
+                        // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)
+                        // REVIEW
+                        if (n_session_consumed < (int) session_tokens.size()) {
+                            size_t i = 0;
+                            for ( ; i < embd.size(); i++) {
+                                if (embd[i] != session_tokens[n_session_consumed]) {
+                                    session_tokens.resize(n_session_consumed);
+                                    break;
+                                }
+
+                                n_past++;
+                                n_session_consumed++;
+
+                                if (n_session_consumed >= (int) session_tokens.size()) {
+                                    i++;
+                                    break;
+                                }
+                            }
+                            if (i > 0) {
+                                embd.erase(embd.begin(), embd.begin() + i);
+                            }
+                        }
+
+                        if (embd.size() > 0 && !path_session.empty()) {
+                            session_tokens.insert(session_tokens.end(), embd.begin(), embd.end());
+                            n_session_consumed = session_tokens.size();
+                        }
+
+                        // prepare batch
+                        {
+                            batch.n_tokens = embd.size();
+
+                            for (int i = 0; i < batch.n_tokens; i++) {
+                                batch.token[i]     = embd[i];
+                                batch.pos[i]       = n_past + i;
+                                batch.n_seq_id[i]  = 1;
+                                batch.seq_id[i][0] = 0;
+                                batch.logits[i]    = i == batch.n_tokens - 1;
+                            }
+                        }
+
+                        if (llama_decode(ctx_llama, batch)) {
+                            fprintf(stderr, "%s : failed to decode\n", __func__);
+                            return 1;
+                        }
+                    }
+
+
+                    embd_inp.insert(embd_inp.end(), embd.begin(), embd.end());
+                    n_past += embd.size();
+
+                    embd.clear();
+
+                    if (done) break;
+
+                    {
+                        // out of user input, sample next token
+
+                        if (!path_session.empty() && need_to_save_session) {
+                            need_to_save_session = false;
+                            llama_state_save_file(ctx_llama, path_session.c_str(), session_tokens.data(), session_tokens.size());
+                        }
+
+                        const llama_token id = llama_sampler_sample(smpl, ctx_llama, -1);
+
+                        if (id != llama_vocab_eos(vocab_llama)) {
+                            // add it to the context
+                            embd.push_back(id);
+
+                            text_to_speak += llama_token_to_piece(ctx_llama, id);
+
+                            printf("%s", llama_token_to_piece(ctx_llama, id).c_str());
+                            fflush(stdout);
+                        }
+                    }
+
+                    {
+                        std::string last_output;
+                        for (int i = embd_inp.size() - 16; i < (int) embd_inp.size(); i++) {
+                            last_output += llama_token_to_piece(ctx_llama, embd_inp[i]);
+                        }
+                        last_output += llama_token_to_piece(ctx_llama, embd[0]);
+
+                        for (std::string & antiprompt : antiprompts) {
+                            if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {
+                                done = true;
+                                text_to_speak = ::replace(text_to_speak, antiprompt, "");
+                                fflush(stdout);
+                                need_to_save_session = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    is_running = sdl_poll_events();
+
+                    if (!is_running) {
+                        break;
+                    }
+                }
+
+                speak_with_file(params.speak, text_to_speak, params.speak_file, voice_id);
+
+                audio.clear();
+            }
+        }
+    }
+
+    audio.pause();
+
+    whisper_print_timings(ctx_wsp);
+    whisper_free(ctx_wsp);
+
+    llama_perf_sampler_print(smpl);
+    llama_perf_context_print(ctx_llama);
+
+    llama_sampler_free(smpl);
+    llama_batch_free(batch);
+    llama_free(ctx_llama);
+
+    llama_backend_free();
+
+    return 0;
+}

examples/test-cmake/.gitignore

@@ -0,0 +1,3 @@
+whisper-build-install
+install
+build

examples/test-cmake/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.14)
+project(whisper-simple)
+
+set(CMAKE_CXX_STANDARD 17)
+
+# request a version that is satisfied by both dev (1.9.2-dev) and release (1.9.2) installs
+# (find_package does not accept a pre-release version as an argument)
+find_package(whisper 1.9.2 REQUIRED)
+
+add_executable(test-cmake test-cmake.cpp)
+
+target_link_libraries(test-cmake PRIVATE whisper)
+
+target_compile_definitions(test-cmake PRIVATE
+    WHISPER_BUILD_NUMBER=${WHISPER_BUILD_NUMBER}
+    WHISPER_BUILD_COMMIT="${WHISPER_BUILD_COMMIT}"
+)

examples/test-cmake/README.md

@@ -0,0 +1,40 @@
+## cmake-test
+
+This is just for manually testing/developing of a whisper.cpp installation to
+enable troubleshooting issues and exploration. The idea is that this can be used
+after making changes to whisper.cpp installation cmake configuration and then
+verify it locally.
+
+### Usage
+The following will configure, build, and install whisper.cpp
+
+Configuring/build/install:
+```console
+./build-install.sh
+```
+The above command will create a directory named `install` in the current directory
+which will have the following files in its lib directory:
+```console
+$ ls install/lib/
+cmake                   libggml-base.so         libggml.so                libparakeet.so          libwhisper.so
+libggml-base.so.0       libggml-base.so.0.20.0  libggml.so.0              libparakeet.so.1        libwhisper.so.1
+libggml-cpu.so          libggml.so.0.20.0       libggml.so.0.20.0         libparakeet.so.1.9.2    libwhisper.so.1.9.2
+libggml-cpu.so.0        pkgconfig               libparakeet.so            libwhisper.so
+```
+
+Build/run this project using the installation created above:
+```console
+$ ./build.sh
+-- The C compiler identification is GNU 13.3.0
+-- The CXX compiler identification is GNU 13.3.0
+-- Detecting C compiler ABI info - done
+-- Detecting CXX compiler ABI info - done
+-- Detecting CXX compile features - done
+-- Configuring done (0.1s)
+-- Generating done (0.0s)
+-- Build files have been written to: /path/to/whisper.cpp/examples/test-cmake/build
+[ 50%] Building CXX object CMakeFiles/test-cmake.dir/test-cmake.cpp.o
+[100%] Linking CXX executable test-cmake
+[100%] Built target test-cmake
+[test-cmake] version: 1.9.2-dev, build: 4925 (1fe009ca)
+```

examples/test-cmake/build-install.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+build_dir=whisper-build-install
+install_dir=install
+
+rm -rf ${build_dir} ${install_dir}
+
+cmake -S ../../. -B ${build_dir} -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=ON \
+  -DWHISPER_BUILD_TESTS=OFF \
+  -DWHISPER_BUILD_EXAMPLES=OFF \
+  -DWHISPER_BUILD_SERVER=OFF \
+  -DCMAKE_INSTALL_PREFIX="${PWD}/${install_dir}"
+
+cmake --build ${build_dir} --parallel 12
+cmake --install ${build_dir}

examples/test-cmake/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
+cmake --build build
+
+LD_LIBRARY_PATH="${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake

examples/test-cmake/test-cmake.cpp

@@ -0,0 +1,9 @@
+#include "whisper.h"
+
+#include <cstdio>
+
+int main(void) {
+    printf("[test-cmake] version: %s, build: %d (%s)\n",
+           whisper_version(), WHISPER_BUILD_NUMBER, WHISPER_BUILD_COMMIT);
+    return 0;
+}

examples/twitch.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Transcribe twitch.tv livestream by feeding audio input to whisper.cpp at regular intervals
+# Thanks to @keyehzy
+# ref: https://github.com/ggml-org/whisper.cpp/issues/209
+#
+# The script currently depends on the third-party tool "streamlink"
+# On Mac OS, you can install it via "brew install streamlink"
+#
+
+set -eo pipefail
+
+step=10
+model=base.en
+threads=4
+
+help()
+{
+    echo "Example program for captioning a livestream from twitch.tv."
+    echo
+    echo "Usage: ./twitch.sh -s [step] -m [model] -t [threads] [url]"
+    echo "options:"
+    echo "-s       Step in seconds (default is $step)."
+    echo "-m       Choose model, options are: 'tiny.en' 'tiny' 'base.en' 'base' 'small.en' 'small' 'medium.en' 'medium' 'large-v1' 'large-v2' 'large-v3' 'large-v3-turbo' (default is '$model')."
+    echo "-t       Number of threads to use."
+    echo "-h       Print this help page."
+    echo
+}
+
+check_requirements()
+{
+    if ! command -v ./build/bin/whisper-cli &>/dev/null; then
+        echo "whisper.cpp main executable is required (make)"
+        exit 1
+    fi
+
+    if ! command -v streamlink &>/dev/null; then
+        echo "streamlink is required (https://streamlink.github.io)"
+        exit 1
+    fi
+
+    if ! command -v ffmpeg &>/dev/null; then
+        echo "ffmpeg is required (https://ffmpeg.org)"
+        exit 1
+    fi
+}
+
+check_requirements
+
+while getopts ":s:m:t:h" option; do
+    case $option in
+	s)
+            step=$OPTARG;;
+	m)
+            model=$OPTARG;;
+	t)
+	    threads=$OPTARG;;
+	h)
+            help
+            exit;;
+	\?)
+	    help
+	    exit;;
+    esac
+done
+
+url=${@:$OPTIND:1}
+
+if [ -z $url ]; then
+    help
+    exit
+fi
+
+echo "Piping from streamlink url=$url model=$model step=$step threads=$threads"
+streamlink $url best -O 2>/dev/null | ffmpeg -loglevel quiet -i - -y -probesize 32 -y -ar 16000 -ac 1 -acodec pcm_s16le /tmp/whisper-live0.wav &
+
+if [ $? -ne 0 ]; then
+    printf "error: ffmpeg failed\n"
+    exit 1
+fi
+
+echo "Buffering stream... (this should take $step seconds)"
+sleep $(($step))
+
+set +e
+
+echo "Starting..."
+
+i=0
+SECONDS=0
+while true
+do
+    err=1
+    while [ $err -ne 0 ]; do
+        if [ $i -gt 0 ]; then
+            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.wav -y -ss $(($i*$step-1)).5 -t $step -c copy /tmp/whisper-live.wav 2> /tmp/whisper-live.err
+        else
+            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.wav -y -ss $(($i*$step)) -t $step -c copy /tmp/whisper-live.wav 2> /tmp/whisper-live.err
+        fi
+        err=$(cat /tmp/whisper-live.err | wc -l)
+    done
+
+    ./build/bin/whisper-cli -t $threads -m ./models/ggml-$model.bin -f /tmp/whisper-live.wav --no-timestamps -otxt 2> /tmp/whispererr | tail -n 1
+
+    while [ $SECONDS -lt $((($i+1)*$step)) ]; do
+        sleep 1
+    done
+    ((i=i+1))
+done

examples/vad-speech-segments/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(TARGET whisper-vad-speech-segments)
+add_executable(${TARGET} speech.cpp)
+
+include(DefaultTargetOptions)
+
+target_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+
+install(TARGETS ${TARGET} RUNTIME)

examples/vad-speech-segments/README.md

@@ -0,0 +1,52 @@
+# whisper.cpp/examples/vad-speech-segments
+
+This examples demonstrates how to use a VAD (Voice Activity Detection) model to
+segment an audio file into speech segments.
+
+### Building the example
+The example can be built using the following command:
+```console
+cmake -S . -B build
+cmake --build build -j8 --target vad-speech-segments
+```
+
+### Running the example
+The examples can be run using the following command, which uses a model
+that we use internally for testing:
+```console
+./build/bin/vad-speech-segments \
+    --vad-model models/for-tests-silero-v6.2.0-ggml.bin \
+    --file samples/jfk.wav \
+    --no-prints
+
+Detected 5 speech segments:
+Speech segment 0: start = 0.29, end = 2.21
+Speech segment 1: start = 3.30, end = 3.77
+Speech segment 2: start = 4.00, end = 4.35
+Speech segment 3: start = 5.38, end = 7.65
+Speech segment 4: start = 8.16, end = 10.59
+```
+To see more output from whisper.cpp remove the `--no-prints` argument.
+
+
+### Command line options
+```console
+./build/bin/vad-speech-segments --help
+
+usage: ./build/bin/vad-speech-segments [options] file
+supported audio formats: flac, mp3, ogg, wav
+
+options:
+  -h,        --help                          [default] show this help message and exit
+  -f FNAME,  --file FNAME                    [       ] input audio file path
+  -t N,      --threads N                     [4      ] number of threads to use during computation
+  -ug,       --use-gpu                       [true   ] use GPU
+  -vm FNAME, --vad-model FNAME               [       ] VAD model path
+  -vt N,     --vad-threshold N               [0.50   ] VAD threshold for speech recognition
+  -vspd N,   --vad-min-speech-duration-ms  N [250    ] VAD min speech duration (0.0-1.0)
+  -vsd N,    --vad-min-silence-duration-ms N [100    ] VAD min silence duration (to split segments)
+  -vmsd N,   --vad-max-speech-duration-s   N [FLT_MAX] VAD max speech duration (auto-split longer)
+  -vp N,     --vad-speech-pad-ms           N [30     ] VAD speech padding (extend segments)
+  -vo N,     --vad-samples-overlap         N [0.10   ] VAD samples overlap (seconds between segments)
+  -np,       --no-prints                     [false  ] do not print anything other than the results
+```

examples/vad-speech-segments/speech.cpp

@@ -0,0 +1,150 @@
+#include "common.h"
+#include "common-whisper.h"
+
+#include "whisper.h"
+
+#include <cstdio>
+#include <cfloat>
+#include <string>
+
+// command-line parameters
+struct cli_params {
+    int32_t     n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    std::string vad_model = "";
+    float       vad_threshold = 0.5f;
+    int         vad_min_speech_duration_ms = 250;
+    int         vad_min_silence_duration_ms = 100;
+    float       vad_max_speech_duration_s = FLT_MAX;
+    int         vad_speech_pad_ms = 30;
+    float       vad_samples_overlap = 0.1f;
+    bool        use_gpu = false;
+    std::string fname_inp = {};
+    bool        no_prints       = false;
+};
+
+static void vad_print_usage(int /*argc*/, char ** argv, const cli_params & params) {
+    fprintf(stderr, "\n");
+    fprintf(stderr, "usage: %s [options] file\n", argv[0]);
+    fprintf(stderr, "supported audio formats: flac, mp3, ogg, wav\n");
+    fprintf(stderr, "Note: Output timestamps are in centiseconds (1/100th of a second).\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "options:\n");
+    fprintf(stderr, "  -h,        --help                          [default] show this help message and exit\n");
+    fprintf(stderr, "  -f FNAME,  --file FNAME                    [%-7s] input audio file path\n",                            "");
+    fprintf(stderr, "  -t N,      --threads N                     [%-7d] number of threads to use during computation\n",      params.n_threads);
+    fprintf(stderr, "  -ug,       --use-gpu                       [%-7s] use GPU\n",                                          params.use_gpu ? "true" : "false");
+    fprintf(stderr, "  -vm FNAME, --vad-model FNAME               [%-7s] VAD model path\n",                                   params.vad_model.c_str());
+    fprintf(stderr, "  -vt N,     --vad-threshold N               [%-7.2f] VAD threshold for speech recognition\n",           params.vad_threshold);
+    fprintf(stderr, "  -vspd N,   --vad-min-speech-duration-ms  N [%-7d] VAD min speech duration (0.0-1.0)\n",                params.vad_min_speech_duration_ms);
+    fprintf(stderr, "  -vsd N,    --vad-min-silence-duration-ms N [%-7d] VAD min silence duration (to split segments)\n",     params.vad_min_silence_duration_ms);
+    fprintf(stderr, "  -vmsd N,   --vad-max-speech-duration-s   N [%-7s] VAD max speech duration (auto-split longer)\n",      params.vad_max_speech_duration_s == FLT_MAX ?
+                                                                                                                                  std::string("FLT_MAX").c_str() :
+                                                                                                                                  std::to_string(params.vad_max_speech_duration_s).c_str());
+    fprintf(stderr, "  -vp N,     --vad-speech-pad-ms           N [%-7d] VAD speech padding (extend segments)\n",             params.vad_speech_pad_ms);
+    fprintf(stderr, "  -vo N,     --vad-samples-overlap         N [%-7.2f] VAD samples overlap (seconds between segments)\n", params.vad_samples_overlap);
+    fprintf(stderr, "  -np,       --no-prints                     [%-7s] do not print anything other than the results\n",     params.no_prints ? "true" : "false");
+    fprintf(stderr, "\n");
+}
+
+static char * requires_value_error(const std::string & arg) {
+    fprintf(stderr, "error: argument %s requires value\n", arg.c_str());
+    exit(0);
+}
+
+static bool vad_params_parse(int argc, char ** argv, cli_params & params) {
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg == "-h" || arg == "--help") {
+            vad_print_usage(argc, argv, params);
+            exit(0);
+        }
+        #define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))
+        else if (arg == "-f"    || arg == "--file")                        { params.fname_inp = ARGV_NEXT; }
+        else if (arg == "-t"    || arg == "--threads")                     { params.n_threads                   = std::stoi(ARGV_NEXT); }
+        else if (arg == "-ug"   || arg == "--use-gpu")                     { params.use_gpu                     = true; }
+        else if (arg == "-vm"   || arg == "--vad-model")                   { params.vad_model                   = ARGV_NEXT; }
+        else if (arg == "-vt"   || arg == "--vad-threshold")               { params.vad_threshold               = std::stof(ARGV_NEXT); }
+        else if (arg == "-vspd" || arg == "--vad-min-speech-duration-ms")  { params.vad_min_speech_duration_ms  = std::stoi(ARGV_NEXT); }
+        else if (arg == "-vsd"  || arg == "--vad-min-silence-duration-ms") { params.vad_min_silence_duration_ms = std::stoi(ARGV_NEXT); }
+        else if (arg == "-vmsd" || arg == "--vad-max-speech-duration-s")   { params.vad_max_speech_duration_s   = std::stof(ARGV_NEXT); }
+        else if (arg == "-vp"   || arg == "--vad-speech-pad-ms")           { params.vad_speech_pad_ms           = std::stoi(ARGV_NEXT); }
+        else if (arg == "-vo"   || arg == "--vad-samples-overlap")         { params.vad_samples_overlap         = std::stof(ARGV_NEXT); }
+        else if (arg == "-np"   || arg == "--no-prints")                   { params.no_prints       = true; }
+        else {
+            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
+            vad_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
+    return true;
+}
+
+static void cb_log_disable(enum ggml_log_level , const char * , void * ) { }
+
+int main(int argc, char ** argv) {
+    ggml_backend_load_all();
+
+    cli_params cli_params;
+
+    if (!vad_params_parse(argc, argv, cli_params)) {
+        vad_print_usage(argc, argv, cli_params);
+        return 1;
+    }
+
+    if (cli_params.no_prints) {
+        whisper_log_set(cb_log_disable, NULL);
+    }
+
+    // Load the input sample audio file.
+    std::vector<float> pcmf32;
+    std::vector<std::vector<float>> pcmf32s;
+    if (!read_audio_data(cli_params.fname_inp.c_str(), pcmf32, pcmf32s, false)) {
+        fprintf(stderr, "error: failed to read audio data from %s\n", cli_params.fname_inp.c_str());
+        return 2;
+    }
+
+    // Initialize the context which loads the VAD model.
+    struct whisper_vad_context_params ctx_params = whisper_vad_default_context_params();
+    ctx_params.n_threads  = cli_params.n_threads;
+    ctx_params.use_gpu    = cli_params.use_gpu;
+    struct whisper_vad_context * vctx = whisper_vad_init_from_file_with_params(
+            cli_params.vad_model.c_str(),
+            ctx_params);
+    if (vctx == nullptr) {
+        fprintf(stderr, "error: failed to initialize whisper context\n");
+        return 2;
+    }
+
+    // Detect speech in the input audio file.
+    if (!whisper_vad_detect_speech(vctx, pcmf32.data(), pcmf32.size())) {
+        fprintf(stderr, "error: failed to detect speech\n");
+        return 3;
+    }
+
+    // Get the the vad segments using the probabilities that have been computed
+    // previously and stored in the whisper_vad_context.
+    struct whisper_vad_params params = whisper_vad_default_params();
+    params.threshold = cli_params.vad_threshold;
+    params.min_speech_duration_ms = cli_params.vad_min_speech_duration_ms;
+    params.min_silence_duration_ms = cli_params.vad_min_silence_duration_ms;
+    params.max_speech_duration_s = cli_params.vad_max_speech_duration_s;
+    params.speech_pad_ms = cli_params.vad_speech_pad_ms;
+    params.samples_overlap = cli_params.vad_samples_overlap;
+    struct whisper_vad_segments * segments = whisper_vad_segments_from_probs(vctx, params);
+
+    printf("\n");
+    printf("Detected %d speech segments:\n", whisper_vad_segments_n_segments(segments));
+    for (int i = 0; i < whisper_vad_segments_n_segments(segments); ++i) {
+        printf("Speech segment %d: start = %.2f, end = %.2f\n", i,
+               whisper_vad_segments_get_segment_t0(segments, i),
+               whisper_vad_segments_get_segment_t1(segments, i));
+    }
+    printf("\n");
+
+    whisper_vad_free_segments(segments);
+    whisper_vad_free(vctx);
+
+    return 0;
+}