Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions tools/kokoro/include/kokoro.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ enum kokoro_status {
KOKORO_E_OOM = 5,
KOKORO_E_NOT_IMPLEMENTED = 6,
KOKORO_E_RUNTIME = 7,
KOKORO_E_CANCELLED = 8,
};

const char * kokoro_status_str(kokoro_status st) noexcept;
Expand Down Expand Up @@ -137,6 +138,8 @@ enum kokoro_g2p_kind {
};
kokoro_g2p_kind kokoro_g2p_kind_of_build() noexcept;

using kokoro_cancel_callback = bool (*)(void * user_data);

// Synthesize a single utterance. `text` is the natural-language input,
// `voice` is the loaded ref_s preset. Output PCM lands in `out`. The
// `speed_mult` parameter scales the predicted durations (1.0 = native rate).
Expand All @@ -148,7 +151,9 @@ kokoro_status kokoro_synthesize(
const std::string & text,
float speed_mult,
kokoro_audio & out,
std::string & err_out) noexcept;
std::string & err_out,
kokoro_cancel_callback cancel = nullptr,
void * cancel_user_data = nullptr) noexcept;

// Synthesize from a precomputed espeak-ng IPA string (UTF-8) instead of raw
// text. The IPA is mapped straight to Kokoro vocab ids via `ipa_to_token_ids`
Expand All @@ -162,7 +167,9 @@ kokoro_status kokoro_synthesize_ipa(
const std::string & ipa,
float speed_mult,
kokoro_audio & out,
std::string & err_out) noexcept;
std::string & err_out,
kokoro_cancel_callback cancel = nullptr,
void * cancel_user_data = nullptr) noexcept;

// Returns the model's audio sample rate (24000 for v1.0).
int kokoro_sample_rate(const kokoro_model * model) noexcept;
Expand Down
30 changes: 25 additions & 5 deletions tools/kokoro/src/kokoro.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ const char * kokoro_status_str(kokoro_status st) noexcept {
case KOKORO_E_OOM: return "out of memory";
case KOKORO_E_NOT_IMPLEMENTED: return "feature not implemented";
case KOKORO_E_RUNTIME: return "runtime error";
case KOKORO_E_CANCELLED: return "cancelled";
}
return "unknown";
}
Expand Down Expand Up @@ -517,8 +518,17 @@ static kokoro_status kokoro_synthesize_from_input_ids(
std::vector<int32_t> phonemes,
float speed_mult,
kokoro_audio & out,
std::string & err_out) noexcept {
std::string & err_out,
kokoro_cancel_callback cancel,
void * cancel_user_data) noexcept {
std::lock_guard<std::mutex> lk(const_cast<kokoro_model *>(model)->mu);
const auto cancelled = [&]() {
if (!cancel || !cancel(cancel_user_data)) return false;
err_out = "cancelled";
out.samples.clear();
return true;
};
if (cancelled()) return KOKORO_E_CANCELLED;

out.sample_rate = model->hparams.sample_rate;

Expand Down Expand Up @@ -572,6 +582,7 @@ static kokoro_status kokoro_synthesize_from_input_ids(
ggml_free(gctx);
}
}
if (cancelled()) return KOKORO_E_CANCELLED;

// 4. Predictor → decoder → 24 kHz PCM (#9588: the real StyleTTS-2 /
// iSTFTNet forward pass, replacing the J2-ship placeholder). The
Expand All @@ -588,6 +599,7 @@ static kokoro_status kokoro_synthesize_from_input_ids(
return KOKORO_E_RUNTIME;
}
}
if (cancelled()) return KOKORO_E_CANCELLED;
const int T = pred.T_frame;
if (T <= 0 || (int) pred.asr.size() != T * 512) {
err_out = "predictor produced empty/invalid asr (T=" + std::to_string(T) + ")";
Expand All @@ -597,11 +609,13 @@ static kokoro_status kokoro_synthesize_from_input_ids(
// Transpose asr [T, 512] (T-major) → [512, T] (channel-major).
std::vector<float> asr_ct((size_t) 512 * (size_t) T);
for (int t = 0; t < T; ++t) {
if ((t & 63) == 0 && cancelled()) return KOKORO_E_CANCELLED;
const float * row = pred.asr.data() + (size_t) t * 512;
for (int c = 0; c < 512; ++c) {
asr_ct[(size_t) c * (size_t) T + t] = row[c];
}
}
if (cancelled()) return KOKORO_E_CANCELLED;

{
kokoro_phase_timer t("decoder forward");
Expand All @@ -612,6 +626,7 @@ static kokoro_status kokoro_synthesize_from_input_ids(
return KOKORO_E_RUNTIME;
}
}
if (cancelled()) return KOKORO_E_CANCELLED;
return KOKORO_OK;
}

Expand Down Expand Up @@ -639,7 +654,9 @@ kokoro_status kokoro_synthesize(
const std::string & text,
float speed_mult,
kokoro_audio & out,
std::string & err_out) noexcept {
std::string & err_out,
kokoro_cancel_callback cancel,
void * cancel_user_data) noexcept {
err_out.clear();
out.samples.clear();
out.sample_rate = 24000;
Expand All @@ -653,7 +670,8 @@ kokoro_status kokoro_synthesize(

// 1. Phonemize (espeak-ng IPA when linked, else lossy ASCII grapheme map).
return kokoro_synthesize_from_input_ids(
model, voice, kokoro_phonemize(text), speed_mult, out, err_out);
model, voice, kokoro_phonemize(text), speed_mult, out, err_out,
cancel, cancel_user_data);
}

kokoro_status kokoro_synthesize_ipa(
Expand All @@ -662,7 +680,9 @@ kokoro_status kokoro_synthesize_ipa(
const std::string & ipa,
float speed_mult,
kokoro_audio & out,
std::string & err_out) noexcept {
std::string & err_out,
kokoro_cancel_callback cancel,
void * cancel_user_data) noexcept {
err_out.clear();
out.samples.clear();
out.sample_rate = 24000;
Expand All @@ -678,7 +698,7 @@ kokoro_status kokoro_synthesize_ipa(
// wrapped as the model input_ids [PAD, *ids, PAD] — no internal G2P.
return kokoro_synthesize_from_input_ids(
model, voice, wrap_input_ids(ipa_to_token_ids(ipa)), speed_mult, out,
err_out);
err_out, cancel, cancel_user_data);
}

int kokoro_sample_rate(const kokoro_model * model) noexcept {
Expand Down
7 changes: 7 additions & 0 deletions tools/omnivoice/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,13 @@ if(TARGET mtmd)
# common/sampling.cpp, and common/common.cpp. PRIVATE so the common ABI
# is not re-exported from the fused .so (only eliza_inference_* + llama).
target_link_libraries(elizainference PRIVATE eliza_voice_classifiers llama-common)
# Static objects folded into the Android/Linux shared library call libm
# directly. The dependency must live on the final shared target; relying on
# a private transitive edge from ggml-base can leave symbols such as powf
# unresolved and produces a .so that links but fails at dlopen().
if(UNIX AND NOT APPLE)
target_link_libraries(elizainference PRIVATE m)
endif()
# Kokoro-82M TTS (ABI v10): fold kokoro_lib into the fused .so so the
# mobile Kokoro path synthesizes in-process (iOS / Google Play forbid the
# local-TCP `llama-server /v1/audio/speech` route). kokoro_lib carries its
Expand Down
16 changes: 14 additions & 2 deletions tools/omnivoice/src/eliza-inference-ffi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1788,8 +1788,14 @@ int eliza_inference_kokoro_synthesize(
const std::string in(text, text_len ? text_len : std::strlen(text));
eliza_kokoro::kokoro_audio audio;
std::string err;
ctx->tts_cancel.store(false, std::memory_order_release);
eliza_kokoro::kokoro_status st = eliza_kokoro::kokoro_synthesize(
ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err);
ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err,
eliza_tts_cancel_requested, ctx);
if (st == eliza_kokoro::KOKORO_E_CANCELLED) {
eliza_set_error(out_error, "[libelizainference] kokoro_synthesize cancelled");
return ELIZA_ERR_CANCELLED;
}
if (st != eliza_kokoro::KOKORO_OK) {
eliza_set_error(out_error,
std::string("[libelizainference] kokoro_synthesize failed: ") + err);
Expand Down Expand Up @@ -1862,8 +1868,14 @@ int eliza_inference_kokoro_synthesize_ipa(
const std::string in(ipa, ipa_len ? ipa_len : std::strlen(ipa));
eliza_kokoro::kokoro_audio audio;
std::string err;
ctx->tts_cancel.store(false, std::memory_order_release);
eliza_kokoro::kokoro_status st = eliza_kokoro::kokoro_synthesize_ipa(
ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err);
ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err,
eliza_tts_cancel_requested, ctx);
if (st == eliza_kokoro::KOKORO_E_CANCELLED) {
eliza_set_error(out_error, "[libelizainference] kokoro_synthesize_ipa cancelled");
return ELIZA_ERR_CANCELLED;
}
if (st != eliza_kokoro::KOKORO_OK) {
eliza_set_error(out_error,
std::string("[libelizainference] kokoro_synthesize_ipa failed: ") + err);
Expand Down
Loading