Add CXX API for KittenTTS (#2469)

.github/workflows/cxx-api.yaml

@@ -87,6 +87,45 @@ jobs:
             otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib
           fi
 
+      - name: Test KittenTTS
+        shell: bash
+        run: |
+          name=kitten-tts-en-cxx-api
+          g++ -std=c++17 -o $name ./cxx-api-examples/$name.cc \
+            -I ./build/install/include \
+            -L ./build/install/lib/ \
+            -l sherpa-onnx-cxx-api \
+            -l sherpa-onnx-c-api \
+            -l onnxruntime
+
+          ls -lh $name
+
+          if [[ ${{ matrix.os }} == ubuntu-latest || ${{ matrix.os }} == ubuntu-22.04-arm ]]; then
+            ls -lh ./$name
+            ldd ./$name
+            echo "----"
+            readelf -d ./$name
+          fi
+
+          curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
+          tar xf kitten-nano-en-v0_1-fp16.tar.bz2
+          rm kitten-nano-en-v0_1-fp16.tar.bz2
+
+          echo "---"
+
+          export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
+          export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
+
+          ./$name
+
+          rm -rf kitten-nano-en-v0_1-fp16
+          rm -v ./$name
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: kitten-tts-wavs-${{ matrix.os }}
+          path: ./generated-kitten-en-cxx.wav
+
       - name: Test NeMo Canary
         shell: bash
         run: |

build-android-arm64-v8a.sh

@@ -185,7 +185,7 @@ libsherpa-onnx-cxx-api.so.
 libsherpa-onnx-c-api.so and libsherpa-onnx-cxx-api.so are for users
 who don't use JNI. In that case, libsherpa-onnx-jni.so is not needed.
 
-In any case, libonnxruntime.is is always needed.
+In any case, libonnxruntime.so is always needed.
 EOF
   ls -lh install/lib/README.md
 fi

build-android-armv7-eabi.sh

@@ -185,7 +185,7 @@ libsherpa-onnx-cxx-api.so.
 libsherpa-onnx-c-api.so and libsherpa-onnx-cxx-api.so are for users
 who don't use JNI. In that case, libsherpa-onnx-jni.so is not needed.
 
-In any case, libonnxruntime.is is always needed.
+In any case, libonnxruntime.so is always needed.
 EOF
   ls -lh install/lib/README.md
 fi

build-android-x86-64.sh

@@ -164,7 +164,7 @@ libsherpa-onnx-cxx-api.so.
 libsherpa-onnx-c-api.so and libsherpa-onnx-cxx-api.so are for users
 who don't use JNI. In that case, libsherpa-onnx-jni.so is not needed.
 
-In any case, libonnxruntime.is is always needed.
+In any case, libonnxruntime.so is always needed.
 EOF
   ls -lh install/lib/README.md
 fi

build-android-x86.sh

@@ -125,7 +125,7 @@ libsherpa-onnx-cxx-api.so.
 libsherpa-onnx-c-api.so and libsherpa-onnx-cxx-api.so are for users
 who don't use JNI. In that case, libsherpa-onnx-jni.so is not needed.
 
-In any case, libonnxruntime.is is always needed.
+In any case, libonnxruntime.so is always needed.
 EOF
   ls -lh install/lib/README.md
 fi

c-api-examples/kitten-tts-en-c-api.c

@@ -55,8 +55,9 @@ int32_t main(int32_t argc, char *argv[]) {
 
   const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config);
   // mapping of sid to voice name
-  // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam
-  // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis
+  // 0->expr-voice-2-m, 1->expr-voice-2-f, 2->expr-voice-3-m
+  // 3->expr-voice-3-f, 4->expr-voice-4-m, 5->expr-voice-4-f
+  // 6->expr-voice-5-m, 7->expr-voice-5-f
   int32_t sid = 0;
   float speed = 1.0;  // larger -> faster in speech speed
 

cxx-api-examples/CMakeLists.txt

@@ -124,6 +124,9 @@ if(SHERPA_ONNX_ENABLE_TTS)
   add_executable(kokoro-tts-en-cxx-api ./kokoro-tts-en-cxx-api.cc)
   target_link_libraries(kokoro-tts-en-cxx-api sherpa-onnx-cxx-api)
 
+  add_executable(kitten-tts-en-cxx-api ./kitten-tts-en-cxx-api.cc)
+  target_link_libraries(kitten-tts-en-cxx-api sherpa-onnx-cxx-api)
+
   add_executable(kokoro-tts-zh-en-cxx-api ./kokoro-tts-zh-en-cxx-api.cc)
   target_link_libraries(kokoro-tts-zh-en-cxx-api sherpa-onnx-cxx-api)
 endif()

cxx-api-examples/kitten-tts-en-cxx-api.cc

@@ -0,0 +1,75 @@
+// cxx-api-examples/kitten-tts-en-cxx-api.cc
+//
+// Copyright (c)  2025  Xiaomi Corporation
+
+// This file shows how to use sherpa-onnx CXX API
+// for English TTS with Kitten.
+//
+// clang-format off
+/*
+Usage
+
+wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
+tar xf kitten-nano-en-v0_1-fp16.tar.bz2
+rm kitten-nano-en-v0_1-fp16.tar.bz2
+
+./kitten-tts-en-cxx-api
+
+ */
+// clang-format on
+
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+#include "sherpa-onnx/c-api/cxx-api.h"
+
+static int32_t ProgressCallback(const float *samples, int32_t num_samples,
+                                float progress, void *arg) {
+  fprintf(stderr, "Progress: %.3f%%\n", progress * 100);
+  // return 1 to continue generating
+  // return 0 to stop generating
+  return 1;
+}
+
+int32_t main(int32_t argc, char *argv[]) {
+  using namespace sherpa_onnx::cxx;  // NOLINT
+  OfflineTtsConfig config;
+
+  config.model.kitten.model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx";
+  config.model.kitten.voices = "./kitten-nano-en-v0_1-fp16/voices.bin";
+  config.model.kitten.tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt";
+  config.model.kitten.data_dir = "./kitten-nano-en-v0_1-fp16/espeak-ng-data";
+
+  config.model.num_threads = 2;
+
+  // If you don't want to see debug messages, please set it to 0
+  config.model.debug = 1;
+
+  std::string filename = "./generated-kitten-en-cxx.wav";
+  std::string text =
+      "Today as always, men fall into two groups: slaves and free men. Whoever "
+      "does not have two-thirds of his day for himself, is a slave, whatever "
+      "he may be: a statesman, a businessman, an official, or a scholar. "
+      "Friends fell out often because life was changing so fast. The easiest "
+      "thing in the world was to lose touch with someone.";
+
+  auto tts = OfflineTts::Create(config);
+  int32_t sid = 0;
+  float speed = 1.0;  // larger -> faster in speech speed
+
+#if 0
+  // If you don't want to use a callback, then please enable this branch
+  GeneratedAudio audio = tts.Generate(text, sid, speed);
+#else
+  GeneratedAudio audio = tts.Generate(text, sid, speed, ProgressCallback);
+#endif
+
+  WriteWave(filename, {audio.samples, audio.sample_rate});
+
+  fprintf(stderr, "Input text is: %s\n", text.c_str());
+  fprintf(stderr, "Speaker ID is: %d\n", sid);
+  fprintf(stderr, "Saved to: %s\n", filename.c_str());
+
+  return 0;
+}

cxx-api-examples/kokoro-tts-en-cxx-api.cc

@@ -1,4 +1,4 @@
-// cxx-api-examples/kokoro-tts-en-cxx-api.c
+// cxx-api-examples/kokoro-tts-en-cxx-api.cc
 //
 // Copyright (c)  2025  Xiaomi Corporation
 
@@ -18,6 +18,8 @@ rm kokoro-en-v0_19.tar.bz2
  */
 // clang-format on
 
+#include <cstdint>
+#include <cstdio>
 #include <string>
 
 #include "sherpa-onnx/c-api/cxx-api.h"
@@ -66,7 +68,7 @@ int32_t main(int32_t argc, char *argv[]) {
   WriteWave(filename, {audio.samples, audio.sample_rate});
 
   fprintf(stderr, "Input text is: %s\n", text.c_str());
-  fprintf(stderr, "Speaker ID is is: %d\n", sid);
+  fprintf(stderr, "Speaker ID is: %d\n", sid);
   fprintf(stderr, "Saved to: %s\n", filename.c_str());
 
   return 0;

cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc

@@ -18,6 +18,8 @@ rm kokoro-multi-lang-v1_0.tar.bz2
  */
 // clang-format on
 
+#include <cstdint>
+#include <cstdio>
 #include <string>
 
 #include "sherpa-onnx/c-api/cxx-api.h"
@@ -67,7 +69,7 @@ int32_t main(int32_t argc, char *argv[]) {
   WriteWave(filename, {audio.samples, audio.sample_rate});
 
   fprintf(stderr, "Input text is: %s\n", text.c_str());
-  fprintf(stderr, "Speaker ID is is: %d\n", sid);
+  fprintf(stderr, "Speaker ID is: %d\n", sid);
   fprintf(stderr, "Saved to: %s\n", filename.c_str());
 
   return 0;

cxx-api-examples/matcha-tts-en-cxx-api.cc

@@ -20,6 +20,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/voco
  */
 // clang-format on
 
+#include <cstdint>
+#include <cstdio>
 #include <string>
 
 #include "sherpa-onnx/c-api/cxx-api.h"
@@ -73,7 +75,7 @@ int32_t main(int32_t argc, char *argv[]) {
   WriteWave(filename, {audio.samples, audio.sample_rate});
 
   fprintf(stderr, "Input text is: %s\n", text.c_str());
-  fprintf(stderr, "Speaker ID is is: %d\n", sid);
+  fprintf(stderr, "Speaker ID is: %d\n", sid);
   fprintf(stderr, "Saved to: %s\n", filename.c_str());
 
   return 0;

cxx-api-examples/matcha-tts-zh-cxx-api.cc

@@ -20,6 +20,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/voco
  */
 // clang-format on
 
+#include <cstdint>
+#include <cstdio>
 #include <string>
 
 #include "sherpa-onnx/c-api/cxx-api.h"
@@ -72,7 +74,7 @@ int32_t main(int32_t argc, char *argv[]) {
   WriteWave(filename, {audio.samples, audio.sample_rate});
 
   fprintf(stderr, "Input text is: %s\n", text.c_str());
-  fprintf(stderr, "Speaker ID is is: %d\n", sid);
+  fprintf(stderr, "Speaker ID is: %d\n", sid);
   fprintf(stderr, "Saved to: %s\n", filename.c_str());
 
   return 0;

sherpa-onnx/c-api/cxx-api.cc

@@ -392,6 +392,12 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) {
   c.model.kokoro.lexicon = config.model.kokoro.lexicon.c_str();
   c.model.kokoro.lang = config.model.kokoro.lang.c_str();
 
+  c.model.kitten.model = config.model.kitten.model.c_str();
+  c.model.kitten.voices = config.model.kitten.voices.c_str();
+  c.model.kitten.tokens = config.model.kitten.tokens.c_str();
+  c.model.kitten.data_dir = config.model.kitten.data_dir.c_str();
+  c.model.kitten.length_scale = config.model.kitten.length_scale;
+
   c.model.num_threads = config.model.num_threads;
   c.model.debug = config.model.debug;
   c.model.provider = config.model.provider.c_str();

sherpa-onnx/c-api/cxx-api.h

@@ -394,10 +394,20 @@ struct OfflineTtsKokoroModelConfig {
   float length_scale = 1.0;  // < 1, faster in speed; > 1, slower in speed
 };
 
+struct OfflineTtsKittenModelConfig {
+  std::string model;
+  std::string voices;
+  std::string tokens;
+  std::string data_dir;
+
+  float length_scale = 1.0;  // < 1, faster in speed; > 1, slower in speed
+};
+
 struct OfflineTtsModelConfig {
   OfflineTtsVitsModelConfig vits;
   OfflineTtsMatchaModelConfig matcha;
   OfflineTtsKokoroModelConfig kokoro;
+  OfflineTtsKittenModelConfig kitten;
   int32_t num_threads = 1;
   bool debug = false;
   std::string provider = "cpu";