k2-fsa_sherpa-onnx/.github/workflows/upload-models.yaml

name: upload-models

on:
  push:
    branches:
      - upload-models
      # - upload-more-models
  workflow_dispatch:

concurrency:
  group: upload-models-${{ github.ref }}
  cancel-in-progress: true

jobs:
  upload-models:
    if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj'
    name: upload models
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]

    steps:
      - uses: actions/checkout@v4

      - name: git config
        shell: bash
        run: |
          git config --global user.email "csukuangfj@gmail.com"
          git config --global user.name "Fangjun Kuang"

      - name: Streaming zipformer from Banafo/Kroko-ASR
        if: false
        shell: bash
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          git lfs install
          git clone https://csukuangfj:$HF_TOKEN@huggingface.co/Banafo/Kroko-ASR src
          pushd src
          curl -SL -O https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Python/resolve/main/de_encoder.onnx
          curl -SL -O https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Python/resolve/main/de_decoder.onnx
          curl -SL -O https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Python/resolve/main/de_joiner.onnx
          curl -SL -O https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Python/resolve/main/de_tokens.txt
          popd

          for lang in en es fr de; do
            repo=sherpa-onnx-streaming-zipformer-$lang-kroko-2025-08-06
            git clone https://huggingface.co/csukuangfj/$repo
            cp src/${lang}_encoder.onnx $repo/encoder.onnx
            cp src/${lang}_decoder.onnx $repo/decoder.onnx
            cp src/${lang}_joiner.onnx $repo/joiner.onnx
            cp src/${lang}_tokens.txt $repo/tokens.txt

            pushd $repo

            echo "See license at https://huggingface.co/Banafo/Kroko-ASR" > README.md

            mkdir -p test_wavs
            pushd test_wavs
            curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$lang.wav
            mv $lang.wav 0.wav
            popd

            git lfs track "*.onnx" "*.wav"
            git status
            ls -lh
            git add .
            git commit -m 'add model files' || true
            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$repo main || true

            popd

            rm -rf $repo/.git*

            tar cjfv $repo.tar.bz2 $repo

            ls -lh *.tar.bz2
          done

      - name: FireRed ASR fp16
        if: false
        shell: bash
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16 hf

          git lfs install
          git clone https://www.modelscope.cn/csukuangfj/sherpa-onnx-fire-red-asr-large-zh_en-fp16-2025-02-16.git ms

          d=sherpa-onnx-fire-red-asr-large-zh_en-fp16-2025-02-16
          git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d
          mv -v hf/test_wavs $d
          mv -v hf/README.md $d
          mv -v hf/tokens.txt $d
          mv -v ms/*.onnx $d

          pushd $d
          git lfs track "*.onnx"
          git lfs track "*.wav"
          git status
          git add .
          git commit -m "add models"
          ls -lh
          git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main
          popd

          rm -rf $d/.git
          rm -rf $d/.gitattributes
          tar cjvf $d.tar.bz2 $d

      - name: Zipformer CTC (non-streaming)
        if: false
        shell: bash
        env:
          MS_TOKEN: ${{ secrets.MODEL_SCOPE_GIT_TOKEN }}
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          git lfs install
          names=(
            sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03
            sherpa-onnx-zipformer-ctc-zh-2025-07-03
            sherpa-onnx-zipformer-ctc-zh-fp16-2025-07-03
            sherpa-onnx-zipformer-ctc-small-zh-int8-2025-07-16
            sherpa-onnx-zipformer-ctc-small-zh-fp16-2025-07-16
            sherpa-onnx-zipformer-ctc-small-zh-2025-07-16
          )
          for name in ${names[@]}; do
            rm -rf ms
            git clone https://oauth2:${MS_TOKEN}@www.modelscope.cn/csukuangfj/$name.git ms
            git clone https://huggingface.co/csukuangfj/$name

            cp -av ms/test_wavs $name
            cp -v ms/*.onnx $name
            cp -v ms/tokens.txt $name
            cp -v ms/bbpe.model $name

            pushd $name
            git lfs track "*.wav" "*.onnx" "*.model"
            git add .
            git status
            git commit -m 'add models' || true
            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$name main || true

            # git lfs pull
            rm -rf .git
            rm -rfv .gitattributes
            ls -lh
            popd

            tar cjfv $name.tar.bz2 $name
            rm -rf $name
            ls -lh *.tar.bz2
          done

      - name: sense-voice
        if: false
        shell: bash
        run: |
          git lfs install
          d=sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2025-09-09
          f=sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09
          git clone https://huggingface.co/csukuangfj/$d
          git clone https://huggingface.co/csukuangfj/$f

          rm -rf $d/.git
          rm -rf $f/.gi*

          rm -rf $d/.gitattributes
          rm -rf $f/.gitattributes

          tar cjfv $d.tar.bz2 $d
          tar cjfv $f.tar.bz2 $f

          ls -lh *.tar.bz2

      - name: u2ppconformer
        if: true
        shell: bash
        run: |
          git lfs install

          d=sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-2025-09-10
          f=sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10

          git clone https://huggingface.co/csukuangfj/$d
          git clone https://huggingface.co/csukuangfj/$f

          rm -rf $d/.git
          rm -rf $f/.gi*

          rm -rf $d/.gitattributes
          rm -rf $f/.gitattributes

          tar cjfv $d.tar.bz2 $d
          tar cjfv $f.tar.bz2 $f

          ls -lh *.tar.bz2

      - name: Vietnamese (zipformer)
        if: false
        shell: bash
        run: |
          rm -rf models
          mkdir models
          cd models
          cat >README.md <<EOF
          # Introduction
          Models in this directory are from
          https://huggingface.co/zzasdf/viet_iter3_pseudo_label
          which are trained on about 70k hours of data.
          EOF

          git lfs install
          git clone https://huggingface.co/csukuangfj/viet_iter3_pseudo_label hf

          ls -lh

          d=sherpa-onnx-zipformer-vi-2025-04-20
          mkdir -p $d
          cp -v hf/exp/encoder-epoch-12-avg-8.onnx $d/
          cp -v hf/exp/decoder-epoch-12-avg-8.onnx $d/
          cp -v hf/exp/joiner-epoch-12-avg-8.onnx $d/
          cp -v hf/data/Vietnam_bpe_2000_new/bpe.model $d/
          cp -v hf/data/Vietnam_bpe_2000_new/tokens.txt $d/
          cp -av hf/test_wavs $d
          cp -v README.md $d

          tar cjfv $d.tar.bz2 $d

          d=sherpa-onnx-zipformer-vi-int8-2025-04-20
          mkdir -p $d

          cp -v hf/exp/encoder-epoch-12-avg-8.int8.onnx $d/
          cp -v hf/exp/decoder-epoch-12-avg-8.onnx $d/
          cp -v hf/exp/joiner-epoch-12-avg-8.int8.onnx $d/
          cp -v hf/data/Vietnam_bpe_2000_new/bpe.model $d/
          cp -v hf/data/Vietnam_bpe_2000_new/tokens.txt $d/
          cp -av hf/test_wavs $d
          cp -v README.md $d

          tar cjfv $d.tar.bz2 $d

          rm -rf hf

          ls -lh

          cd ..

          mv models/* .

      - name: Publish to huggingface (Vietnamese zipformer)
        if: false
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        uses: nick-fields/retry@v3
        with:
          max_attempts: 20
          timeout_seconds: 200
          shell: bash
          command: |
            git config --global user.email "csukuangfj@gmail.com"
            git config --global user.name "Fangjun Kuang"
            models=(
              sherpa-onnx-zipformer-vi-2025-04-20
              sherpa-onnx-zipformer-vi-int8-2025-04-20
            )
            for d in ${models[@]}; do
              export GIT_LFS_SKIP_SMUDGE=1
              export GIT_CLONE_PROTECTION_ACTIVE=false
              rm -rf huggingface
              git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface
              cp -av $d/* huggingface

              pushd huggingface
              git lfs track "*.onnx"
              git lfs track "bpe.model"
              git lfs track "*.wav"
              git status
              git add .

              git commit -m "add models"
              git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main

              popd
            done

      - name: vosk-model-ru (stream zipformer)
        if: false
        shell: bash
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          git config --global user.email "csukuangfj@gmail.com"
          git config --global user.name "Fangjun Kuang"

          cat >README.md <<EOF
          # Introduction
          Models in this directory are from
          https://huggingface.co/alphacep/vosk-model-small-streaming-ru
          EOF

          git lfs install
          git clone https://huggingface.co/alphacep/vosk-model-small-streaming-ru hf

          git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-small-ru-vosk-int8-2025-08-16 int8
          git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-small-ru-vosk-2025-08-16 fp32

          rm -fv int8/*.onnx
          rm -fv fp32/*.onnx

          mkdir -p int8/test_wavs
          mkdir -p fp32/test_wavs

          curl -SL -O https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition/resolve/main/test_wavs/russian/russian-i-love-you.wav
          curl -SL -O https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition/resolve/main/test_wavs/russian/test.wav

          mv russian-i-love-you.wav 0.wav
          mv test.wav 1.wav

          cp -v README.md int8/
          cp -v README.md fp32/

          cp -v *.wav int8/test_wavs
          cp -v *.wav fp32/test_wavs

          cp -v hf/am-onnx/{encoder,decoder,joiner}.onnx fp32/

          cp -v hf/am-onnx/{encoder,joiner}.int8.onnx int8/
          cp -v hf/am-onnx/decoder.onnx int8/

          cp -v hf/lang/tokens.txt int8/
          cp -v hf/lang/bpe.model int8/

          cp -v hf/lang/tokens.txt fp32/
          cp -v hf/lang/bpe.model fp32/

          mv int8 sherpa-onnx-streaming-zipformer-small-ru-vosk-int8-2025-08-16
          mv fp32 sherpa-onnx-streaming-zipformer-small-ru-vosk-2025-08-16

          models=(
            sherpa-onnx-streaming-zipformer-small-ru-vosk-2025-08-16
            sherpa-onnx-streaming-zipformer-small-ru-vosk-int8-2025-08-16
          )

          for d in ${models[@]}; do
            export GIT_LFS_SKIP_SMUDGE=1
            export GIT_CLONE_PROTECTION_ACTIVE=false
            pushd $d
            git lfs track "*.onnx"
            git lfs track "bpe.model"
            git lfs track "*.wav"
            git status
            git add .

            git commit -m "add models"
            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main
            popd

            rm -rf $d/.git*

            tar cjfv $d.tar.bz2 $d
          done
          ls -lh *.tar.bz2

      - name: vosk-model-ru (zipformer)
        if: false
        shell: bash
        run: |
          rm -rf models
          mkdir models
          cd models
          cat >README.md <<EOF
          # Introduction
          Models in this directory are from
          https://huggingface.co/alphacep/vosk-model-ru/tree/main
          EOF

          git lfs install
          git clone https://huggingface.co/alphacep/vosk-model-ru hf

          ls -lh

          mkdir test_wavs
          pushd test_wavs
          curl -SL -O https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition/resolve/main/test_wavs/russian/russian-i-love-you.wav
          curl -SL -O https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition/resolve/main/test_wavs/russian/test.wav

          mv russian-i-love-you.wav 0.wav
          mv test.wav 1.wav
          popd

          d=sherpa-onnx-zipformer-ru-2025-04-20
          mkdir $d
          cp -v hf/am-onnx/encoder.onnx $d
          cp -v hf/am-onnx/decoder.onnx $d
          cp -v hf/am-onnx/joiner.onnx $d
          cp -v hf/lang/bpe.model $d
          cp -v hf/lang/tokens.txt $d
          cp -av test_wavs $d/
          cp -v README.md $d

          tar cjfv $d.tar.bz2 $d

          d=sherpa-onnx-zipformer-ru-int8-2025-04-20
          mkdir $d
          cp -v hf/am-onnx/encoder.int8.onnx $d
          cp -v hf/am-onnx/decoder.onnx $d
          cp -v hf/am-onnx/joiner.int8.onnx $d
          cp -v hf/lang/bpe.model $d
          cp -v hf/lang/tokens.txt $d
          cp -av test_wavs $d
          cp -v README.md $d

          tar cjfv $d.tar.bz2 $d

          rm -rf hf

          ls -lh

          cd ..

          mv models/* .

      - name: Publish to huggingface (Russian zipformer)
        if: false
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        uses: nick-fields/retry@v3
        with:
          max_attempts: 20
          timeout_seconds: 200
          shell: bash
          command: |
            git config --global user.email "csukuangfj@gmail.com"
            git config --global user.name "Fangjun Kuang"
            models=(
              sherpa-onnx-zipformer-ru-2025-04-20
              sherpa-onnx-zipformer-ru-int8-2025-04-20
            )
            for d in ${models[@]}; do
              export GIT_LFS_SKIP_SMUDGE=1
              export GIT_CLONE_PROTECTION_ACTIVE=false
              rm -rf huggingface
              git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface
              cp -av $d/* huggingface

              pushd huggingface
              git lfs track "*.onnx"
              git lfs track "bpe.model"
              git lfs track "*.wav"
              git status
              git add .

              git commit -m "add models"
              git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main

              popd
            done

      - name: Release
        uses: svenstaro/upload-release-action@v2
        with:
          file_glob: true
          file: ./*.tar.bz2
          overwrite: true
          repo_name: k2-fsa/sherpa-onnx
          repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
          tag: asr-models