rust/sbv2-api
1
0
Fork 0
mirror of https://github.com/neodyland/sbv2-api.git synced 2025-12-24 08:19:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: rust:commit-3caf9344
Branches Tags
rust:main rust:dependabot/cargo/axum-0.8.8 rust:ndarray-update rust:revert-274-dependabot/cargo/ndarray-0.17.1 rust:renovate/esbuild-0.x-lockfile rust:renovate/onnxruntime-web-1.x-lockfile rust:renovate/node-22.x-lockfile rust:marine rust:mecab rust:v0.1.x
rust:commit-b54874a7 rust:commit-e23d0ebf rust:commit-9b5cf6d7 rust:commit-7c0991a1 rust:commit-783a5e65 rust:commit-8f3b7ff5 rust:commit-84a16766 rust:v0.2.0 rust:commit-42cbdd5f rust:commit-896f1e52 rust:commit-9f543377 rust:commit-2cb661fc rust:commit-d6471ad9 rust:commit-75c44553 rust:commit-767bfadf rust:commit-82c9f13c rust:commit-a7e6fadf rust:commit-7034c1aa rust:commit-f00ce040 rust:commit-7d32db19 rust:v0.2.0-alpha9 rust:commit-2cb43aaf rust:commit-d5a0f16b rust:commit-9fbe7579 rust:commit-ca5887e8 rust:commit-f587a2c7 rust:commit-36f031bf rust:commit-47f2af14 rust:commit-b9e2c0fe rust:commit-0651b492 rust:commit-56e663c3 rust:commit-8d97d422 rust:commit-404fa0f7 rust:commit-33d6f1cc rust:commit-8e031387 rust:commit-51c0bf85 rust:commit-043f380c rust:commit-f98c6b82 rust:commit-703035cc rust:commit-3553d01a rust:commit-91ac2fad rust:commit-93c121da rust:v0.2.0-alpha8 rust:commit-69d2857d rust:commit-5cb4480a rust:commit-ebcb6f90 rust:commit-e47f57ad rust:commit-dc5fa884 rust:commit-25ca89e3 rust:commit-84e9118d rust:commit-d5fcacd7 rust:commit-3050cc1e rust:commit-9f22694d rust:commit-62ba2c80 rust:commit-e9ced32b rust:commit-e7a1575c rust:commit-320664ea rust:commit-55f05580 rust:commit-1725863f rust:commit-87903827 rust:commit-ee56e959 rust:commit-00f4787f rust:commit-4b6c72aa rust:commit-e9529be5 rust:commit-a6694b5d rust:commit-096859de rust:v0.2.0-alpha7 rust:commit-98ddaa3c rust:commit-120bc608 rust:commit-45c3255a rust:commit-159f5802 rust:commit-9d6aa46f rust:commit-02ac0885 rust:commit-1f96b09f rust:commit-d583c1ca rust:commit-61914129 rust:commit-97c63a2e rust:commit-bca6d04e rust:commit-96b53d42 rust:commit-e68f58d6 rust:commit-1de09597 rust:commit-38d86c92 rust:commit-2124fe46 rust:commit-0217c0a4 rust:commit-d7065ac6 rust:commit-1ad588bf rust:commit-843c1699 rust:v0.2.0-alpha6 rust:commit-9733ba95 rust:commit-f0821ea9 rust:commit-e5e92f62 rust:commit-3caf9344 rust:commit-9052fd41 rust:commit-b8355773 rust:commit-9174aa9b rust:commit-bbb3f000 rust:commit-6bccf046 rust:commit-4deefc59 rust:commit-306284c7 rust:commit-bf79a063 rust:commit-ed5fa41c rust:commit-eaff6102 rust:commit-b5aa4891 rust:commit-b007e1b1 rust:commit-f081b2ed rust:commit-cb125023 rust:commit-4ae9b63b rust:commit-1089dc1a rust:commit-110e0753 rust:commit-9063e620 rust:commit-8a0b2d56 rust:commit-463d988a rust:commit-990e7e9b rust:commit-04e50088 rust:commit-e9811765 rust:commit-9a8b4fde rust:commit-5a805342 rust:commit-9e2534fa rust:commit-3bc1d7ce rust:commit-8913148f rust:commit-1d57a58c rust:commit-de3dfb50 rust:commit-dd84c3a9 rust:commit-a8bd62ce rust:commit-7b20a414 rust:commit-33cc9581 rust:commit-c4d2f017 rust:commit-3142b1ea rust:commit-0ab1da9e rust:commit-99a4b130 rust:commit-e8ee2c06 rust:commit-61aae68d rust:commit-ecf58659 rust:commit-f86d3233 rust:commit-7bf2c487 rust:commit-adb699ef rust:commit-8e8f58b4 rust:commit-38c5471d rust:v0.2.0-alpha.5 rust:v0.2.0-rc.8 rust:v0.2.0-rc.7 rust:v0.2.0-rc.6 rust:v0.2.0-rc.5 rust:v0.2.0-rc.4 rust:v0.2.0-rc.3 rust:v0.2.0-rc.2 rust:v0.2.0-rc.1 rust:v0.2.0-rc rust:v0.2.0-alpha5 rust:v0.2.0-alpha4 rust:v0.2.0-alpha3 rust:v0.2.0-alpha2-1 rust:v0.2.0-alpha2 rust:v0.2.0-alpha1 rust:v0.2.0-alpha rust:py-v0.1.1 rust:py-v0.1.0 rust:v0.1.4-1 rust:v0.1.4
..
compare: rust:mecab
Branches Tags
rust:dependabot/cargo/axum-0.8.8 rust:main rust:ndarray-update rust:revert-274-dependabot/cargo/ndarray-0.17.1 rust:renovate/esbuild-0.x-lockfile rust:renovate/onnxruntime-web-1.x-lockfile rust:renovate/node-22.x-lockfile rust:marine rust:mecab rust:v0.1.x
rust:commit-b54874a7 rust:commit-e23d0ebf rust:commit-9b5cf6d7 rust:commit-7c0991a1 rust:commit-783a5e65 rust:commit-8f3b7ff5 rust:commit-84a16766 rust:v0.2.0 rust:commit-42cbdd5f rust:commit-896f1e52 rust:commit-9f543377 rust:commit-2cb661fc rust:commit-d6471ad9 rust:commit-75c44553 rust:commit-767bfadf rust:commit-82c9f13c rust:commit-a7e6fadf rust:commit-7034c1aa rust:commit-f00ce040 rust:commit-7d32db19 rust:v0.2.0-alpha9 rust:commit-2cb43aaf rust:commit-d5a0f16b rust:commit-9fbe7579 rust:commit-ca5887e8 rust:commit-f587a2c7 rust:commit-36f031bf rust:commit-47f2af14 rust:commit-b9e2c0fe rust:commit-0651b492 rust:commit-56e663c3 rust:commit-8d97d422 rust:commit-404fa0f7 rust:commit-33d6f1cc rust:commit-8e031387 rust:commit-51c0bf85 rust:commit-043f380c rust:commit-f98c6b82 rust:commit-703035cc rust:commit-3553d01a rust:commit-91ac2fad rust:commit-93c121da rust:v0.2.0-alpha8 rust:commit-69d2857d rust:commit-5cb4480a rust:commit-ebcb6f90 rust:commit-e47f57ad rust:commit-dc5fa884 rust:commit-25ca89e3 rust:commit-84e9118d rust:commit-d5fcacd7 rust:commit-3050cc1e rust:commit-9f22694d rust:commit-62ba2c80 rust:commit-e9ced32b rust:commit-e7a1575c rust:commit-320664ea rust:commit-55f05580 rust:commit-1725863f rust:commit-87903827 rust:commit-ee56e959 rust:commit-00f4787f rust:commit-4b6c72aa rust:commit-e9529be5 rust:commit-a6694b5d rust:commit-096859de rust:v0.2.0-alpha7 rust:commit-98ddaa3c rust:commit-120bc608 rust:commit-45c3255a rust:commit-159f5802 rust:commit-9d6aa46f rust:commit-02ac0885 rust:commit-1f96b09f rust:commit-d583c1ca rust:commit-61914129 rust:commit-97c63a2e rust:commit-bca6d04e rust:commit-96b53d42 rust:commit-e68f58d6 rust:commit-1de09597 rust:commit-38d86c92 rust:commit-2124fe46 rust:commit-0217c0a4 rust:commit-d7065ac6 rust:commit-1ad588bf rust:commit-843c1699 rust:v0.2.0-alpha6 rust:commit-9733ba95 rust:commit-f0821ea9 rust:commit-e5e92f62 rust:commit-3caf9344 rust:commit-9052fd41 rust:commit-b8355773 rust:commit-9174aa9b rust:commit-bbb3f000 rust:commit-6bccf046 rust:commit-4deefc59 rust:commit-306284c7 rust:commit-bf79a063 rust:commit-ed5fa41c rust:commit-eaff6102 rust:commit-b5aa4891 rust:commit-b007e1b1 rust:commit-f081b2ed rust:commit-cb125023 rust:commit-4ae9b63b rust:commit-1089dc1a rust:commit-110e0753 rust:commit-9063e620 rust:commit-8a0b2d56 rust:commit-463d988a rust:commit-990e7e9b rust:commit-04e50088 rust:commit-e9811765 rust:commit-9a8b4fde rust:commit-5a805342 rust:commit-9e2534fa rust:commit-3bc1d7ce rust:commit-8913148f rust:commit-1d57a58c rust:commit-de3dfb50 rust:commit-dd84c3a9 rust:commit-a8bd62ce rust:commit-7b20a414 rust:commit-33cc9581 rust:commit-c4d2f017 rust:commit-3142b1ea rust:commit-0ab1da9e rust:commit-99a4b130 rust:commit-e8ee2c06 rust:commit-61aae68d rust:commit-ecf58659 rust:commit-f86d3233 rust:commit-7bf2c487 rust:commit-adb699ef rust:commit-8e8f58b4 rust:commit-38c5471d rust:v0.2.0-alpha.5 rust:v0.2.0-rc.8 rust:v0.2.0-rc.7 rust:v0.2.0-rc.6 rust:v0.2.0-rc.5 rust:v0.2.0-rc.4 rust:v0.2.0-rc.3 rust:v0.2.0-rc.2 rust:v0.2.0-rc.1 rust:v0.2.0-rc rust:v0.2.0-alpha5 rust:v0.2.0-alpha4 rust:v0.2.0-alpha3 rust:v0.2.0-alpha2-1 rust:v0.2.0-alpha2 rust:v0.2.0-alpha1 rust:v0.2.0-alpha rust:py-v0.1.1 rust:py-v0.1.0 rust:v0.1.4-1 rust:v0.1.4

2 Commits
commit-3ca ... mecab

Author SHA1 Message Date
tuna2134
4744f02792 fix 2024-10-08 09:48:04 +00:00
tuna2134
5de9514546 add vibrato feature 2024-10-08 09:30:16 +00:00
76 changed files with 1989 additions and 3577 deletions
Expand all files Collapse all files

3
.env.sample
View File

@@ -3,5 +3,4 @@ MODEL_PATH=models/tsukuyomi.sbv2
MODELS_PATH=models
TOKENIZER_PATH=models/tokenizer.json
ADDR=localhost:3000
RUST_LOG=warn
HOLDER_MAX_LOADED_MODElS=20
RUST_LOG=warn

3
.github/FUNDING.yml vendored
View File

@@ -1,3 +0,0 @@
# These are supported funding model platforms
github: [tuna2134]

4
.github/workflows/CI.Dockerfile vendored
View File

@@ -1,4 +0,0 @@
FROM ubuntu:latest
RUN apt update && apt install openssl libssl-dev curl pkg-config software-properties-common -y && add-apt-repository ppa:deadsnakes/ppa && apt update && apt install python3.7 python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3-pip python3 -y
ENV PIP_BREAK_SYSTEM_PACKAGES=1
RUN mkdir -p /root/.cache/sbv2 && curl https://huggingface.co/neody/sbv2-api-assets/resolve/main/dic/all.bin -o /root/.cache/sbv2/all.bin -L

146
.github/workflows/CI.yml vendored
View File

@@ -1,35 +1,39 @@
# This file is autogenerated by maturin v1.7.1
# To update, run
#
# maturin generate-ci github
#
name: CI
on:
push:
branches:
- main
- master
tags:
- '*'
pull_request:
workflow_dispatch:
permissions:
contents: write
contents: read
id-token: write
packages: write
jobs:
python-linux:
linux:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
platform:
- runner: ubuntu-latest
target: x86_64
- runner: ubuntu-24.04-arm
- runner: ubuntu-latest
target: aarch64
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: 3.x
- run: docker build . -f .github/workflows/CI.Dockerfile --tag ci
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
@@ -37,15 +41,14 @@ jobs:
args: --release --out dist --find-interpreter
sccache: 'true'
manylinux: auto
container: ci
working-directory: ./crates/sbv2_bindings
working-directory: sbv2_bindings
- name: Upload wheels
uses: actions/upload-artifact@v4
with:
name: wheels-linux-${{ matrix.platform.target }}
path: ./crates/sbv2_bindings/dist
path: sbv2_bindings/dist
python-windows:
windows:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
@@ -64,18 +67,20 @@ jobs:
target: ${{ matrix.platform.target }}
args: --release --out dist --find-interpreter
sccache: 'true'
working-directory: ./crates/sbv2_bindings
working-directory: sbv2_bindings
- name: Upload wheels
uses: actions/upload-artifact@v4
with:
name: wheels-windows-${{ matrix.platform.target }}
path: ./crates/sbv2_bindings/dist
path: sbv2_bindings/dist
python-macos:
macos:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
platform:
- runner: macos-12
target: x86_64
- runner: macos-14
target: aarch64
steps:
@@ -89,14 +94,14 @@ jobs:
target: ${{ matrix.platform.target }}
args: --release --out dist --find-interpreter
sccache: 'true'
working-directory: ./crates/sbv2_bindings
working-directory: sbv2_bindings
- name: Upload wheels
uses: actions/upload-artifact@v4
with:
name: wheels-macos-${{ matrix.platform.target }}
path: ./crates/sbv2_bindings/dist
path: sbv2_bindings/dist
python-sdist:
sdist:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -105,116 +110,57 @@ jobs:
with:
command: sdist
args: --out dist
working-directory: ./crates/sbv2_bindings
working-directory: sbv2_bindings
- name: Upload sdist
uses: actions/upload-artifact@v4
with:
name: wheels-sdist
path: ./crates/sbv2_bindings/dist
python-wheel:
name: Wheel Upload
runs-on: ubuntu-latest
needs: [python-linux, python-windows, python-macos, python-sdist]
env:
GH_TOKEN: ${{ github.token }}
steps:
- uses: actions/checkout@v4
- run: gh run download ${{ github.run_id }} -p wheels-*
- name: release
run: |
gh release create commit-${GITHUB_SHA:0:8} --prerelease wheels-*/*
path: sbv2_bindings/dist
python-release:
release:
name: Release
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
needs: [python-linux, python-windows, python-macos, python-sdist]
needs: [linux, windows, macos, sdist]
environment: release
env:
GH_TOKEN: ${{ github.token }}
steps:
- uses: actions/checkout@v4
- run: gh run download ${{ github.run_id }} -p wheels-*
- uses: actions/download-artifact@v4
- name: Publish to PyPI
uses: PyO3/maturin-action@v1
with:
command: upload
args: --non-interactive --skip-existing wheels-*/*
docker:
runs-on: ${{ matrix.machine.runner }}
push-docker:
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
machine:
- platform: amd64
runner: ubuntu-latest
- platform: arm64
runner: ubuntu-24.04-arm
tag: [cpu, cuda]
platform:
- linux/amd64
- linux/arm64
steps:
- name: Prepare
run: |
platform=${{ matrix.machine.platform }}
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/${{ github.repository }}
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and push by digest
id: build
uses: docker/build-push-action@v6
with:
labels: ${{ steps.meta.outputs.labels }}
file: ./scripts/docker/${{ matrix.tag }}.Dockerfile
push: true
tags: |
ghcr.io/${{ github.repository }}:latest-${{ matrix.tag }}-${{ matrix.machine.platform }}
docker-merge:
runs-on: ubuntu-latest
needs:
- docker
steps:
- name: Download digests
uses: actions/download-artifact@v4
with:
path: ${{ runner.temp }}/digests
pattern: digests-*
merge-multiple: true
- name: Login to GHCR
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Merge
run: |
docker buildx imagetools create -t ghcr.io/${{ github.repository }}:cuda \
ghcr.io/${{ github.repository }}:latest-cuda-amd64 \
ghcr.io/${{ github.repository }}:latest-cuda-arm64
docker buildx imagetools create -t ghcr.io/${{ github.repository }}:cpu \
ghcr.io/${{ github.repository }}:latest-cpu-amd64 \
ghcr.io/${{ github.repository }}:latest-cpu-arm64
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: .
push: true
tags: |
ghcr.io/${{ github.repository }}:${{ matrix.tag }}
file: docker/${{ matrix.tag }}.Dockerfile
platforms: ${{ matrix.platform }}

10
.gitignore vendored
View File

@@ -1,10 +1,8 @@
target/
target
models/
!models/.gitkeep
venv/
.env
*.wav
node_modules/
dist/
*.csv
*.bin
output.wav
node_modules
dist/

2097
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

22
Cargo.toml
View File

@@ -1,25 +1,15 @@
[workspace]
resolver = "2"
members = ["./crates/sbv2_api", "./crates/sbv2_core", "./crates/sbv2_bindings", "./crates/sbv2_wasm"]
[workspace.package]
version = "0.2.0-alpha6"
edition = "2021"
description = "Style-Bert-VITSの推論ライブラリ"
license = "MIT"
readme = "./README.md"
repository = "https://github.com/tuna2134/sbv2-api"
documentation = "https://docs.rs/sbv2_core"
members = ["sbv2_api", "sbv2_core", "sbv2_bindings", "sbv2_wasm"]
[workspace.dependencies]
anyhow = "1.0.96"
anyhow = "1.0.86"
dotenvy = "0.15.7"
env_logger = "0.11.6"
env_logger = "0.11.5"
ndarray = "0.16.1"
once_cell = "1.20.3"
once_cell = "1.19.0"
[profile.release]
strip = true
opt-level = "z"
lto = true
codegen-units = 1
debug = false
strip = true

40
README.md
View File

@@ -1,19 +1,7 @@
# SBV2-API
> [!CAUTION]
> 本バージョンはアルファ版です。
>
> 安定版を利用したい場合は[こちら](https://github.com/neodyland/sbv2-api/tree/v0.1.x)をご覧ください。
> [!CAUTION]
> オプションの辞書はLGPLです。
>
> オプションの辞書を使用する場合、バイナリの内部の辞書部分について、LGPLが適用されます。
> [!NOTE]
> このレポジトリはメンテナンスの都合上、[tuna2134](https:://github.com/tuna2134)氏の所属する[Neodyland](https://neody.land/)へとリポジトリ所在地を移動しました。
>
> 引き続きtuna2134氏がメインメンテナとして管理しています。
## 注意：本バージョンはアルファ版です。
安定版を利用したい場合は[こちら](https://github.com/tuna2134/sbv2-api/tree/v0.1.x)をご覧ください。
## プログラミングに詳しくない方向け
@@ -29,7 +17,7 @@ JP-Extra しか対応していません。(基本的に対応する予定もあ
## 変換方法
[こちら](https://github.com/neodyland/sbv2-api/tree/main/scripts/convert)を参照してください。
[こちら](https://github.com/tuna2134/sbv2-api/tree/main/convert)を参照してください。
## Todo
@@ -41,27 +29,23 @@ JP-Extra しか対応していません。(基本的に対応する予定もあ
- [x] GPU 対応(CUDA)
- [x] GPU 対応(DirectML)
- [x] GPU 対応(CoreML)
- [x] WASM 変換
- [ ] WASM 変換(依存ライブラリの関係により現在は不可)
- [x] arm64のdockerサポート
- [x] aivis形式のサポート
- [ ] MeCabを利用する
## 構造説明
- `crates/sbv2_api` - 推論用 REST API
- `crates/sbv2_core` - 推論コア部分
- `scripts/docker` - docker ビルドスクリプト
- `scripts/convert` - onnx, sbv2フォーマットへの変換スクリプト
- `sbv2_api` - 推論用 REST API
- `sbv2_core` - 推論コア部分
- `docker` - docker ビルドスクリプト
- `convert` - onnx, sbv2フォーマットへの変換スクリプト
## プログラミングある程度できる人向けREST API起動方法
### models をインストール
https://huggingface.co/neody/sbv2-api-assets/tree/main/deberta
から`tokenizer.json`,`debert.onnx`
https://huggingface.co/neody/sbv2-api-assets/tree/main/model
から`tsukuyomi.sbv2`
を models フォルダに配置
https://huggingface.co/googlefan/sbv2_onnx_models/tree/main
の`tokenizer.json`,`debert.onnx`,`tsukuyomi.sbv2`を models フォルダに配置
### .env ファイルの作成
@@ -130,10 +114,8 @@ curl http://localhost:3000/models
- `ADDR` `localhost:3000`などのようにサーバー起動アドレスをコントロールできます。
- `MODELS_PATH` sbv2モデルの存在するフォルダを指定できます。
- `RUST_LOG` おなじみlog levelです。
- `HOLDER_MAX_LOADED_MODElS` RAMにロードされるモデルの最大数を指定します。
## 謝辞
- [litagin02/Style-Bert-VITS2](https://github.com/litagin02/Style-Bert-VITS2) - このコードを書くにあたり、ベースとなる部分を参考にさせていただきました。
- [litagin02/Style-Bert-VITS2](https://github.com/litagin02/Style-Bert-VITS2) - このコードの書くにあたり、ベースとなる部分を参考にさせていただきました。
- [Googlefan](https://github.com/Googlefan256) - 彼にモデルを ONNX ヘ変換および効率化をする方法を教わりました。
- [Aivis Project](https://github.com/Aivis-Project/AivisSpeech-Engine) - 辞書部分

14
content.txt Normal file
View File

@@ -0,0 +1,14 @@
悪徳貴族として名高いヴェレット家の長男――オウガ・ヴェレットは転生者である。
ブラック企業に勤め、過労死した彼には一つの夢があった。
「可愛いハーレム作って、美味い物を食べる。領民の税金で楽して好き放題な生活を送ってみせる！」
素晴らしき異世界ライフを夢見た彼は実現へ向けて、努力を始めた。
ハーレムを築くためにいじめられてる平民の子を助けて恩を売ってやったり。
労働力を手に入れるために多くの孤児を雇って教育したり。
反乱を起きても鎮圧できるように魔法学院へ通って魔法を極める。
「クックック……！　順調、順調！　未来は明るいなぁ！」
――オウガはまだ知らない。
楽な生活を送るためにしてきたことが評価され、世間から『聖者』様として呼ばれる未来を。

0
scripts/convert/README.md → convert/README.md
View File

6
scripts/convert/convert_deberta.py → convert/convert_deberta.py
View File

@@ -5,7 +5,6 @@ from transformers import AutoModelForMaskedLM, AutoTokenizer
import torch
from torch import nn
from argparse import ArgumentParser
import os
parser = ArgumentParser()
parser.add_argument("--model", default="ku-nlp/deberta-v2-large-japanese-char-wwm")
@@ -16,7 +15,7 @@ bert_models.load_tokenizer(Languages.JP, model_name)
tokenizer = bert_models.load_tokenizer(Languages.JP)
converter = BertConverter(tokenizer)
tokenizer = converter.converted()
tokenizer.save("../../models/tokenizer.json")
tokenizer.save("../models/tokenizer.json")
class ORTDeberta(nn.Module):
@@ -43,10 +42,9 @@ inputs = AutoTokenizer.from_pretrained(model_name)(
torch.onnx.export(
model,
(inputs["input_ids"], inputs["token_type_ids"], inputs["attention_mask"]),
"../../models/deberta.onnx",
"../models/deberta.onnx",
input_names=["input_ids", "token_type_ids", "attention_mask"],
output_names=["output"],
verbose=True,
dynamic_axes={"input_ids": {1: "batch_size"}, "attention_mask": {1: "batch_size"}},
)
os.system("onnxsim ../../models/deberta.onnx ../../models/deberta.onnx")

26
scripts/convert/convert_model.py → convert/convert_model.py
View File

@@ -36,7 +36,7 @@ data = array.tolist()
hyper_parameters = HyperParameters.load_from_json(config_file)
out_name = hyper_parameters.model_name
with open(f"../../models/style_vectors_{out_name}.json", "w") as f:
with open(f"../models/style_vectors_{out_name}.json", "w") as f:
json.dump(
{
"data": data,
@@ -94,7 +94,7 @@ model = get_net_g(
)
def forward(x, x_len, sid, tone, lang, bert, style, length_scale, sdp_ratio, noise_scale, noise_scale_w):
def forward(x, x_len, sid, tone, lang, bert, style, length_scale, sdp_ratio):
return model.infer(
x,
x_len,
@@ -105,8 +105,6 @@ def forward(x, x_len, sid, tone, lang, bert, style, length_scale, sdp_ratio, noi
style,
sdp_ratio=sdp_ratio,
length_scale=length_scale,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
)
@@ -124,10 +122,8 @@ torch.onnx.export(
style_vec_tensor,
torch.tensor(1.0),
torch.tensor(0.0),
torch.tensor(0.6777),
torch.tensor(0.8),
),
f"../../models/model_{out_name}.onnx",
f"../models/model_{out_name}.onnx",
verbose=True,
dynamic_axes={
"x_tst": {0: "batch_size", 1: "x_tst_max_length"},
@@ -148,16 +144,14 @@ torch.onnx.export(
"style_vec",
"length_scale",
"sdp_ratio",
"noise_scale",
"noise_scale_w"
],
output_names=["output"],
)
os.system(f"onnxsim ../../models/model_{out_name}.onnx ../../models/model_{out_name}.onnx")
onnxfile = open(f"../../models/model_{out_name}.onnx", "rb").read()
stylefile = open(f"../../models/style_vectors_{out_name}.json", "rb").read()
os.system(f"onnxsim ../models/model_{out_name}.onnx ../models/model_{out_name}.onnx")
onnxfile = open(f"../models/model_{out_name}.onnx", "rb").read()
stylefile = open(f"../models/style_vectors_{out_name}.json", "rb").read()
version = bytes("1", "utf8")
with taropen(f"../../models/tmp_{out_name}.sbv2tar", "w") as w:
with taropen(f"../models/tmp_{out_name}.sbv2tar", "w") as w:
def add_tar(f, b):
t = TarInfo(f)
@@ -167,9 +161,9 @@ with taropen(f"../../models/tmp_{out_name}.sbv2tar", "w") as w:
add_tar("version.txt", version)
add_tar("model.onnx", onnxfile)
add_tar("style_vectors.json", stylefile)
open(f"../../models/{out_name}.sbv2", "wb").write(
open(f"../models/{out_name}.sbv2", "wb").write(
ZstdCompressor(threads=-1, level=22).compress(
open(f"../../models/tmp_{out_name}.sbv2tar", "rb").read()
open(f"../models/tmp_{out_name}.sbv2tar", "rb").read()
)
)
os.unlink(f"../../models/tmp_{out_name}.sbv2tar")
os.unlink(f"../models/tmp_{out_name}.sbv2tar")

0
scripts/convert/requirements.txt → convert/requirements.txt
View File

24
crates/sbv2_bindings/Cargo.toml
View File

@@ -1,24 +0,0 @@
[package]
name = "sbv2_bindings"
version.workspace = true
edition.workspace = true
description.workspace = true
readme.workspace = true
repository.workspace = true
documentation.workspace = true
license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "sbv2_bindings"
crate-type = ["cdylib"]
[dependencies]
anyhow.workspace = true
ndarray.workspace = true
pyo3 = { version = "0.24.0", features = ["anyhow"] }
sbv2_core = { path = "../sbv2_core", features = ["std"], default-features = false }
[features]
agpl_dict = ["sbv2_core/agpl_dict"]
default = ["agpl_dict"]

25
crates/sbv2_core/build.rs
View File

@@ -1,25 +0,0 @@
use dirs::home_dir;
use std::env;
use std::fs;
use std::io::copy;
use std::path::PathBuf;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let static_path = home_dir().unwrap().join(".cache/sbv2/all.bin");
let out_path = PathBuf::from(&env::var("OUT_DIR").unwrap()).join("all.bin");
println!("cargo:rerun-if-changed=build.rs");
if static_path.exists() {
if fs::hard_link(&static_path, &out_path).is_err() {
fs::copy(static_path, out_path).unwrap();
};
} else {
println!("cargo:warning=Downloading dictionary file...");
let mut response =
ureq::get("https://huggingface.co/neody/sbv2-api-assets/resolve/main/dic/all.bin")
.call()?;
let mut response = response.body_mut().as_reader();
let mut file = fs::File::create(&out_path)?;
copy(&mut response, &mut file)?;
}
Ok(())
}

24
crates/sbv2_core/src/bert.rs
View File

@@ -1,24 +0,0 @@
use crate::error::Result;
use ndarray::{Array2, Ix2};
use ort::session::Session;
use ort::value::TensorRef;
pub fn predict(
session: &mut Session,
token_ids: Vec<i64>,
attention_masks: Vec<i64>,
) -> Result<Array2<f32>> {
let outputs = session.run(
ort::inputs! {
"input_ids" => TensorRef::from_array_view((vec![1, token_ids.len() as i64], token_ids.as_slice()))?,
"attention_mask" => TensorRef::from_array_view((vec![1, attention_masks.len() as i64], attention_masks.as_slice()))?,
}
)?;
let output = outputs["output"]
.try_extract_tensor::<f32>()?
.into_dimensionality::<Ix2>()?
.to_owned();
Ok(output)
}

48
crates/sbv2_core/src/main.rs
View File

@@ -1,48 +0,0 @@
use std::env;
use std::fs;
#[cfg(feature = "std")]
fn main_inner() -> anyhow::Result<()> {
use sbv2_core::tts;
dotenvy::dotenv_override().ok();
env_logger::init();
let text = "今日の天気は快晴です。";
let ident = "aaa";
let mut tts_holder = tts::TTSModelHolder::new(
&fs::read(env::var("BERT_MODEL_PATH")?)?,
&fs::read(env::var("TOKENIZER_PATH")?)?,
env::var("HOLDER_MAX_LOADED_MODElS")
.ok()
.and_then(|x| x.parse().ok()),
)?;
let mp = env::var("MODEL_PATH")?;
let b = fs::read(&mp)?;
#[cfg(not(feature = "aivmx"))]
{
tts_holder.load_sbv2file(ident, b)?;
}
#[cfg(feature = "aivmx")]
{
if mp.ends_with(".sbv2") {
tts_holder.load_sbv2file(ident, b)?;
} else {
tts_holder.load_aivmx(ident, b)?;
}
}
let audio = tts_holder.easy_synthesize(ident, text, 0, 0, tts::SynthesizeOptions::default())?;
fs::write("output.wav", audio)?;
Ok(())
}
#[cfg(not(feature = "std"))]
fn main_inner() -> anyhow::Result<()> {
Ok(())
}
fn main() {
if let Err(e) = main_inner() {
println!("Error: {e}");
}
}

111
crates/sbv2_core/src/model.rs
View File

@@ -1,111 +0,0 @@
use crate::error::Result;
use ndarray::{array, Array1, Array2, Array3, Axis, Ix3};
use ort::session::{builder::GraphOptimizationLevel, Session};
#[allow(clippy::vec_init_then_push, unused_variables)]
pub fn load_model<P: AsRef<[u8]>>(model_file: P, bert: bool) -> Result<Session> {
let mut exp = Vec::new();
#[cfg(feature = "tensorrt")]
{
if bert {
exp.push(
ort::execution_providers::TensorRTExecutionProvider::default()
.with_fp16(true)
.with_profile_min_shapes("input_ids:1x1,attention_mask:1x1")
.with_profile_max_shapes("input_ids:1x100,attention_mask:1x100")
.with_profile_opt_shapes("input_ids:1x25,attention_mask:1x25")
.build(),
);
}
}
#[cfg(feature = "cuda")]
{
#[allow(unused_mut)]
let mut cuda = ort::execution_providers::CUDAExecutionProvider::default()
.with_conv_algorithm_search(
ort::execution_providers::cuda::CUDAExecutionProviderCuDNNConvAlgoSearch::Default,
);
#[cfg(feature = "cuda_tf32")]
{
cuda = cuda.with_tf32(true);
}
exp.push(cuda.build());
}
#[cfg(feature = "directml")]
{
exp.push(ort::execution_providers::DirectMLExecutionProvider::default().build());
}
#[cfg(feature = "coreml")]
{
exp.push(ort::execution_providers::CoreMLExecutionProvider::default().build());
}
exp.push(ort::execution_providers::CPUExecutionProvider::default().build());
Ok(Session::builder()?
.with_execution_providers(exp)?
.with_optimization_level(GraphOptimizationLevel::Level3)?
.with_intra_threads(num_cpus::get_physical())?
.with_parallel_execution(true)?
.with_inter_threads(num_cpus::get_physical())?
.commit_from_memory(model_file.as_ref())?)
}
#[allow(clippy::too_many_arguments)]
pub fn synthesize(
session: &mut Session,
bert_ori: Array2<f32>,
x_tst: Array1<i64>,
mut spk_ids: Array1<i64>,
tones: Array1<i64>,
lang_ids: Array1<i64>,
style_vector: Array1<f32>,
sdp_ratio: f32,
length_scale: f32,
noise_scale: f32,
noise_scale_w: f32,
) -> Result<Array3<f32>> {
let bert_ori = bert_ori.insert_axis(Axis(0));
let bert_ori = bert_ori.as_standard_layout();
let bert = ort::value::TensorRef::from_array_view(&bert_ori)?;
let mut x_tst_lengths = array![x_tst.shape()[0] as i64];
let x_tst_lengths = ort::value::TensorRef::from_array_view(&mut x_tst_lengths)?;
let mut x_tst = x_tst.insert_axis(Axis(0));
let x_tst = ort::value::TensorRef::from_array_view(&mut x_tst)?;
let mut lang_ids = lang_ids.insert_axis(Axis(0));
let lang_ids = ort::value::TensorRef::from_array_view(&mut lang_ids)?;
let mut tones = tones.insert_axis(Axis(0));
let tones = ort::value::TensorRef::from_array_view(&mut tones)?;
let mut style_vector = style_vector.insert_axis(Axis(0));
let style_vector = ort::value::TensorRef::from_array_view(&mut style_vector)?;
let sid = ort::value::TensorRef::from_array_view(&mut spk_ids)?;
let sdp_ratio = vec![sdp_ratio];
let sdp_ratio = ort::value::TensorRef::from_array_view((vec![1_i64], sdp_ratio.as_slice()))?;
let length_scale = vec![length_scale];
let length_scale =
ort::value::TensorRef::from_array_view((vec![1_i64], length_scale.as_slice()))?;
let noise_scale = vec![noise_scale];
let noise_scale =
ort::value::TensorRef::from_array_view((vec![1_i64], noise_scale.as_slice()))?;
let noise_scale_w = vec![noise_scale_w];
let noise_scale_w =
ort::value::TensorRef::from_array_view((vec![1_i64], noise_scale_w.as_slice()))?;
let outputs = session.run(ort::inputs! {
"x_tst" => x_tst,
"x_tst_lengths" => x_tst_lengths,
"sid" => sid,
"tones" => tones,
"language" => lang_ids,
"bert" => bert,
"style_vec" => style_vector,
"sdp_ratio" => sdp_ratio,
"length_scale" => length_scale,
"noise_scale" => noise_scale,
"noise_scale_w" => noise_scale_w,
})?;
let audio_array = outputs["output"]
.try_extract_tensor::<f32>()?
.into_dimensionality::<Ix3>()?
.to_owned();
Ok(audio_array)
}

465
crates/sbv2_core/src/tts.rs
View File

@@ -1,465 +0,0 @@
use crate::error::{Error, Result};
use crate::{jtalk, model, style, tokenizer, tts_util};
#[cfg(feature = "aivmx")]
use base64::prelude::{Engine as _, BASE64_STANDARD};
#[cfg(feature = "aivmx")]
use ndarray::ShapeBuilder;
use ndarray::{concatenate, Array1, Array2, Array3, Axis};
use ort::session::Session;
#[cfg(feature = "aivmx")]
use std::io::Cursor;
use tokenizers::Tokenizer;
#[derive(PartialEq, Eq, Clone)]
pub struct TTSIdent(String);
impl std::fmt::Display for TTSIdent {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)?;
Ok(())
}
}
impl<S> From<S> for TTSIdent
where
S: AsRef<str>,
{
fn from(value: S) -> Self {
TTSIdent(value.as_ref().to_string())
}
}
pub struct TTSModel {
vits2: Option<Session>,
style_vectors: Array2<f32>,
ident: TTSIdent,
bytes: Option<Vec<u8>>,
}
/// High-level Style-Bert-VITS2's API
pub struct TTSModelHolder {
tokenizer: Tokenizer,
bert: Session,
models: Vec<TTSModel>,
pub jtalk: jtalk::JTalk,
max_loaded_models: Option<usize>,
}
impl TTSModelHolder {
/// Initialize a new TTSModelHolder
///
/// # Examples
///
/// ```rs
/// let mut tts_holder = TTSModelHolder::new(std::fs::read("deberta.onnx")?, std::fs::read("tokenizer.json")?, None)?;
/// ```
pub fn new<P: AsRef<[u8]>>(
bert_model_bytes: P,
tokenizer_bytes: P,
max_loaded_models: Option<usize>,
) -> Result<Self> {
let bert = model::load_model(bert_model_bytes, true)?;
let jtalk = jtalk::JTalk::new()?;
let tokenizer = tokenizer::get_tokenizer(tokenizer_bytes)?;
Ok(TTSModelHolder {
bert,
models: vec![],
jtalk,
tokenizer,
max_loaded_models,
})
}
/// Return a list of model names
pub fn models(&self) -> Vec<String> {
self.models.iter().map(|m| m.ident.to_string()).collect()
}
#[cfg(feature = "aivmx")]
pub fn load_aivmx<I: Into<TTSIdent>, P: AsRef<[u8]>>(
&mut self,
ident: I,
aivmx_bytes: P,
) -> Result<()> {
let ident = ident.into();
if self.find_model(ident.clone()).is_err() {
let mut load = true;
if let Some(max) = self.max_loaded_models {
if self.models.iter().filter(|x| x.vits2.is_some()).count() >= max {
load = false;
}
}
let model = model::load_model(&aivmx_bytes, false)?;
let metadata = model.metadata()?;
if let Some(aivm_style_vectors) = metadata.custom("aivm_style_vectors")? {
let aivm_style_vectors = BASE64_STANDARD.decode(aivm_style_vectors)?;
let style_vectors = Cursor::new(&aivm_style_vectors);
let reader = npyz::NpyFile::new(style_vectors)?;
let style_vectors = {
let shape = reader.shape().to_vec();
let order = reader.order();
let data = reader.into_vec::<f32>()?;
let shape = match shape[..] {
[i1, i2] => [i1 as usize, i2 as usize],
_ => panic!("expected 2D array"),
};
let true_shape = shape.set_f(order == npyz::Order::Fortran);
ndarray::Array2::from_shape_vec(true_shape, data)?
};
drop(metadata);
self.models.push(TTSModel {
vits2: if load { Some(model) } else { None },
bytes: if self.max_loaded_models.is_some() {
Some(aivmx_bytes.as_ref().to_vec())
} else {
None
},
ident,
style_vectors,
})
}
}
Ok(())
}
/// Load a .sbv2 file binary
///
/// # Examples
///
/// ```rs
/// tts_holder.load_sbv2file("tsukuyomi", std::fs::read("tsukuyomi.sbv2")?)?;
/// ```
pub fn load_sbv2file<I: Into<TTSIdent>, P: AsRef<[u8]>>(
&mut self,
ident: I,
sbv2_bytes: P,
) -> Result<()> {
let (style_vectors, vits2) = crate::sbv2file::parse_sbv2file(sbv2_bytes)?;
self.load(ident, style_vectors, vits2)?;
Ok(())
}
/// Load a style vector and onnx model binary
///
/// # Examples
///
/// ```rs
/// tts_holder.load("tsukuyomi", std::fs::read("style_vectors.json")?, std::fs::read("model.onnx")?)?;
/// ```
pub fn load<I: Into<TTSIdent>, P: AsRef<[u8]>>(
&mut self,
ident: I,
style_vectors_bytes: P,
vits2_bytes: P,
) -> Result<()> {
let ident = ident.into();
if self.find_model(ident.clone()).is_err() {
let mut load = true;
if let Some(max) = self.max_loaded_models {
if self.models.iter().filter(|x| x.vits2.is_some()).count() >= max {
load = false;
}
}
self.models.push(TTSModel {
vits2: if load {
Some(model::load_model(&vits2_bytes, false)?)
} else {
None
},
style_vectors: style::load_style(style_vectors_bytes)?,
ident,
bytes: if self.max_loaded_models.is_some() {
Some(vits2_bytes.as_ref().to_vec())
} else {
None
},
})
}
Ok(())
}
/// Unload a model
pub fn unload<I: Into<TTSIdent>>(&mut self, ident: I) -> bool {
let ident = ident.into();
if let Some((i, _)) = self
.models
.iter()
.enumerate()
.find(|(_, m)| m.ident == ident)
{
self.models.remove(i);
true
} else {
false
}
}
/// Parse text and return the input for synthesize
///
/// # Note
/// This function is for low-level usage, use `easy_synthesize` for high-level usage.
#[allow(clippy::type_complexity)]
pub fn parse_text(
&mut self,
text: &str,
) -> Result<(Array2<f32>, Array1<i64>, Array1<i64>, Array1<i64>)> {
crate::tts_util::parse_text_blocking(
text,
None,
&self.jtalk,
&self.tokenizer,
|token_ids, attention_masks| {
crate::bert::predict(&mut self.bert, token_ids, attention_masks)
},
)
}
pub fn parse_text_neo(
&mut self,
text: String,
given_tones: Option<Vec<i32>>,
) -> Result<(Array2<f32>, Array1<i64>, Array1<i64>, Array1<i64>)> {
crate::tts_util::parse_text_blocking(
&text,
given_tones,
&self.jtalk,
&self.tokenizer,
|token_ids, attention_masks| {
crate::bert::predict(&mut self.bert, token_ids, attention_masks)
},
)
}
fn find_model<I: Into<TTSIdent>>(&mut self, ident: I) -> Result<&mut TTSModel> {
let ident = ident.into();
self.models
.iter_mut()
.find(|m| m.ident == ident)
.ok_or(Error::ModelNotFoundError(ident.to_string()))
}
fn find_and_load_model<I: Into<TTSIdent>>(&mut self, ident: I) -> Result<bool> {
let ident = ident.into();
let (bytes, style_vectors) = {
let model = self
.models
.iter()
.find(|m| m.ident == ident)
.ok_or(Error::ModelNotFoundError(ident.to_string()))?;
if model.vits2.is_some() {
return Ok(true);
}
(model.bytes.clone().unwrap(), model.style_vectors.clone())
};
self.unload(ident.clone());
let s = model::load_model(&bytes, false)?;
if let Some(max) = self.max_loaded_models {
if self.models.iter().filter(|x| x.vits2.is_some()).count() >= max {
self.unload(self.models.first().unwrap().ident.clone());
}
}
self.models.push(TTSModel {
bytes: Some(bytes.to_vec()),
vits2: Some(s),
style_vectors,
ident: ident.clone(),
});
let model = self
.models
.iter()
.find(|m| m.ident == ident)
.ok_or(Error::ModelNotFoundError(ident.to_string()))?;
if model.vits2.is_some() {
return Ok(true);
}
Err(Error::ModelNotFoundError(ident.to_string()))
}
/// Get style vector by style id and weight
///
/// # Note
/// This function is for low-level usage, use `easy_synthesize` for high-level usage.
pub fn get_style_vector<I: Into<TTSIdent>>(
&mut self,
ident: I,
style_id: i32,
weight: f32,
) -> Result<Array1<f32>> {
style::get_style_vector(&self.find_model(ident)?.style_vectors, style_id, weight)
}
/// Synthesize text to audio
///
/// # Examples
///
/// ```rs
/// let audio = tts_holder.easy_synthesize("tsukuyomi", "こんにちは", 0, SynthesizeOptions::default())?;
/// ```
pub fn easy_synthesize<I: Into<TTSIdent> + Copy>(
&mut self,
ident: I,
text: &str,
style_id: i32,
speaker_id: i64,
options: SynthesizeOptions,
) -> Result<Vec<u8>> {
self.find_and_load_model(ident)?;
let style_vector = self.get_style_vector(ident, style_id, options.style_weight)?;
let audio_array = if options.split_sentences {
let texts: Vec<&str> = text.split('\n').collect();
let mut audios = vec![];
for (i, t) in texts.iter().enumerate() {
if t.is_empty() {
continue;
}
let (bert_ori, phones, tones, lang_ids) = self.parse_text(t)?;
let vits2 = self
.find_model(ident)?
.vits2
.as_mut()
.ok_or(Error::ModelNotFoundError(ident.into().to_string()))?;
let audio = model::synthesize(
vits2,
bert_ori.to_owned(),
phones,
Array1::from_vec(vec![speaker_id]),
tones,
lang_ids,
style_vector.clone(),
options.sdp_ratio,
options.length_scale,
0.677,
0.8,
)?;
audios.push(audio.clone());
if i != texts.len() - 1 {
audios.push(Array3::zeros((1, 1, 22050)));
}
}
concatenate(
Axis(2),
&audios.iter().map(|x| x.view()).collect::<Vec<_>>(),
)?
} else {
let (bert_ori, phones, tones, lang_ids) = self.parse_text(text)?;
let vits2 = self
.find_model(ident)?
.vits2
.as_mut()
.ok_or(Error::ModelNotFoundError(ident.into().to_string()))?;
model::synthesize(
vits2,
bert_ori.to_owned(),
phones,
Array1::from_vec(vec![speaker_id]),
tones,
lang_ids,
style_vector,
options.sdp_ratio,
options.length_scale,
0.677,
0.8,
)?
};
tts_util::array_to_vec(audio_array)
}
pub fn easy_synthesize_neo<I: Into<TTSIdent> + Copy>(
&mut self,
ident: I,
text: &str,
given_tones: Option<Vec<i32>>,
style_id: i32,
speaker_id: i64,
options: SynthesizeOptions,
) -> Result<Vec<u8>> {
self.find_and_load_model(ident)?;
let style_vector = self.get_style_vector(ident, style_id, options.style_weight)?;
let audio_array = if options.split_sentences {
let texts: Vec<&str> = text.split('\n').collect();
let mut audios = vec![];
for (i, t) in texts.iter().enumerate() {
if t.is_empty() {
continue;
}
let (bert_ori, phones, tones, lang_ids) =
self.parse_text_neo(t.to_string(), given_tones.clone())?;
let vits2 = self
.find_model(ident)?
.vits2
.as_mut()
.ok_or(Error::ModelNotFoundError(ident.into().to_string()))?;
let audio = model::synthesize(
vits2,
bert_ori.to_owned(),
phones,
Array1::from_vec(vec![speaker_id]),
tones,
lang_ids,
style_vector.clone(),
options.sdp_ratio,
options.length_scale,
0.677,
0.8,
)?;
audios.push(audio.clone());
if i != texts.len() - 1 {
audios.push(Array3::zeros((1, 1, 22050)));
}
}
concatenate(
Axis(2),
&audios.iter().map(|x| x.view()).collect::<Vec<_>>(),
)?
} else {
let (bert_ori, phones, tones, lang_ids) = self.parse_text(text)?;
let vits2 = self
.find_model(ident)?
.vits2
.as_mut()
.ok_or(Error::ModelNotFoundError(ident.into().to_string()))?;
model::synthesize(
vits2,
bert_ori.to_owned(),
phones,
Array1::from_vec(vec![speaker_id]),
tones,
lang_ids,
style_vector,
options.sdp_ratio,
options.length_scale,
0.677,
0.8,
)?
};
tts_util::array_to_vec(audio_array)
}
}
/// Synthesize options
///
/// # Fields
/// - `sdp_ratio`: SDP ratio
/// - `length_scale`: Length scale
/// - `style_weight`: Style weight
/// - `split_sentences`: Split sentences
pub struct SynthesizeOptions {
pub sdp_ratio: f32,
pub length_scale: f32,
pub style_weight: f32,
pub split_sentences: bool,
}
impl Default for SynthesizeOptions {
fn default() -> Self {
SynthesizeOptions {
sdp_ratio: 0.0,
length_scale: 1.0,
style_weight: 1.0,
split_sentences: true,
}
}
}

19
crates/sbv2_editor/Cargo.toml
View File

@@ -1,19 +0,0 @@
[package]
name = "sbv2_editor"
version.workspace = true
edition.workspace = true
description.workspace = true
license.workspace = true
readme.workspace = true
repository.workspace = true
documentation.workspace = true
[dependencies]
anyhow.workspace = true
axum = "0.8.1"
dotenvy.workspace = true
env_logger.workspace = true
log = "0.4.27"
sbv2_core = { version = "0.2.0-alpha6", path = "../sbv2_core", features = ["aivmx"] }
serde = { version = "1.0.219", features = ["derive"] }
tokio = { version = "1.44.1", features = ["full"] }

2
crates/sbv2_editor/README.md
View File

@@ -1,2 +0,0 @@
# sbv2-voicevox
sbv2-apiをvoicevox化します。

226
crates/sbv2_editor/query2.json
View File

@@ -1,226 +0,0 @@
{
"accent_phrases": [
{
"moras": [
{
"text": "コ",
"consonant": "k",
"consonant_length": 0.10002632439136505,
"vowel": "o",
"vowel_length": 0.15740256011486053,
"pitch": 5.749961853027344
},
{
"text": "ン",
"consonant": null,
"consonant_length": null,
"vowel": "N",
"vowel_length": 0.08265873789787292,
"pitch": 5.89122200012207
},
{
"text": "ニ",
"consonant": "n",
"consonant_length": 0.03657080978155136,
"vowel": "i",
"vowel_length": 0.1175866425037384,
"pitch": 5.969866752624512
},
{
"text": "チ",
"consonant": "ch",
"consonant_length": 0.09005842357873917,
"vowel": "i",
"vowel_length": 0.08666137605905533,
"pitch": 5.958892822265625
},
{
"text": "ワ",
"consonant": "w",
"consonant_length": 0.07833231985569,
"vowel": "a",
"vowel_length": 0.21250136196613312,
"pitch": 5.949411392211914
}
],
"accent": 5,
"pause_mora": {
"text": "、",
"consonant": null,
"consonant_length": null,
"vowel": "pau",
"vowel_length": 0.4723339378833771,
"pitch": 0.0
},
"is_interrogative": false
},
{
"moras": [
{
"text": "オ",
"consonant": null,
"consonant_length": null,
"vowel": "o",
"vowel_length": 0.22004225850105286,
"pitch": 5.6870927810668945
},
{
"text": "ン",
"consonant": null,
"consonant_length": null,
"vowel": "N",
"vowel_length": 0.09161105751991272,
"pitch": 5.93472957611084
},
{
"text": "セ",
"consonant": "s",
"consonant_length": 0.08924821764230728,
"vowel": "e",
"vowel_length": 0.14142127335071564,
"pitch": 6.121850490570068
},
{
"text": "エ",
"consonant": null,
"consonant_length": null,
"vowel": "e",
"vowel_length": 0.10636933892965317,
"pitch": 6.157896041870117
},
{
"text": "ゴ",
"consonant": "g",
"consonant_length": 0.07600915431976318,
"vowel": "o",
"vowel_length": 0.09598273783922195,
"pitch": 6.188933849334717
},
{
"text": "オ",
"consonant": null,
"consonant_length": null,
"vowel": "o",
"vowel_length": 0.1079121008515358,
"pitch": 6.235202789306641
},
{
"text": "セ",
"consonant": "s",
"consonant_length": 0.09591838717460632,
"vowel": "e",
"vowel_length": 0.10286372154951096,
"pitch": 6.153214454650879
},
{
"text": "エ",
"consonant": null,
"consonant_length": null,
"vowel": "e",
"vowel_length": 0.08992656320333481,
"pitch": 6.02571439743042
},
{
"text": "ノ",
"consonant": "n",
"consonant_length": 0.05660202354192734,
"vowel": "o",
"vowel_length": 0.09676017612218857,
"pitch": 5.711844444274902
}
],
"accent": 5,
"pause_mora": null,
"is_interrogative": false
},
{
"moras": [
{
"text": "セ",
"consonant": "s",
"consonant_length": 0.07805486768484116,
"vowel": "e",
"vowel_length": 0.09617523103952408,
"pitch": 5.774399280548096
},
{
"text": "カ",
"consonant": "k",
"consonant_length": 0.06712044775485992,
"vowel": "a",
"vowel_length": 0.148829385638237,
"pitch": 6.063965797424316
},
{
"text": "イ",
"consonant": null,
"consonant_length": null,
"vowel": "i",
"vowel_length": 0.11061104387044907,
"pitch": 6.040698051452637
},
{
"text": "エ",
"consonant": null,
"consonant_length": null,
"vowel": "e",
"vowel_length": 0.13046696782112122,
"pitch": 5.806027889251709
}
],
"accent": 1,
"pause_mora": null,
"is_interrogative": false
},
{
"moras": [
{
"text": "ヨ",
"consonant": "y",
"consonant_length": 0.07194744795560837,
"vowel": "o",
"vowel_length": 0.08622600883245468,
"pitch": 5.694094657897949
},
{
"text": "オ",
"consonant": null,
"consonant_length": null,
"vowel": "o",
"vowel_length": 0.10635452717542648,
"pitch": 5.787222385406494
},
{
"text": "コ",
"consonant": "k",
"consonant_length": 0.07077334076166153,
"vowel": "o",
"vowel_length": 0.09248624742031097,
"pitch": 5.793357849121094
},
{
"text": "ソ",
"consonant": "s",
"consonant_length": 0.08705667406320572,
"vowel": "o",
"vowel_length": 0.2238258570432663,
"pitch": 5.643765449523926
}
],
"accent": 1,
"pause_mora": null,
"is_interrogative": false
}
],
"speedScale": 1.0,
"pitchScale": 0.0,
"intonationScale": 1.0,
"volumeScale": 1.0,
"prePhonemeLength": 0.1,
"postPhonemeLength": 0.1,
"pauseLength": null,
"pauseLengthScale": 1.0,
"outputSamplingRate": 24000,
"outputStereo": false,
"kana": "コンニチワ'、オンセエゴ'オセエノ/セ'カイエ/ヨ'オコソ"
}

27
crates/sbv2_editor/src/error.rs
View File

@@ -1,27 +0,0 @@
use axum::{
http::StatusCode,
response::{IntoResponse, Response},
};
pub type AppResult<T> = std::result::Result<T, AppError>;
pub struct AppError(anyhow::Error);
impl IntoResponse for AppError {
fn into_response(self) -> Response {
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("Something went wrong: {}", self.0),
)
.into_response()
}
}
impl<E> From<E> for AppError
where
E: Into<anyhow::Error>,
{
fn from(err: E) -> Self {
Self(err.into())
}
}

197
crates/sbv2_editor/src/main.rs
View File

@@ -1,197 +0,0 @@
use axum::extract::State;
use axum::{
extract::Query,
http::header::CONTENT_TYPE,
response::IntoResponse,
routing::{get, post},
Json, Router,
};
use sbv2_core::tts_util::kata_tone2phone_tone;
use sbv2_core::{
tts::{SynthesizeOptions, TTSModelHolder},
tts_util::preprocess_parse_text,
};
use serde::{Deserialize, Serialize};
use tokio::{fs, net::TcpListener, sync::Mutex};
use std::env;
use std::sync::Arc;
use error::AppResult;
mod error;
#[derive(Deserialize)]
struct RequestCreateAudioQuery {
text: String,
}
#[derive(Serialize, Deserialize)]
struct AudioQuery {
kana: String,
tone: i32,
}
#[derive(Serialize)]
struct ResponseCreateAudioQuery {
audio_query: Vec<AudioQuery>,
text: String,
}
async fn create_audio_query(
State(state): State<AppState>,
Query(request): Query<RequestCreateAudioQuery>,
) -> AppResult<impl IntoResponse> {
let (text, process) = {
let tts_model = state.tts_model.lock().await;
preprocess_parse_text(&request.text, &tts_model.jtalk)?
};
let kana_tone_list = process.g2kana_tone()?;
let audio_query = kana_tone_list
.iter()
.map(|(kana, tone)| AudioQuery {
kana: kana.clone(),
tone: *tone,
})
.collect::<Vec<_>>();
Ok(Json(ResponseCreateAudioQuery { audio_query, text }))
}
#[derive(Deserialize)]
pub struct RequestSynthesis {
text: String,
speaker_id: i64,
sdp_ratio: f32,
length_scale: f32,
style_id: i32,
audio_query: Vec<AudioQuery>,
ident: String,
}
async fn synthesis(
State(state): State<AppState>,
Json(request): Json<RequestSynthesis>,
) -> AppResult<impl IntoResponse> {
let phone_tone = request
.audio_query
.iter()
.map(|query| (query.kana.clone(), query.tone))
.collect::<Vec<_>>();
let phone_tone = kata_tone2phone_tone(phone_tone);
let tones = phone_tone.iter().map(|(_, tone)| *tone).collect::<Vec<_>>();
let buffer = {
let mut tts_model = state.tts_model.lock().await;
tts_model.easy_synthesize_neo(
&request.ident,
&request.text,
Some(tones),
request.style_id,
request.speaker_id,
SynthesizeOptions {
sdp_ratio: request.sdp_ratio,
length_scale: request.length_scale,
..Default::default()
},
)?
};
Ok(([(CONTENT_TYPE, "audio/wav")], buffer))
}
#[derive(Clone)]
struct AppState {
tts_model: Arc<Mutex<TTSModelHolder>>,
}
impl AppState {
pub async fn new() -> anyhow::Result<Self> {
let mut tts_model = TTSModelHolder::new(
&fs::read(env::var("BERT_MODEL_PATH")?).await?,
&fs::read(env::var("TOKENIZER_PATH")?).await?,
env::var("HOLDER_MAX_LOADED_MODElS")
.ok()
.and_then(|x| x.parse().ok()),
)?;
let models = env::var("MODELS_PATH").unwrap_or("models".to_string());
let mut f = fs::read_dir(&models).await?;
let mut entries = vec![];
while let Ok(Some(e)) = f.next_entry().await {
let name = e.file_name().to_string_lossy().to_string();
if name.ends_with(".onnx") && name.starts_with("model_") {
let name_len = name.len();
let name = name.chars();
entries.push(
name.collect::<Vec<_>>()[6..name_len - 5]
.iter()
.collect::<String>(),
);
} else if name.ends_with(".sbv2") {
let entry = &name[..name.len() - 5];
log::info!("Try loading: {entry}");
let sbv2_bytes = match fs::read(format!("{models}/{entry}.sbv2")).await {
Ok(b) => b,
Err(e) => {
log::warn!("Error loading sbv2_bytes from file {entry}: {e}");
continue;
}
};
if let Err(e) = tts_model.load_sbv2file(entry, sbv2_bytes) {
log::warn!("Error loading {entry}: {e}");
};
log::info!("Loaded: {entry}");
} else if name.ends_with(".aivmx") {
let entry = &name[..name.len() - 6];
log::info!("Try loading: {entry}");
let aivmx_bytes = match fs::read(format!("{models}/{entry}.aivmx")).await {
Ok(b) => b,
Err(e) => {
log::warn!("Error loading aivmx bytes from file {entry}: {e}");
continue;
}
};
if let Err(e) = tts_model.load_aivmx(entry, aivmx_bytes) {
log::error!("Error loading {entry}: {e}");
}
log::info!("Loaded: {entry}");
}
}
for entry in entries {
log::info!("Try loading: {entry}");
let style_vectors_bytes =
match fs::read(format!("{models}/style_vectors_{entry}.json")).await {
Ok(b) => b,
Err(e) => {
log::warn!("Error loading style_vectors_bytes from file {entry}: {e}");
continue;
}
};
let vits2_bytes = match fs::read(format!("{models}/model_{entry}.onnx")).await {
Ok(b) => b,
Err(e) => {
log::warn!("Error loading vits2_bytes from file {entry}: {e}");
continue;
}
};
if let Err(e) = tts_model.load(&entry, style_vectors_bytes, vits2_bytes) {
log::warn!("Error loading {entry}: {e}");
};
log::info!("Loaded: {entry}");
}
Ok(Self {
tts_model: Arc::new(Mutex::new(tts_model)),
})
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
dotenvy::dotenv_override().ok();
env_logger::init();
let app = Router::new()
.route("/", get(|| async { "Hello, world!" }))
.route("/audio_query", get(create_audio_query))
.route("/synthesis", post(synthesis))
.with_state(AppState::new().await?);
let listener = TcpListener::bind("0.0.0.0:8080").await?;
axum::serve(listener, app).await?;
Ok(())
}

8
crates/sbv2_wasm/build.sh
View File

@@ -1,8 +0,0 @@
#!/bin/sh
wasm-pack build --target web ./crates/sbv2_wasm --release
wasm-opt -O3 -o ./crates/sbv2_wasm/pkg/sbv2_wasm_bg.wasm ./crates/sbv2_wasm/pkg/sbv2_wasm_bg.wasm
wasm-strip ./crates/sbv2_wasm/pkg/sbv2_wasm_bg.wasm
mkdir -p ./crates/sbv2_wasm/dist
cp ./crates/sbv2_wasm/pkg/sbv2_wasm_bg.wasm ./crates/sbv2_wasm/dist/sbv2_wasm_bg.wasm
cd ./crates/sbv2_wasm
pnpm build

504
crates/sbv2_wasm/pnpm-lock.yaml generated
View File

@@ -1,504 +0,0 @@
lockfileVersion: '9.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
importers:
.:
dependencies:
onnxruntime-web:
specifier: ^1.20.1
version: 1.20.1
devDependencies:
'@biomejs/biome':
specifier: ^1.9.4
version: 1.9.4
'@types/node':
specifier: ^22.13.5
version: 22.13.5
esbuild:
specifier: ^0.25.0
version: 0.25.0
typescript:
specifier: ^5.7.3
version: 5.8.3
packages:
'@biomejs/biome@1.9.4':
resolution: {integrity: sha512-1rkd7G70+o9KkTn5KLmDYXihGoTaIGO9PIIN2ZB7UJxFrWw04CZHPYiMRjYsaDvVV7hP1dYNRLxSANLaBFGpog==}
engines: {node: '>=14.21.3'}
hasBin: true
'@biomejs/cli-darwin-arm64@1.9.4':
resolution: {integrity: sha512-bFBsPWrNvkdKrNCYeAp+xo2HecOGPAy9WyNyB/jKnnedgzl4W4Hb9ZMzYNbf8dMCGmUdSavlYHiR01QaYR58cw==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [darwin]
'@biomejs/cli-darwin-x64@1.9.4':
resolution: {integrity: sha512-ngYBh/+bEedqkSevPVhLP4QfVPCpb+4BBe2p7Xs32dBgs7rh9nY2AIYUL6BgLw1JVXV8GlpKmb/hNiuIxfPfZg==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [darwin]
'@biomejs/cli-linux-arm64-musl@1.9.4':
resolution: {integrity: sha512-v665Ct9WCRjGa8+kTr0CzApU0+XXtRgwmzIf1SeKSGAv+2scAlW6JR5PMFo6FzqqZ64Po79cKODKf3/AAmECqA==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [linux]
'@biomejs/cli-linux-arm64@1.9.4':
resolution: {integrity: sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [linux]
'@biomejs/cli-linux-x64-musl@1.9.4':
resolution: {integrity: sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [linux]
'@biomejs/cli-linux-x64@1.9.4':
resolution: {integrity: sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [linux]
'@biomejs/cli-win32-arm64@1.9.4':
resolution: {integrity: sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [win32]
'@biomejs/cli-win32-x64@1.9.4':
resolution: {integrity: sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [win32]
'@esbuild/aix-ppc64@0.25.0':
resolution: {integrity: sha512-O7vun9Sf8DFjH2UtqK8Ku3LkquL9SZL8OLY1T5NZkA34+wG3OQF7cl4Ql8vdNzM6fzBbYfLaiRLIOZ+2FOCgBQ==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [aix]
'@esbuild/android-arm64@0.25.0':
resolution: {integrity: sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==}
engines: {node: '>=18'}
cpu: [arm64]
os: [android]
'@esbuild/android-arm@0.25.0':
resolution: {integrity: sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==}
engines: {node: '>=18'}
cpu: [arm]
os: [android]
'@esbuild/android-x64@0.25.0':
resolution: {integrity: sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==}
engines: {node: '>=18'}
cpu: [x64]
os: [android]
'@esbuild/darwin-arm64@0.25.0':
resolution: {integrity: sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [darwin]
'@esbuild/darwin-x64@0.25.0':
resolution: {integrity: sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==}
engines: {node: '>=18'}
cpu: [x64]
os: [darwin]
'@esbuild/freebsd-arm64@0.25.0':
resolution: {integrity: sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==}
engines: {node: '>=18'}
cpu: [arm64]
os: [freebsd]
'@esbuild/freebsd-x64@0.25.0':
resolution: {integrity: sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==}
engines: {node: '>=18'}
cpu: [x64]
os: [freebsd]
'@esbuild/linux-arm64@0.25.0':
resolution: {integrity: sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==}
engines: {node: '>=18'}
cpu: [arm64]
os: [linux]
'@esbuild/linux-arm@0.25.0':
resolution: {integrity: sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==}
engines: {node: '>=18'}
cpu: [arm]
os: [linux]
'@esbuild/linux-ia32@0.25.0':
resolution: {integrity: sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==}
engines: {node: '>=18'}
cpu: [ia32]
os: [linux]
'@esbuild/linux-loong64@0.25.0':
resolution: {integrity: sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==}
engines: {node: '>=18'}
cpu: [loong64]
os: [linux]
'@esbuild/linux-mips64el@0.25.0':
resolution: {integrity: sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==}
engines: {node: '>=18'}
cpu: [mips64el]
os: [linux]
'@esbuild/linux-ppc64@0.25.0':
resolution: {integrity: sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [linux]
'@esbuild/linux-riscv64@0.25.0':
resolution: {integrity: sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==}
engines: {node: '>=18'}
cpu: [riscv64]
os: [linux]
'@esbuild/linux-s390x@0.25.0':
resolution: {integrity: sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==}
engines: {node: '>=18'}
cpu: [s390x]
os: [linux]
'@esbuild/linux-x64@0.25.0':
resolution: {integrity: sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==}
engines: {node: '>=18'}
cpu: [x64]
os: [linux]
'@esbuild/netbsd-arm64@0.25.0':
resolution: {integrity: sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [netbsd]
'@esbuild/netbsd-x64@0.25.0':
resolution: {integrity: sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==}
engines: {node: '>=18'}
cpu: [x64]
os: [netbsd]
'@esbuild/openbsd-arm64@0.25.0':
resolution: {integrity: sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [openbsd]
'@esbuild/openbsd-x64@0.25.0':
resolution: {integrity: sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==}
engines: {node: '>=18'}
cpu: [x64]
os: [openbsd]
'@esbuild/sunos-x64@0.25.0':
resolution: {integrity: sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==}
engines: {node: '>=18'}
cpu: [x64]
os: [sunos]
'@esbuild/win32-arm64@0.25.0':
resolution: {integrity: sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [win32]
'@esbuild/win32-ia32@0.25.0':
resolution: {integrity: sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==}
engines: {node: '>=18'}
cpu: [ia32]
os: [win32]
'@esbuild/win32-x64@0.25.0':
resolution: {integrity: sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [win32]
'@protobufjs/aspromise@1.1.2':
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
'@protobufjs/base64@1.1.2':
resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
'@protobufjs/codegen@2.0.4':
resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==}
'@protobufjs/eventemitter@1.1.0':
resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
'@protobufjs/fetch@1.1.0':
resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
'@protobufjs/float@1.0.2':
resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
'@protobufjs/inquire@1.1.0':
resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==}
'@protobufjs/path@1.1.2':
resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
'@protobufjs/pool@1.1.0':
resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
'@protobufjs/utf8@1.1.0':
resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
'@types/node@22.13.5':
resolution: {integrity: sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg==}
esbuild@0.25.0:
resolution: {integrity: sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==}
engines: {node: '>=18'}
hasBin: true
flatbuffers@1.12.0:
resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==}
guid-typescript@1.0.9:
resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
long@5.3.1:
resolution: {integrity: sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng==}
onnxruntime-common@1.20.1:
resolution: {integrity: sha512-YiU0s0IzYYC+gWvqD1HzLc46Du1sXpSiwzKb63PACIJr6LfL27VsXSXQvt68EzD3V0D5Bc0vyJTjmMxp0ylQiw==}
onnxruntime-web@1.20.1:
resolution: {integrity: sha512-TePF6XVpLL1rWVMIl5Y9ACBQcyCNFThZON/jgElNd9Txb73CIEGlklhYR3UEr1cp5r0rbGI6nDwwrs79g7WjoA==}
platform@1.3.6:
resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
protobufjs@7.4.0:
resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==}
engines: {node: '>=12.0.0'}
typescript@5.8.3:
resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==}
engines: {node: '>=14.17'}
hasBin: true
undici-types@6.20.0:
resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
snapshots:
'@biomejs/biome@1.9.4':
optionalDependencies:
'@biomejs/cli-darwin-arm64': 1.9.4
'@biomejs/cli-darwin-x64': 1.9.4
'@biomejs/cli-linux-arm64': 1.9.4
'@biomejs/cli-linux-arm64-musl': 1.9.4
'@biomejs/cli-linux-x64': 1.9.4
'@biomejs/cli-linux-x64-musl': 1.9.4
'@biomejs/cli-win32-arm64': 1.9.4
'@biomejs/cli-win32-x64': 1.9.4
'@biomejs/cli-darwin-arm64@1.9.4':
optional: true
'@biomejs/cli-darwin-x64@1.9.4':
optional: true
'@biomejs/cli-linux-arm64-musl@1.9.4':
optional: true
'@biomejs/cli-linux-arm64@1.9.4':
optional: true
'@biomejs/cli-linux-x64-musl@1.9.4':
optional: true
'@biomejs/cli-linux-x64@1.9.4':
optional: true
'@biomejs/cli-win32-arm64@1.9.4':
optional: true
'@biomejs/cli-win32-x64@1.9.4':
optional: true
'@esbuild/aix-ppc64@0.25.0':
optional: true
'@esbuild/android-arm64@0.25.0':
optional: true
'@esbuild/android-arm@0.25.0':
optional: true
'@esbuild/android-x64@0.25.0':
optional: true
'@esbuild/darwin-arm64@0.25.0':
optional: true
'@esbuild/darwin-x64@0.25.0':
optional: true
'@esbuild/freebsd-arm64@0.25.0':
optional: true
'@esbuild/freebsd-x64@0.25.0':
optional: true
'@esbuild/linux-arm64@0.25.0':
optional: true
'@esbuild/linux-arm@0.25.0':
optional: true
'@esbuild/linux-ia32@0.25.0':
optional: true
'@esbuild/linux-loong64@0.25.0':
optional: true
'@esbuild/linux-mips64el@0.25.0':
optional: true
'@esbuild/linux-ppc64@0.25.0':
optional: true
'@esbuild/linux-riscv64@0.25.0':
optional: true
'@esbuild/linux-s390x@0.25.0':
optional: true
'@esbuild/linux-x64@0.25.0':
optional: true
'@esbuild/netbsd-arm64@0.25.0':
optional: true
'@esbuild/netbsd-x64@0.25.0':
optional: true
'@esbuild/openbsd-arm64@0.25.0':
optional: true
'@esbuild/openbsd-x64@0.25.0':
optional: true
'@esbuild/sunos-x64@0.25.0':
optional: true
'@esbuild/win32-arm64@0.25.0':
optional: true
'@esbuild/win32-ia32@0.25.0':
optional: true
'@esbuild/win32-x64@0.25.0':
optional: true
'@protobufjs/aspromise@1.1.2': {}
'@protobufjs/base64@1.1.2': {}
'@protobufjs/codegen@2.0.4': {}
'@protobufjs/eventemitter@1.1.0': {}
'@protobufjs/fetch@1.1.0':
dependencies:
'@protobufjs/aspromise': 1.1.2
'@protobufjs/inquire': 1.1.0
'@protobufjs/float@1.0.2': {}
'@protobufjs/inquire@1.1.0': {}
'@protobufjs/path@1.1.2': {}
'@protobufjs/pool@1.1.0': {}
'@protobufjs/utf8@1.1.0': {}
'@types/node@22.13.5':
dependencies:
undici-types: 6.20.0
esbuild@0.25.0:
optionalDependencies:
'@esbuild/aix-ppc64': 0.25.0
'@esbuild/android-arm': 0.25.0
'@esbuild/android-arm64': 0.25.0
'@esbuild/android-x64': 0.25.0
'@esbuild/darwin-arm64': 0.25.0
'@esbuild/darwin-x64': 0.25.0
'@esbuild/freebsd-arm64': 0.25.0
'@esbuild/freebsd-x64': 0.25.0
'@esbuild/linux-arm': 0.25.0
'@esbuild/linux-arm64': 0.25.0
'@esbuild/linux-ia32': 0.25.0
'@esbuild/linux-loong64': 0.25.0
'@esbuild/linux-mips64el': 0.25.0
'@esbuild/linux-ppc64': 0.25.0
'@esbuild/linux-riscv64': 0.25.0
'@esbuild/linux-s390x': 0.25.0
'@esbuild/linux-x64': 0.25.0
'@esbuild/netbsd-arm64': 0.25.0
'@esbuild/netbsd-x64': 0.25.0
'@esbuild/openbsd-arm64': 0.25.0
'@esbuild/openbsd-x64': 0.25.0
'@esbuild/sunos-x64': 0.25.0
'@esbuild/win32-arm64': 0.25.0
'@esbuild/win32-ia32': 0.25.0
'@esbuild/win32-x64': 0.25.0
flatbuffers@1.12.0: {}
guid-typescript@1.0.9: {}
long@5.3.1: {}
onnxruntime-common@1.20.1: {}
onnxruntime-web@1.20.1:
dependencies:
flatbuffers: 1.12.0
guid-typescript: 1.0.9
long: 5.3.1
onnxruntime-common: 1.20.1
platform: 1.3.6
protobufjs: 7.4.0
platform@1.3.6: {}
protobufjs@7.4.0:
dependencies:
'@protobufjs/aspromise': 1.1.2
'@protobufjs/base64': 1.1.2
'@protobufjs/codegen': 2.0.4
'@protobufjs/eventemitter': 1.1.0
'@protobufjs/fetch': 1.1.0
'@protobufjs/float': 1.0.2
'@protobufjs/inquire': 1.1.0
'@protobufjs/path': 1.1.2
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
'@types/node': 22.13.5
long: 5.3.1
typescript@5.8.3: {}
undici-types@6.20.0: {}

0
scripts/docker/cpu.Dockerfile → docker/cpu.Dockerfile
View File

0
scripts/docker/cuda.Dockerfile → docker/cuda.Dockerfile
View File

0
scripts/docker/run_cpu.sh → docker/run_cpu.sh
View File

0
scripts/docker/run_cuda.sh → docker/run_cuda.sh
View File

17
crates/sbv2_api/Cargo.toml → sbv2_api/Cargo.toml
View File

@@ -1,24 +1,19 @@
[package]
name = "sbv2_api"
version.workspace = true
edition.workspace = true
description.workspace = true
readme.workspace = true
repository.workspace = true
documentation.workspace = true
license.workspace = true
version = "0.2.0-alpha"
edition = "2021"
[dependencies]
anyhow.workspace = true
axum = "0.8.0"
axum = "0.7.5"
dotenvy.workspace = true
env_logger.workspace = true
log = "0.4.22"
sbv2_core = { version = "0.2.0-alpha6", path = "../sbv2_core", features = ["aivmx"] }
sbv2_core = { version = "0.2.0-alpha", path = "../sbv2_core" }
serde = { version = "1.0.210", features = ["derive"] }
tokio = { version = "1.40.0", features = ["full"] }
utoipa = { version = "5.0.0", features = ["axum_extras"] }
utoipa-scalar = { version = "0.3.0", features = ["axum"] }
utoipa = { version = "4.2.3", features = ["axum_extras"] }
utoipa-scalar = { version = "0.1.0", features = ["axum"] }
[features]
coreml = ["sbv2_core/coreml"]

0
crates/sbv2_api/build.rs → sbv2_api/build.rs
View File

0
crates/sbv2_api/src/error.rs → sbv2_api/src/error.rs
View File

36
crates/sbv2_api/src/main.rs → sbv2_api/src/main.rs
View File

@@ -40,14 +40,6 @@ fn length_default() -> f32 {
1.0
}
fn style_id_default() -> i32 {
0
}
fn speaker_id_default() -> i64 {
0
}
#[derive(Deserialize, ToSchema)]
struct SynthesizeRequest {
text: String,
@@ -56,10 +48,6 @@ struct SynthesizeRequest {
sdp_ratio: f32,
#[serde(default = "length_default")]
length_scale: f32,
#[serde(default = "style_id_default")]
style_id: i32,
#[serde(default = "speaker_id_default")]
speaker_id: i64,
}
#[utoipa::path(
@@ -77,18 +65,15 @@ async fn synthesize(
ident,
sdp_ratio,
length_scale,
style_id,
speaker_id,
}): Json<SynthesizeRequest>,
) -> AppResult<impl IntoResponse> {
log::debug!("processing request: text={text}, ident={ident}, sdp_ratio={sdp_ratio}, length_scale={length_scale}");
let buffer = {
let mut tts_model = state.tts_model.lock().await;
let tts_model = state.tts_model.lock().await;
tts_model.easy_synthesize(
&ident,
&text,
style_id,
speaker_id,
0,
SynthesizeOptions {
sdp_ratio,
length_scale,
@@ -109,9 +94,6 @@ impl AppState {
let mut tts_model = TTSModelHolder::new(
&fs::read(env::var("BERT_MODEL_PATH")?).await?,
&fs::read(env::var("TOKENIZER_PATH")?).await?,
env::var("HOLDER_MAX_LOADED_MODElS")
.ok()
.and_then(|x| x.parse().ok()),
)?;
let models = env::var("MODELS_PATH").unwrap_or("models".to_string());
let mut f = fs::read_dir(&models).await?;
@@ -140,20 +122,6 @@ impl AppState {
log::warn!("Error loading {entry}: {e}");
};
log::info!("Loaded: {entry}");
} else if name.ends_with(".aivmx") {
let entry = &name[..name.len() - 6];
log::info!("Try loading: {entry}");
let aivmx_bytes = match fs::read(format!("{models}/{entry}.aivmx")).await {
Ok(b) => b,
Err(e) => {
log::warn!("Error loading aivmx bytes from file {entry}: {e}");
continue;
}
};
if let Err(e) = tts_model.load_aivmx(entry, aivmx_bytes) {
log::error!("Error loading {entry}: {e}");
}
log::info!("Loaded: {entry}");
}
}
for entry in entries {

15
sbv2_bindings/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "sbv2_bindings"
version = "0.2.0-alpha1"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "sbv2_bindings"
crate-type = ["cdylib"]
[dependencies]
anyhow.workspace = true
ndarray.workspace = true
pyo3 = { version = "0.22.0", features = ["anyhow"] }
sbv2_core = { version = "0.2.0-alpha", path = "../sbv2_core" }

6
scripts/sbv2-test-bindings.py → sbv2_bindings/examples/basic.py
View File

@@ -3,16 +3,16 @@ from sbv2_bindings import TTSModel
def main():
print("Loading models...")
model = TTSModel.from_path("./models/debert.onnx", "./models/tokenizer.json")
model = TTSModel.from_path("../models/debert.onnx", "../models/tokenizer.json")
print("Models loaded!")
model.load_sbv2file_from_path("amitaro", "./models/amitaro.sbv2")
model.load_sbv2file_from_path("amitaro", "../models/amitaro.sbv2")
print("All setup is done!")
style_vector = model.get_style_vector("amitaro", 0, 1.0)
with open("output.wav", "wb") as f:
f.write(
model.synthesize("おはようございます。", "amitaro", 0, 0, 0.0, 0.5)
model.synthesize("おはようございます。", "amitaro", style_vector, 0.0, 0.5)
)

0
crates/sbv2_bindings/pyproject.toml → sbv2_bindings/pyproject.toml
View File

0
crates/sbv2_bindings/src/lib.rs → sbv2_bindings/src/lib.rs
View File

34
crates/sbv2_bindings/src/sbv2.rs → sbv2_bindings/src/sbv2.rs
View File

@@ -23,15 +23,10 @@ pub struct TTSModel {
#[pymethods]
impl TTSModel {
#[pyo3(signature = (bert_model_bytes, tokenizer_bytes, max_loaded_models=None))]
#[new]
fn new(
bert_model_bytes: Vec<u8>,
tokenizer_bytes: Vec<u8>,
max_loaded_models: Option<usize>,
) -> anyhow::Result<Self> {
fn new(bert_model_bytes: Vec<u8>, tokenizer_bytes: Vec<u8>) -> anyhow::Result<Self> {
Ok(Self {
model: TTSModelHolder::new(bert_model_bytes, tokenizer_bytes, max_loaded_models)?,
model: TTSModelHolder::new(bert_model_bytes, tokenizer_bytes)?,
})
}
@@ -43,21 +38,10 @@ impl TTSModel {
/// BERTモデルのパス
/// tokenizer_path : str
/// トークナイザーのパス
/// max_loaded_models: int | None
/// 同時にVRAMに存在するモデルの数
#[pyo3(signature = (bert_model_path, tokenizer_path, max_loaded_models=None))]
#[staticmethod]
fn from_path(
bert_model_path: String,
tokenizer_path: String,
max_loaded_models: Option<usize>,
) -> anyhow::Result<Self> {
fn from_path(bert_model_path: String, tokenizer_path: String) -> anyhow::Result<Self> {
Ok(Self {
model: TTSModelHolder::new(
fs::read(bert_model_path)?,
fs::read(tokenizer_path)?,
max_loaded_models,
)?,
model: TTSModelHolder::new(fs::read(bert_model_path)?, fs::read(tokenizer_path)?)?,
})
}
@@ -107,7 +91,7 @@ impl TTSModel {
/// style_vector : StyleVector
/// スタイルベクトル
fn get_style_vector(
&mut self,
&self,
ident: String,
style_id: i32,
weight: f32,
@@ -137,27 +121,25 @@ impl TTSModel {
/// voice_data : bytes
/// 音声データ
fn synthesize<'p>(
&'p mut self,
&'p self,
py: Python<'p>,
text: String,
ident: String,
style_id: i32,
speaker_id: i64,
sdp_ratio: f32,
length_scale: f32,
) -> anyhow::Result<Bound<'p, PyBytes>> {
) -> anyhow::Result<Bound<PyBytes>> {
let data = self.model.easy_synthesize(
ident.as_str(),
&text,
style_id,
speaker_id,
SynthesizeOptions {
sdp_ratio,
length_scale,
..Default::default()
},
)?;
Ok(PyBytes::new(py, &data))
Ok(PyBytes::new_bound(py, &data))
}
fn unload(&mut self, ident: String) -> bool {

0
crates/sbv2_bindings/src/style.rs → sbv2_bindings/src/style.rs
View File

35
crates/sbv2_core/Cargo.toml → sbv2_core/Cargo.toml
View File

@@ -1,47 +1,40 @@
[package]
name = "sbv2_core"
version.workspace = true
edition.workspace = true
description.workspace = true
readme.workspace = true
repository.workspace = true
documentation.workspace = true
license.workspace = true
description = "Style-Bert-VITSの推論ライブラリ"
version = "0.2.0-alpha1"
edition = "2021"
license = "MIT"
readme = "../README.md"
repository = "https://github.com/tuna2134/sbv2-api"
documentation = "https://docs.rs/sbv2_core"
[dependencies]
anyhow.workspace = true
base64 = { version = "0.22.1", optional = true }
dotenvy.workspace = true
env_logger.workspace = true
hound = "3.5.1"
jpreprocess = { version = "0.12.0", features = ["naist-jdic"] }
jpreprocess = { version = "0.10.0", features = ["naist-jdic"] }
ndarray.workspace = true
npyz = { version = "0.8.3", optional = true }
num_cpus = "1.16.0"
once_cell.workspace = true
ort = { git = "https://github.com/pykeio/ort.git", version = "2.0.0-rc.9", optional = true }
ort = { git = "https://github.com/pykeio/ort.git", version = "2.0.0-rc.6", optional = true }
regex = "1.10.6"
serde = { version = "1.0.210", features = ["derive"] }
serde_json = "1.0.128"
tar = "0.4.41"
thiserror = "2.0.11"
tokenizers = { version = "0.21.0", default-features = false }
thiserror = "1.0.63"
tokenizers = { version = "0.20.0", default-features = false }
vibrato = { version = "0.5.1", optional = true }
zstd = "0.13.2"
[features]
cuda = ["ort/cuda", "std"]
cuda_tf32 = ["std", "cuda"]
agpl_dict = []
std = ["dep:ort", "tokenizers/progressbar", "tokenizers/onig", "tokenizers/esaxx_fast"]
dynamic = ["ort/load-dynamic", "std"]
directml = ["ort/directml", "std"]
tensorrt = ["ort/tensorrt", "std"]
coreml = ["ort/coreml", "std"]
default = ["std", "agpl_dict"]
default = ["std"]
no_std = ["tokenizers/unstable_wasm"]
aivmx = ["npyz", "base64"]
base64 = ["dep:base64"]
[build-dependencies]
dirs = "6.0.0"
ureq = "3.0.6"
mecab = ["vibrato"]

0
crates/sbv2_core/mora_convert.py → sbv2_core/mora_convert.py
View File

23
sbv2_core/src/bert.rs Normal file
View File

@@ -0,0 +1,23 @@
use crate::error::Result;
use ndarray::Array2;
use ort::Session;
pub fn predict(
session: &Session,
token_ids: Vec<i64>,
attention_masks: Vec<i64>,
) -> Result<Array2<f32>> {
let outputs = session.run(
ort::inputs! {
"input_ids" => Array2::from_shape_vec((1, token_ids.len()), token_ids).unwrap(),
"attention_mask" => Array2::from_shape_vec((1, attention_masks.len()), attention_masks).unwrap(),
}?
)?;
let output = outputs.get("output").unwrap();
let content = output.try_extract_tensor::<f32>()?.to_owned();
let (data, _) = content.clone().into_raw_vec_and_offset();
Ok(Array2::from_shape_vec((content.shape()[0], content.shape()[1]), data).unwrap())
}

5
crates/sbv2_core/src/error.rs → sbv2_core/src/error.rs
View File

@@ -6,8 +6,6 @@ pub enum Error {
TokenizerError(#[from] tokenizers::Error),
#[error("JPreprocess error: {0}")]
JPreprocessError(#[from] jpreprocess::error::JPreprocessError),
#[error("Lindera error: {0}")]
LinderaError(String),
#[cfg(feature = "std")]
#[error("ONNX error: {0}")]
OrtError(#[from] ort::Error),
@@ -23,9 +21,6 @@ pub enum Error {
HoundError(#[from] hound::Error),
#[error("model not found error")]
ModelNotFoundError(String),
#[cfg(feature = "base64")]
#[error("base64 error")]
Base64Error(#[from] base64::DecodeError),
#[error("other")]
OtherError(String),
}

60
crates/sbv2_core/src/jtalk.rs → sbv2_core/src/jtalk.rs
View File

@@ -1,32 +1,21 @@
use crate::error::{Error, Result};
use crate::mora::{CONSONANTS, MORA_KATA_TO_MORA_PHONEMES, MORA_PHONEMES_TO_MORA_KATA, VOWELS};
use crate::mora::{MORA_KATA_TO_MORA_PHONEMES, VOWELS};
use crate::norm::{replace_punctuation, PUNCTUATIONS};
use jpreprocess::{kind, DefaultTokenizer, JPreprocess, SystemDictionaryConfig, UserDictionary};
use jpreprocess::*;
use once_cell::sync::Lazy;
use regex::Regex;
use std::cmp::Reverse;
use std::collections::HashSet;
use std::sync::Arc;
type JPreprocessType = JPreprocess<DefaultTokenizer>;
#[cfg(feature = "agpl_dict")]
fn agpl_dict() -> Result<Option<UserDictionary>> {
Ok(Some(
UserDictionary::load(include_bytes!(concat!(env!("OUT_DIR"), "/all.bin")))
.map_err(|e| Error::LinderaError(e.to_string()))?,
))
}
#[cfg(not(feature = "agpl_dict"))]
fn agpl_dict() -> Result<Option<UserDictionary>> {
Ok(None)
}
type JPreprocessType = JPreprocess<DefaultFetcher>;
fn initialize_jtalk() -> Result<JPreprocessType> {
let sdic =
SystemDictionaryConfig::Bundled(kind::JPreprocessDictionaryKind::NaistJdic).load()?;
let jpreprocess = JPreprocess::with_dictionaries(sdic, agpl_dict()?);
let config = JPreprocessConfig {
dictionary: SystemDictionaryConfig::Bundled(kind::JPreprocessDictionaryKind::NaistJdic),
user_dictionary: None,
};
let jpreprocess = JPreprocess::from_config(config)?;
Ok(jpreprocess)
}
@@ -76,34 +65,6 @@ static MORA_PATTERN: Lazy<Vec<String>> = Lazy::new(|| {
});
static LONG_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\w)(ー*)").unwrap());
fn phone_tone_to_kana(phones: Vec<String>, tones: Vec<i32>) -> Vec<(String, i32)> {
let phones = &phones[1..];
let tones = &tones[1..];
let mut results = Vec::new();
let mut current_mora = String::new();
for ((phone, next_phone), (&tone, &next_tone)) in phones
.iter()
.zip(phones.iter().skip(1))
.zip(tones.iter().zip(tones.iter().skip(1)))
{
if PUNCTUATIONS.contains(&phone.clone().as_str()) {
results.push((phone.to_string(), tone));
continue;
}
if CONSONANTS.contains(&phone.clone()) {
assert_eq!(current_mora, "");
assert_eq!(tone, next_tone);
current_mora = phone.to_string()
} else {
current_mora += phone;
let kana = MORA_PHONEMES_TO_MORA_KATA.get(&current_mora).unwrap();
results.push((kana.to_string(), tone));
current_mora = String::new();
}
}
results
}
pub struct JTalkProcess {
jpreprocess: Arc<JPreprocessType>,
parsed: Vec<String>,
@@ -193,11 +154,6 @@ impl JTalkProcess {
Ok((phones, tones, new_word2ph))
}
pub fn g2kana_tone(&self) -> Result<Vec<(String, i32)>> {
let (phones, tones, _) = self.g2p()?;
Ok(phone_tone_to_kana(phones, tones))
}
fn distribute_phone(n_phone: i32, n_word: i32) -> Vec<i32> {
let mut phones_per_word = vec![0; n_word as usize];
for _ in 0..n_phone {

3
crates/sbv2_core/src/lib.rs → sbv2_core/src/lib.rs
View File

@@ -14,3 +14,6 @@ pub mod tokenizer;
pub mod tts;
pub mod tts_util;
pub mod utils;
#[cfg(feature = "mecab")]
pub mod mecab;

31
sbv2_core/src/main.rs Normal file
View File

@@ -0,0 +1,31 @@
use std::env;
use std::fs;
#[cfg(feature = "std")]
fn main_inner() -> anyhow::Result<()> {
use sbv2_core::tts;
dotenvy::dotenv_override().ok();
env_logger::init();
let text = fs::read_to_string("content.txt")?;
let ident = "aaa";
let mut tts_holder = tts::TTSModelHolder::new(
&fs::read(env::var("BERT_MODEL_PATH")?)?,
&fs::read(env::var("TOKENIZER_PATH")?)?,
)?;
tts_holder.load_sbv2file(ident, fs::read(env::var("MODEL_PATH")?)?)?;
let audio = tts_holder.easy_synthesize(ident, &text, 0, tts::SynthesizeOptions::default())?;
fs::write("output.wav", audio)?;
Ok(())
}
#[cfg(not(feature = "std"))]
fn main_inner() -> anyhow::Result<()> {
Ok(())
}
fn main() {
if let Err(e) = main_inner() {
println!("Error: {e}");
}
}

0
sbv2_core/src/mecab.rs Normal file
View File

93
sbv2_core/src/model.rs Normal file
View File

@@ -0,0 +1,93 @@
use crate::error::Result;
use ndarray::{array, Array1, Array2, Array3, Axis};
use ort::{GraphOptimizationLevel, Session};
#[allow(clippy::vec_init_then_push, unused_variables)]
pub fn load_model<P: AsRef<[u8]>>(model_file: P, bert: bool) -> Result<Session> {
let mut exp = Vec::new();
#[cfg(feature = "tensorrt")]
{
if bert {
exp.push(
ort::TensorRTExecutionProvider::default()
.with_fp16(true)
.with_profile_min_shapes("input_ids:1x1,attention_mask:1x1")
.with_profile_max_shapes("input_ids:1x100,attention_mask:1x100")
.with_profile_opt_shapes("input_ids:1x25,attention_mask:1x25")
.build(),
);
}
}
#[cfg(feature = "cuda")]
{
#[allow(unused_mut)]
let mut cuda = ort::CUDAExecutionProvider::default()
.with_conv_algorithm_search(ort::CUDAExecutionProviderCuDNNConvAlgoSearch::Default);
#[cfg(feature = "cuda_tf32")]
{
cuda = cuda.with_tf32(true);
}
exp.push(cuda.build());
}
#[cfg(feature = "directml")]
{
exp.push(ort::DirectMLExecutionProvider::default().build());
}
#[cfg(feature = "coreml")]
{
exp.push(ort::CoreMLExecutionProvider::default().build());
}
exp.push(ort::CPUExecutionProvider::default().build());
Ok(Session::builder()?
.with_execution_providers(exp)?
.with_optimization_level(GraphOptimizationLevel::Level3)?
.with_intra_threads(num_cpus::get_physical())?
.with_parallel_execution(true)?
.with_inter_threads(num_cpus::get_physical())?
.commit_from_memory(model_file.as_ref())?)
}
#[allow(clippy::too_many_arguments)]
pub fn synthesize(
session: &Session,
bert_ori: Array2<f32>,
x_tst: Array1<i64>,
tones: Array1<i64>,
lang_ids: Array1<i64>,
style_vector: Array1<f32>,
sdp_ratio: f32,
length_scale: f32,
) -> Result<Array3<f32>> {
let bert = bert_ori.insert_axis(Axis(0));
let x_tst_lengths: Array1<i64> = array![x_tst.shape()[0] as i64];
let x_tst = x_tst.insert_axis(Axis(0));
let lang_ids = lang_ids.insert_axis(Axis(0));
let tones = tones.insert_axis(Axis(0));
let style_vector = style_vector.insert_axis(Axis(0));
let outputs = session.run(ort::inputs! {
"x_tst" => x_tst,
"x_tst_lengths" => x_tst_lengths,
"sid" => array![0_i64],
"tones" => tones,
"language" => lang_ids,
"bert" => bert,
"style_vec" => style_vector,
"sdp_ratio" => array![sdp_ratio],
"length_scale" => array![length_scale],
}?)?;
let audio_array = outputs
.get("output")
.unwrap()
.try_extract_tensor::<f32>()?
.to_owned();
Ok(Array3::from_shape_vec(
(
audio_array.shape()[0],
audio_array.shape()[1],
audio_array.shape()[2],
),
audio_array.into_raw_vec_and_offset().0,
)?)
}

23
crates/sbv2_core/src/mora.rs → sbv2_core/src/mora.rs
View File

@@ -25,21 +25,6 @@ static MORA_LIST_ADDITIONAL: Lazy<Vec<Mora>> = Lazy::new(|| {
data.additional
});
pub static MORA_PHONEMES_TO_MORA_KATA: Lazy<HashMap<String, String>> = Lazy::new(|| {
let mut map = HashMap::new();
for mora in MORA_LIST_MINIMUM.iter() {
map.insert(
format!(
"{}{}",
mora.consonant.clone().unwrap_or("".to_string()),
mora.vowel
),
mora.mora.clone(),
);
}
map
});
pub static MORA_KATA_TO_MORA_PHONEMES: Lazy<HashMap<String, (Option<String>, String)>> =
Lazy::new(|| {
let mut map = HashMap::new();
@@ -52,12 +37,4 @@ pub static MORA_KATA_TO_MORA_PHONEMES: Lazy<HashMap<String, (Option<String>, Str
map
});
pub static CONSONANTS: Lazy<Vec<String>> = Lazy::new(|| {
let consonants = MORA_KATA_TO_MORA_PHONEMES
.values()
.filter_map(|(consonant, _)| consonant.clone())
.collect::<Vec<_>>();
consonants
});
pub const VOWELS: [&str; 6] = ["a", "i", "u", "e", "o", "N"];

0
crates/sbv2_core/src/mora_list.json → sbv2_core/src/mora_list.json
View File

0
crates/sbv2_core/src/nlp.rs → sbv2_core/src/nlp.rs
View File

0
crates/sbv2_core/src/norm.rs → sbv2_core/src/norm.rs
View File

0
crates/sbv2_core/src/sbv2file.rs → sbv2_core/src/sbv2file.rs
View File

0
crates/sbv2_core/src/style.rs → sbv2_core/src/style.rs
View File

0
crates/sbv2_core/src/tokenizer.rs → sbv2_core/src/tokenizer.rs
View File

273
sbv2_core/src/tts.rs Normal file
View File

@@ -0,0 +1,273 @@
use crate::error::{Error, Result};
use crate::{jtalk, model, style, tokenizer, tts_util};
use ndarray::{concatenate, Array1, Array2, Array3, Axis};
use ort::Session;
use tokenizers::Tokenizer;
#[derive(PartialEq, Eq, Clone)]
pub struct TTSIdent(String);
impl std::fmt::Display for TTSIdent {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)?;
Ok(())
}
}
impl<S> From<S> for TTSIdent
where
S: AsRef<str>,
{
fn from(value: S) -> Self {
TTSIdent(value.as_ref().to_string())
}
}
pub struct TTSModel {
vits2: Session,
style_vectors: Array2<f32>,
ident: TTSIdent,
}
/// High-level Style-Bert-VITS2's API
pub struct TTSModelHolder {
tokenizer: Tokenizer,
bert: Session,
models: Vec<TTSModel>,
jtalk: jtalk::JTalk,
}
impl TTSModelHolder {
/// Initialize a new TTSModelHolder
///
/// # Examples
///
/// ```rs
/// let mut tts_holder = TTSModelHolder::new(std::fs::read("deberta.onnx")?, std::fs::read("tokenizer.json")?)?;
/// ```
pub fn new<P: AsRef<[u8]>>(bert_model_bytes: P, tokenizer_bytes: P) -> Result<Self> {
let bert = model::load_model(bert_model_bytes, true)?;
let jtalk = jtalk::JTalk::new()?;
let tokenizer = tokenizer::get_tokenizer(tokenizer_bytes)?;
Ok(TTSModelHolder {
bert,
models: vec![],
jtalk,
tokenizer,
})
}
/// Return a list of model names
pub fn models(&self) -> Vec<String> {
self.models.iter().map(|m| m.ident.to_string()).collect()
}
/// Load a .sbv2 file binary
///
/// # Examples
///
/// ```rs
/// tts_holder.load_sbv2file("tsukuyomi", std::fs::read("tsukuyomi.sbv2")?)?;
/// ```
pub fn load_sbv2file<I: Into<TTSIdent>, P: AsRef<[u8]>>(
&mut self,
ident: I,
sbv2_bytes: P,
) -> Result<()> {
let (style_vectors, vits2) = crate::sbv2file::parse_sbv2file(sbv2_bytes)?;
self.load(ident, style_vectors, vits2)?;
Ok(())
}
/// Load a style vector and onnx model binary
///
/// # Examples
///
/// ```rs
/// tts_holder.load("tsukuyomi", std::fs::read("style_vectors.json")?, std::fs::read("model.onnx")?)?;
/// ```
pub fn load<I: Into<TTSIdent>, P: AsRef<[u8]>>(
&mut self,
ident: I,
style_vectors_bytes: P,
vits2_bytes: P,
) -> Result<()> {
let ident = ident.into();
if self.find_model(ident.clone()).is_err() {
self.models.push(TTSModel {
vits2: model::load_model(vits2_bytes, false)?,
style_vectors: style::load_style(style_vectors_bytes)?,
ident,
})
}
Ok(())
}
/// Unload a model
pub fn unload<I: Into<TTSIdent>>(&mut self, ident: I) -> bool {
let ident = ident.into();
if let Some((i, _)) = self
.models
.iter()
.enumerate()
.find(|(_, m)| m.ident == ident)
{
self.models.remove(i);
true
} else {
false
}
}
/// Parse text and return the input for synthesize
///
/// # Note
/// This function is for low-level usage, use `easy_synthesize` for high-level usage.
#[allow(clippy::type_complexity)]
pub fn parse_text(
&self,
text: &str,
) -> Result<(Array2<f32>, Array1<i64>, Array1<i64>, Array1<i64>)> {
crate::tts_util::parse_text_blocking(
text,
&self.jtalk,
&self.tokenizer,
|token_ids, attention_masks| {
crate::bert::predict(&self.bert, token_ids, attention_masks)
},
)
}
fn find_model<I: Into<TTSIdent>>(&self, ident: I) -> Result<&TTSModel> {
let ident = ident.into();
self.models
.iter()
.find(|m| m.ident == ident)
.ok_or(Error::ModelNotFoundError(ident.to_string()))
}
/// Get style vector by style id and weight
///
/// # Note
/// This function is for low-level usage, use `easy_synthesize` for high-level usage.
pub fn get_style_vector<I: Into<TTSIdent>>(
&self,
ident: I,
style_id: i32,
weight: f32,
) -> Result<Array1<f32>> {
style::get_style_vector(&self.find_model(ident)?.style_vectors, style_id, weight)
}
/// Synthesize text to audio
///
/// # Examples
///
/// ```rs
/// let audio = tts_holder.easy_synthesize("tsukuyomi", "こんにちは", 0, SynthesizeOptions::default())?;
/// ```
pub fn easy_synthesize<I: Into<TTSIdent> + Copy>(
&self,
ident: I,
text: &str,
style_id: i32,
options: SynthesizeOptions,
) -> Result<Vec<u8>> {
let style_vector = self.get_style_vector(ident, style_id, options.style_weight)?;
let audio_array = if options.split_sentences {
let texts: Vec<&str> = text.split('\n').collect();
let mut audios = vec![];
for (i, t) in texts.iter().enumerate() {
if t.is_empty() {
continue;
}
let (bert_ori, phones, tones, lang_ids) = self.parse_text(t)?;
let audio = model::synthesize(
&self.find_model(ident)?.vits2,
bert_ori.to_owned(),
phones,
tones,
lang_ids,
style_vector.clone(),
options.sdp_ratio,
options.length_scale,
)?;
audios.push(audio.clone());
if i != texts.len() - 1 {
audios.push(Array3::zeros((1, 1, 22050)));
}
}
concatenate(
Axis(2),
&audios.iter().map(|x| x.view()).collect::<Vec<_>>(),
)?
} else {
let (bert_ori, phones, tones, lang_ids) = self.parse_text(text)?;
model::synthesize(
&self.find_model(ident)?.vits2,
bert_ori.to_owned(),
phones,
tones,
lang_ids,
style_vector,
options.sdp_ratio,
options.length_scale,
)?
};
tts_util::array_to_vec(audio_array)
}
/// Synthesize text to audio
///
/// # Note
/// This function is for low-level usage, use `easy_synthesize` for high-level usage.
#[allow(clippy::too_many_arguments)]
pub fn synthesize<I: Into<TTSIdent>>(
&self,
ident: I,
bert_ori: Array2<f32>,
phones: Array1<i64>,
tones: Array1<i64>,
lang_ids: Array1<i64>,
style_vector: Array1<f32>,
sdp_ratio: f32,
length_scale: f32,
) -> Result<Vec<u8>> {
let audio_array = model::synthesize(
&self.find_model(ident)?.vits2,
bert_ori.to_owned(),
phones,
tones,
lang_ids,
style_vector,
sdp_ratio,
length_scale,
)?;
tts_util::array_to_vec(audio_array)
}
}
/// Synthesize options
///
/// # Fields
/// - `sdp_ratio`: SDP ratio
/// - `length_scale`: Length scale
/// - `style_weight`: Style weight
/// - `split_sentences`: Split sentences
pub struct SynthesizeOptions {
pub sdp_ratio: f32,
pub length_scale: f32,
pub style_weight: f32,
pub split_sentences: bool,
}
impl Default for SynthesizeOptions {
fn default() -> Self {
SynthesizeOptions {
sdp_ratio: 0.0,
length_scale: 1.0,
style_weight: 1.0,
split_sentences: true,
}
}
}

44
crates/sbv2_core/src/tts_util.rs → sbv2_core/src/tts_util.rs
View File

@@ -1,22 +1,10 @@
use std::io::Cursor;
use crate::error::Result;
use crate::jtalk::JTalkProcess;
use crate::mora::MORA_KATA_TO_MORA_PHONEMES;
use crate::norm::PUNCTUATIONS;
use crate::{jtalk, nlp, norm, tokenizer, utils};
use hound::{SampleFormat, WavSpec, WavWriter};
use ndarray::{concatenate, s, Array, Array1, Array2, Array3, Axis};
use tokenizers::Tokenizer;
pub fn preprocess_parse_text(text: &str, jtalk: &jtalk::JTalk) -> Result<(String, JTalkProcess)> {
let text = jtalk.num2word(text)?;
let normalized_text = norm::normalize_text(&text);
let process = jtalk.process_text(&normalized_text)?;
Ok((normalized_text, process))
}
/// Parse text and return the input for synthesize
///
/// # Note
@@ -33,9 +21,13 @@ pub async fn parse_text(
Box<dyn std::future::Future<Output = Result<ndarray::Array2<f32>>>>,
>,
) -> Result<(Array2<f32>, Array1<i64>, Array1<i64>, Array1<i64>)> {
let (normalized_text, process) = preprocess_parse_text(text, jtalk)?;
let text = jtalk.num2word(text)?;
let normalized_text = norm::normalize_text(&text);
let process = jtalk.process_text(&normalized_text)?;
let (phones, tones, mut word2ph) = process.g2p()?;
let (phones, tones, lang_ids) = nlp::cleaned_text_to_sequence(phones, tones);
let phones = utils::intersperse(&phones, 0);
let tones = utils::intersperse(&tones, 0);
let lang_ids = utils::intersperse(&lang_ids, 0);
@@ -100,7 +92,6 @@ pub async fn parse_text(
#[allow(clippy::type_complexity)]
pub fn parse_text_blocking(
text: &str,
given_tones: Option<Vec<i32>>,
jtalk: &jtalk::JTalk,
tokenizer: &Tokenizer,
bert_predict: impl FnOnce(Vec<i64>, Vec<i64>) -> Result<ndarray::Array2<f32>>,
@@ -109,10 +100,7 @@ pub fn parse_text_blocking(
let normalized_text = norm::normalize_text(&text);
let process = jtalk.process_text(&normalized_text)?;
let (phones, mut tones, mut word2ph) = process.g2p()?;
if let Some(given_tones) = given_tones {
tones = given_tones;
}
let (phones, tones, mut word2ph) = process.g2p()?;
let (phones, tones, lang_ids) = nlp::cleaned_text_to_sequence(phones, tones);
let phones = utils::intersperse(&phones, 0);
@@ -190,23 +178,3 @@ pub fn array_to_vec(audio_array: Array3<f32>) -> Result<Vec<u8>> {
writer.finalize()?;
Ok(cursor.into_inner())
}
pub fn kata_tone2phone_tone(kata_tone: Vec<(String, i32)>) -> Vec<(String, i32)> {
let mut results = vec![("_".to_string(), 0)];
for (mora, tone) in kata_tone {
if PUNCTUATIONS.contains(&mora.as_str()) {
results.push((mora, 0));
continue;
} else {
let (consonant, vowel) = MORA_KATA_TO_MORA_PHONEMES.get(&mora).unwrap();
if let Some(consonant) = consonant {
results.push((consonant.to_string(), tone));
results.push((vowel.to_string(), tone));
} else {
results.push((vowel.to_string(), tone));
}
}
}
results.push(("_".to_string(), 0));
results
}

0
crates/sbv2_core/src/utils.rs → sbv2_core/src/utils.rs
View File

13
crates/sbv2_wasm/Cargo.toml → sbv2_wasm/Cargo.toml
View File

@@ -1,12 +1,7 @@
[package]
name = "sbv2_wasm"
version.workspace = true
edition.workspace = true
description.workspace = true
readme.workspace = true
repository.workspace = true
documentation.workspace = true
license.workspace = true
version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["cdylib", "rlib"]
@@ -18,3 +13,7 @@ once_cell.workspace = true
js-sys = "0.3.70"
ndarray.workspace = true
wasm-bindgen-futures = "0.4.43"
[profile.release]
lto = true
opt-level = "s"

0
crates/sbv2_wasm/README.md → sbv2_wasm/README.md
View File

0
crates/sbv2_wasm/biome.json → sbv2_wasm/biome.json
View File

4
sbv2_wasm/build.sh Executable file
View File

@@ -0,0 +1,4 @@
wasm-pack build --target web sbv2_wasm
wasm-opt -O3 -o sbv2_wasm/pkg/sbv2_wasm_bg.wasm sbv2_wasm/pkg/sbv2_wasm_bg.wasm
mkdir -p sbv2_wasm/dist
cp sbv2_wasm/sbv2_wasm/pkg/sbv2_wasm_bg.wasm sbv2_wasm/dist/sbv2_wasm_bg.wasm

51
sbv2_wasm/example.html Normal file
View File

@@ -0,0 +1,51 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Style Bert VITS2 Web</title>
<script type="importmap">
{
"imports": {
"onnxruntime-web": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/ort.all.min.mjs",
"sbv2": "https://cdn.jsdelivr.net/npm/sbv2@0.1.1+esm"
}
}
</script>
<script type="module" async defer>
import { ModelHolder } from "sbv2";
await ModelHolder.globalInit(
await (
await fetch("https://esm.sh/sbv2@0.1.1/dist/sbv2_wasm_bg.wasm", { cache: "force-cache" })
).arrayBuffer(),
);
const holder = await ModelHolder.create(
await (
await fetch("/models/tokenizer.json", { cache: "force-cache" })
).text(),
await (
await fetch("/models/deberta.onnx", { cache: "force-cache" })
).arrayBuffer(),
);
if (typeof window.onready == "function") {
window.onready(holder);
}
</script>
<script type="module" async defer>
window.onready = async function (holder) {
await holder.load(
"amitaro",
await (await fetch("/models/amitaro.sbv2")).arrayBuffer(),
);
const wave = await holder.synthesize("amitaro", "おはよう");
console.log(wave);
};
</script>
</head>
<body>
<div id="root"></div>
</body>
</html>

9
crates/sbv2_wasm/example.js → sbv2_wasm/example.js
View File

@@ -3,12 +3,9 @@ import fs from "node:fs/promises";
ModelHolder.globalInit(await fs.readFile("./dist/sbv2_wasm_bg.wasm"));
const holder = await ModelHolder.create(
(await fs.readFile("../../models/tokenizer.json")).toString("utf-8"),
await fs.readFile("../../models/deberta.onnx"),
);
await holder.load(
"tsukuyomi",
await fs.readFile("../../models/tsukuyomi.sbv2"),
(await fs.readFile("../models/tokenizer.json")).toString("utf-8"),
await fs.readFile("../models/deberta.onnx"),
);
await holder.load("tsukuyomi", await fs.readFile("../models/iroha2.sbv2"));
await fs.writeFile("out.wav", await holder.synthesize("tsukuyomi", "おはよう"));
holder.unload("tsukuyomi");

14
crates/sbv2_wasm/package.json → sbv2_wasm/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "sbv2",
"version": "0.2.0-alpha6",
"version": "0.1.1",
"description": "Style Bert VITS2 wasm",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -13,13 +13,13 @@
"author": "tuna2134",
"license": "MIT",
"devDependencies": {
"@biomejs/biome": "^1.9.4",
"@types/node": "^22.13.5",
"esbuild": "^0.25.0",
"typescript": "^5.7.3"
"@biomejs/biome": "^1.9.2",
"@types/node": "^22.7.4",
"esbuild": "^0.24.0",
"typescript": "^5.6.2"
},
"dependencies": {
"onnxruntime-web": "^1.20.1"
"onnxruntime-web": "^1.19.2"
},
"files": ["dist/*", "package.json", "README.md", "pkg/*.ts", "pkg/*.js"]
"files": ["dist/*", "package.json", "README.md"]
}

494
sbv2_wasm/pnpm-lock.yaml generated Normal file
View File

@@ -0,0 +1,494 @@
lockfileVersion: '9.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
importers:
.:
dependencies:
onnxruntime-web:
specifier: ^1.19.2
version: 1.19.2
devDependencies:
'@biomejs/biome':
specifier: ^1.9.2
version: 1.9.3
'@types/node':
specifier: ^22.7.4
version: 22.7.4
esbuild:
specifier: ^0.24.0
version: 0.24.0
typescript:
specifier: ^5.6.2
version: 5.6.2
packages:
'@biomejs/biome@1.9.3':
resolution: {integrity: sha512-POjAPz0APAmX33WOQFGQrwLvlu7WLV4CFJMlB12b6ZSg+2q6fYu9kZwLCOA+x83zXfcPd1RpuWOKJW0GbBwLIQ==}
engines: {node: '>=14.21.3'}
hasBin: true
'@biomejs/cli-darwin-arm64@1.9.3':
resolution: {integrity: sha512-QZzD2XrjJDUyIZK+aR2i5DDxCJfdwiYbUKu9GzkCUJpL78uSelAHAPy7m0GuPMVtF/Uo+OKv97W3P9nuWZangQ==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [darwin]
'@biomejs/cli-darwin-x64@1.9.3':
resolution: {integrity: sha512-vSCoIBJE0BN3SWDFuAY/tRavpUtNoqiceJ5PrU3xDfsLcm/U6N93JSM0M9OAiC/X7mPPfejtr6Yc9vSgWlEgVw==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [darwin]
'@biomejs/cli-linux-arm64-musl@1.9.3':
resolution: {integrity: sha512-VBzyhaqqqwP3bAkkBrhVq50i3Uj9+RWuj+pYmXrMDgjS5+SKYGE56BwNw4l8hR3SmYbLSbEo15GcV043CDSk+Q==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [linux]
'@biomejs/cli-linux-arm64@1.9.3':
resolution: {integrity: sha512-vJkAimD2+sVviNTbaWOGqEBy31cW0ZB52KtpVIbkuma7PlfII3tsLhFa+cwbRAcRBkobBBhqZ06hXoZAN8NODQ==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [linux]
'@biomejs/cli-linux-x64-musl@1.9.3':
resolution: {integrity: sha512-TJmnOG2+NOGM72mlczEsNki9UT+XAsMFAOo8J0me/N47EJ/vkLXxf481evfHLlxMejTY6IN8SdRSiPVLv6AHlA==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [linux]
'@biomejs/cli-linux-x64@1.9.3':
resolution: {integrity: sha512-x220V4c+romd26Mu1ptU+EudMXVS4xmzKxPVb9mgnfYlN4Yx9vD5NZraSx/onJnd3Gh/y8iPUdU5CDZJKg9COA==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [linux]
'@biomejs/cli-win32-arm64@1.9.3':
resolution: {integrity: sha512-lg/yZis2HdQGsycUvHWSzo9kOvnGgvtrYRgoCEwPBwwAL8/6crOp3+f47tPwI/LI1dZrhSji7PNsGKGHbwyAhw==}
engines: {node: '>=14.21.3'}
cpu: [arm64]
os: [win32]
'@biomejs/cli-win32-x64@1.9.3':
resolution: {integrity: sha512-cQMy2zanBkVLpmmxXdK6YePzmZx0s5Z7KEnwmrW54rcXK3myCNbQa09SwGZ8i/8sLw0H9F3X7K4rxVNGU8/D4Q==}
engines: {node: '>=14.21.3'}
cpu: [x64]
os: [win32]
'@esbuild/aix-ppc64@0.24.0':
resolution: {integrity: sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [aix]
'@esbuild/android-arm64@0.24.0':
resolution: {integrity: sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==}
engines: {node: '>=18'}
cpu: [arm64]
os: [android]
'@esbuild/android-arm@0.24.0':
resolution: {integrity: sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==}
engines: {node: '>=18'}
cpu: [arm]
os: [android]
'@esbuild/android-x64@0.24.0':
resolution: {integrity: sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [android]
'@esbuild/darwin-arm64@0.24.0':
resolution: {integrity: sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [darwin]
'@esbuild/darwin-x64@0.24.0':
resolution: {integrity: sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==}
engines: {node: '>=18'}
cpu: [x64]
os: [darwin]
'@esbuild/freebsd-arm64@0.24.0':
resolution: {integrity: sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==}
engines: {node: '>=18'}
cpu: [arm64]
os: [freebsd]
'@esbuild/freebsd-x64@0.24.0':
resolution: {integrity: sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [freebsd]
'@esbuild/linux-arm64@0.24.0':
resolution: {integrity: sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==}
engines: {node: '>=18'}
cpu: [arm64]
os: [linux]
'@esbuild/linux-arm@0.24.0':
resolution: {integrity: sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==}
engines: {node: '>=18'}
cpu: [arm]
os: [linux]
'@esbuild/linux-ia32@0.24.0':
resolution: {integrity: sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==}
engines: {node: '>=18'}
cpu: [ia32]
os: [linux]
'@esbuild/linux-loong64@0.24.0':
resolution: {integrity: sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==}
engines: {node: '>=18'}
cpu: [loong64]
os: [linux]
'@esbuild/linux-mips64el@0.24.0':
resolution: {integrity: sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==}
engines: {node: '>=18'}
cpu: [mips64el]
os: [linux]
'@esbuild/linux-ppc64@0.24.0':
resolution: {integrity: sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [linux]
'@esbuild/linux-riscv64@0.24.0':
resolution: {integrity: sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==}
engines: {node: '>=18'}
cpu: [riscv64]
os: [linux]
'@esbuild/linux-s390x@0.24.0':
resolution: {integrity: sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==}
engines: {node: '>=18'}
cpu: [s390x]
os: [linux]
'@esbuild/linux-x64@0.24.0':
resolution: {integrity: sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==}
engines: {node: '>=18'}
cpu: [x64]
os: [linux]
'@esbuild/netbsd-x64@0.24.0':
resolution: {integrity: sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==}
engines: {node: '>=18'}
cpu: [x64]
os: [netbsd]
'@esbuild/openbsd-arm64@0.24.0':
resolution: {integrity: sha512-MD9uzzkPQbYehwcN583yx3Tu5M8EIoTD+tUgKF982WYL9Pf5rKy9ltgD0eUgs8pvKnmizxjXZyLt0z6DC3rRXg==}
engines: {node: '>=18'}
cpu: [arm64]
os: [openbsd]
'@esbuild/openbsd-x64@0.24.0':
resolution: {integrity: sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==}
engines: {node: '>=18'}
cpu: [x64]
os: [openbsd]
'@esbuild/sunos-x64@0.24.0':
resolution: {integrity: sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==}
engines: {node: '>=18'}
cpu: [x64]
os: [sunos]
'@esbuild/win32-arm64@0.24.0':
resolution: {integrity: sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==}
engines: {node: '>=18'}
cpu: [arm64]
os: [win32]
'@esbuild/win32-ia32@0.24.0':
resolution: {integrity: sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==}
engines: {node: '>=18'}
cpu: [ia32]
os: [win32]
'@esbuild/win32-x64@0.24.0':
resolution: {integrity: sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==}
engines: {node: '>=18'}
cpu: [x64]
os: [win32]
'@protobufjs/aspromise@1.1.2':
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
'@protobufjs/base64@1.1.2':
resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
'@protobufjs/codegen@2.0.4':
resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==}
'@protobufjs/eventemitter@1.1.0':
resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
'@protobufjs/fetch@1.1.0':
resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
'@protobufjs/float@1.0.2':
resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
'@protobufjs/inquire@1.1.0':
resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==}
'@protobufjs/path@1.1.2':
resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
'@protobufjs/pool@1.1.0':
resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
'@protobufjs/utf8@1.1.0':
resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
'@types/node@22.7.4':
resolution: {integrity: sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==}
esbuild@0.24.0:
resolution: {integrity: sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==}
engines: {node: '>=18'}
hasBin: true
flatbuffers@1.12.0:
resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==}
guid-typescript@1.0.9:
resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
long@5.2.3:
resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==}
onnxruntime-common@1.19.2:
resolution: {integrity: sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q==}
onnxruntime-web@1.19.2:
resolution: {integrity: sha512-r0ok6KpTUXR4WA+rHvUiZn7JoH02e8iS7XE1p5bXk7q3E0UaRFfYvpMNUHqEPiTBMuIssfBxDCQjUihV8dDFPg==}
platform@1.3.6:
resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
protobufjs@7.4.0:
resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==}
engines: {node: '>=12.0.0'}
typescript@5.6.2:
resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}
engines: {node: '>=14.17'}
hasBin: true
undici-types@6.19.8:
resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
snapshots:
'@biomejs/biome@1.9.3':
optionalDependencies:
'@biomejs/cli-darwin-arm64': 1.9.3
'@biomejs/cli-darwin-x64': 1.9.3
'@biomejs/cli-linux-arm64': 1.9.3
'@biomejs/cli-linux-arm64-musl': 1.9.3
'@biomejs/cli-linux-x64': 1.9.3
'@biomejs/cli-linux-x64-musl': 1.9.3
'@biomejs/cli-win32-arm64': 1.9.3
'@biomejs/cli-win32-x64': 1.9.3
'@biomejs/cli-darwin-arm64@1.9.3':
optional: true
'@biomejs/cli-darwin-x64@1.9.3':
optional: true
'@biomejs/cli-linux-arm64-musl@1.9.3':
optional: true
'@biomejs/cli-linux-arm64@1.9.3':
optional: true
'@biomejs/cli-linux-x64-musl@1.9.3':
optional: true
'@biomejs/cli-linux-x64@1.9.3':
optional: true
'@biomejs/cli-win32-arm64@1.9.3':
optional: true
'@biomejs/cli-win32-x64@1.9.3':
optional: true
'@esbuild/aix-ppc64@0.24.0':
optional: true
'@esbuild/android-arm64@0.24.0':
optional: true
'@esbuild/android-arm@0.24.0':
optional: true
'@esbuild/android-x64@0.24.0':
optional: true
'@esbuild/darwin-arm64@0.24.0':
optional: true
'@esbuild/darwin-x64@0.24.0':
optional: true
'@esbuild/freebsd-arm64@0.24.0':
optional: true
'@esbuild/freebsd-x64@0.24.0':
optional: true
'@esbuild/linux-arm64@0.24.0':
optional: true
'@esbuild/linux-arm@0.24.0':
optional: true
'@esbuild/linux-ia32@0.24.0':
optional: true
'@esbuild/linux-loong64@0.24.0':
optional: true
'@esbuild/linux-mips64el@0.24.0':
optional: true
'@esbuild/linux-ppc64@0.24.0':
optional: true
'@esbuild/linux-riscv64@0.24.0':
optional: true
'@esbuild/linux-s390x@0.24.0':
optional: true
'@esbuild/linux-x64@0.24.0':
optional: true
'@esbuild/netbsd-x64@0.24.0':
optional: true
'@esbuild/openbsd-arm64@0.24.0':
optional: true
'@esbuild/openbsd-x64@0.24.0':
optional: true
'@esbuild/sunos-x64@0.24.0':
optional: true
'@esbuild/win32-arm64@0.24.0':
optional: true
'@esbuild/win32-ia32@0.24.0':
optional: true
'@esbuild/win32-x64@0.24.0':
optional: true
'@protobufjs/aspromise@1.1.2': {}
'@protobufjs/base64@1.1.2': {}
'@protobufjs/codegen@2.0.4': {}
'@protobufjs/eventemitter@1.1.0': {}
'@protobufjs/fetch@1.1.0':
dependencies:
'@protobufjs/aspromise': 1.1.2
'@protobufjs/inquire': 1.1.0
'@protobufjs/float@1.0.2': {}
'@protobufjs/inquire@1.1.0': {}
'@protobufjs/path@1.1.2': {}
'@protobufjs/pool@1.1.0': {}
'@protobufjs/utf8@1.1.0': {}
'@types/node@22.7.4':
dependencies:
undici-types: 6.19.8
esbuild@0.24.0:
optionalDependencies:
'@esbuild/aix-ppc64': 0.24.0
'@esbuild/android-arm': 0.24.0
'@esbuild/android-arm64': 0.24.0
'@esbuild/android-x64': 0.24.0
'@esbuild/darwin-arm64': 0.24.0
'@esbuild/darwin-x64': 0.24.0
'@esbuild/freebsd-arm64': 0.24.0
'@esbuild/freebsd-x64': 0.24.0
'@esbuild/linux-arm': 0.24.0
'@esbuild/linux-arm64': 0.24.0
'@esbuild/linux-ia32': 0.24.0
'@esbuild/linux-loong64': 0.24.0
'@esbuild/linux-mips64el': 0.24.0
'@esbuild/linux-ppc64': 0.24.0
'@esbuild/linux-riscv64': 0.24.0
'@esbuild/linux-s390x': 0.24.0
'@esbuild/linux-x64': 0.24.0
'@esbuild/netbsd-x64': 0.24.0
'@esbuild/openbsd-arm64': 0.24.0
'@esbuild/openbsd-x64': 0.24.0
'@esbuild/sunos-x64': 0.24.0
'@esbuild/win32-arm64': 0.24.0
'@esbuild/win32-ia32': 0.24.0
'@esbuild/win32-x64': 0.24.0
flatbuffers@1.12.0: {}
guid-typescript@1.0.9: {}
long@5.2.3: {}
onnxruntime-common@1.19.2: {}
onnxruntime-web@1.19.2:
dependencies:
flatbuffers: 1.12.0
guid-typescript: 1.0.9
long: 5.2.3
onnxruntime-common: 1.19.2
platform: 1.3.6
protobufjs: 7.4.0
platform@1.3.6: {}
protobufjs@7.4.0:
dependencies:
'@protobufjs/aspromise': 1.1.2
'@protobufjs/base64': 1.1.2
'@protobufjs/codegen': 2.0.4
'@protobufjs/eventemitter': 1.1.0
'@protobufjs/fetch': 1.1.0
'@protobufjs/float': 1.0.2
'@protobufjs/inquire': 1.1.0
'@protobufjs/path': 1.1.2
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
'@types/node': 22.7.4
long: 5.2.3
typescript@5.6.2: {}
undici-types@6.19.8: {}

2
crates/sbv2_wasm/src-js/index.ts → sbv2_wasm/src-js/index.ts
View File

@@ -74,8 +74,6 @@ export class ModelHolder {
style_vec: e,
sdp_ratio: new Tensor("float32", [f]),
length_scale: new Tensor("float32", [g]),
noise_scale: new Tensor("float32", [0.677]),
noise_scale_w: new Tensor("float32", [0.8]),
})
).output;
return [new Uint32Array(res.dims), await res.getData(true)];

0
crates/sbv2_wasm/src/array_helper.rs → sbv2_wasm/src/array_helper.rs
View File

0
crates/sbv2_wasm/src/lib.rs → sbv2_wasm/src/lib.rs
View File

0
crates/sbv2_wasm/tsconfig.json → sbv2_wasm/tsconfig.json
View File

5
scripts/.gitignore vendored
View File

@@ -1,5 +0,0 @@
*.json
venv/
tmp/
*.safetensors
*.npy

14
scripts/make_dict.sh
View File

@@ -1,14 +0,0 @@
#!/bin/bash
set -e
git clone https://github.com/Aivis-Project/AivisSpeech-Engine ./scripts/tmp --filter=blob:none -n
cd ./scripts/tmp
git checkout 168b2a1144afe300b0490d9a6dd773ec6e927667 -- resources/dictionaries/*.csv
cd ../..
rm -rf ./crates/sbv2_core/src/dic
cp -r ./scripts/tmp/resources/dictionaries ./crates/sbv2_core/src/dic
rm -rf ./scripts/tmp
for file in ./crates/sbv2_core/src/dic/0*.csv; do
/usr/bin/cat "$file"
echo
done > ./crates/sbv2_core/src/all.csv
lindera build ./crates/sbv2_core/src/all.csv ./crates/sbv2_core/src/dic/all.dic -u -k ipadic

180
scripts/sbv2-bindings-colab.ipynb
View File

@@ -1,180 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 音声合成プログラム\n",
"\n",
"このノートブックでは、`sbv2_bindings` パッケージを使用して音声合成を行います。必要なモデルをダウンロードし、ユーザーが入力したテキストから音声を生成します。音声合成が終わったら、再度テキストの入力を求め、ユーザーが終了するまで繰り返します。"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# 必要なパッケージのインストール\n",
"%pip install sbv2_bindings\n",
"\n",
"# 必要なモジュールのインポート\n",
"import os\n",
"import urllib.request\n",
"import time\n",
"from sbv2_bindings import TTSModel"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## モデルのダウンロード\n",
"\n",
"モデルファイルとトークナイザーをダウンロードします。ユーザーが独自のモデルを使用したい場合は、該当するURLまたはローカルパスを指定してください。"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# モデルの URL またはローカルパスの指定\n",
"user_sbv2_model_url = \"\" # カスタムモデルのURLがあればここに指定\n",
"user_sbv2_model_path = \"\" # カスタムモデルのローカルパスがあればここに指定\n",
"\n",
"# モデル用のディレクトリを作成\n",
"model_dir = 'models'\n",
"os.makedirs(model_dir, exist_ok=True)\n",
"\n",
"# ダウンロードするファイルの URL\n",
"file_urls = [\n",
" \"https://huggingface.co/googlefan/sbv2_onnx_models/resolve/main/tokenizer.json\",\n",
" \"https://huggingface.co/googlefan/sbv2_onnx_models/resolve/main/deberta.onnx\",\n",
"]\n",
"\n",
"# モデルのパス決定\n",
"if user_sbv2_model_path:\n",
" sbv2_model_path = user_sbv2_model_path # ローカルモデルのパスを使用\n",
"elif user_sbv2_model_url:\n",
" sbv2_model_filename = os.path.basename(user_sbv2_model_url)\n",
" sbv2_model_path = os.path.join(model_dir, sbv2_model_filename)\n",
" file_urls.append(user_sbv2_model_url)\n",
"else:\n",
" # デフォルトのモデルを使用\n",
" sbv2_model_filename = \"tsukuyomi.sbv2\"\n",
" sbv2_model_path = os.path.join(model_dir, sbv2_model_filename)\n",
" file_urls.append(\"https://huggingface.co/googlefan/sbv2_onnx_models/resolve/main/tsukuyomi.sbv2\")\n",
"\n",
"# ファイルをダウンロード\n",
"for url in file_urls:\n",
" file_name = os.path.join(model_dir, os.path.basename(url))\n",
" if not os.path.exists(file_name):\n",
" print(f\"{file_name} をダウンロードしています...\")\n",
" urllib.request.urlretrieve(url, file_name)\n",
" else:\n",
" print(f\"{file_name} は既に存在します。\")\n",
"\n",
"# ダウンロードまたは使用するファイルを確認\n",
"print(\"\\n使用するファイル:\")\n",
"for file in os.listdir(model_dir):\n",
" print(file)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## モデルの読み込みと音声合成\n",
"\n",
"モデルを読み込み、ユーザーが入力したテキストから音声を生成します。話者名は使用する `.sbv2` ファイル名から自動的に取得します。音声合成が終わったら、再度テキストの入力を求め、ユーザーが終了するまで繰り返します。"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# 音声合成の実行\n",
"def main():\n",
" try:\n",
" print(\"\\nモデルを読み込んでいます...\")\n",
" model = TTSModel.from_path(\n",
" os.path.join(model_dir, \"deberta.onnx\"),\n",
" os.path.join(model_dir, \"tokenizer.json\")\n",
" )\n",
" print(\"モデルの読み込みが完了しました！\")\n",
" except Exception as e:\n",
" print(f\"モデルの読み込みに失敗しました: {e}\")\n",
" return\n",
"\n",
" # 話者名を取得（.sbv2 ファイル名の拡張子を除いた部分）\n",
" speaker_name = os.path.splitext(os.path.basename(sbv2_model_path))[0]\n",
" \n",
" # 指定されたモデルのパスを使用\n",
" try:\n",
" model.load_sbv2file_from_path(speaker_name, sbv2_model_path)\n",
" print(f\"話者 '{speaker_name}' のセットアップが完了しました！\")\n",
" except Exception as e:\n",
" print(f\"SBV2ファイルの読み込みに失敗しました: {e}\")\n",
" return\n",
"\n",
" # 音声合成を繰り返し実行\n",
" while True:\n",
" # 合成したいテキストをユーザーから入力\n",
" user_input = input(\"\\n音声合成したいテキストを入力してください（終了するには 'exit' と入力）: \")\n",
" \n",
" if user_input.strip().lower() == 'exit':\n",
" print(\"音声合成を終了します。\")\n",
" break\n",
"\n",
" # 出力ファイル名\n",
" output_file = \"output.wav\"\n",
"\n",
" # 音声合成を実行\n",
" try:\n",
" print(\"\\n音声合成を開始します...\")\n",
" start_time = time.time()\n",
"\n",
" audio_data = model.synthesize(user_input, speaker_name, 0, 0.0, 1)\n",
"\n",
" with open(output_file, \"wb\") as f:\n",
" f.write(audio_data)\n",
"\n",
" end_time = time.time()\n",
" elapsed_time = end_time - start_time\n",
"\n",
" print(f\"\\n音声が '{output_file}' に保存されました。\")\n",
" print(f\"音声合成にかかった時間: {elapsed_time:.2f} 秒\")\n",
" except Exception as e:\n",
" print(f\"音声合成に失敗しました: {e}\")\n",
"\n",
"if __name__ == \"__main__\":\n",
" main()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.x"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

8
scripts/sbv2-test-api.py
View File

@@ -1,8 +0,0 @@
import requests
res = requests.post(
"http://localhost:3000/synthesize",
json={"text": "おはようございます", "ident": "tsukuyomi"},
)
with open("output.wav", "wb") as f:
f.write(res.content)

23
test.py
View File

@@ -1,19 +1,8 @@
import requests
data = (requests.get("http://localhost:8080/audio_query", params={
"text": "こんにちは、今日はいい天気ですね。",
})).json()
print(data)
data = (requests.post("http://localhost:8080/synthesis", json={
"text": data["text"],
"ident": "tsukuyomi",
"speaker_id": 0,
"style_id": 0,
"sdp_ratio": 0.5,
"length_scale": 0.5,
"audio_query": data["audio_query"],
})).content
with open("test.wav", "wb") as f:
f.write(data)
res = requests.post(
"http://localhost:3000/synthesize",
json={"text": "おはようございます", "ident": "tsukuyomi"},
)
with open("output.wav", "wb") as f:
f.write(res.content)