API usage

Overview of ailia AI Voice API

Basic usage

Below is an example of how to use the text-to-speech API in C++. After having created an AILIAVoice instance, and opened the model with ailiaVoiceOpenModelFile, use ailiaVoiceGraphemeToPhoneme to convert the text to phonemes, then use ailiaVoiceInference to perform the text-to-speech conversion, after which it is possible to get the resulting audio waveforms with ailiaVoiceGetWave. When using GPS-SoVITS, provide a reference audio file with ailiaVoiceSetReference before using ailiaVoiceInference.

#include "ailia_voice.h"
#include "ailia_voice_util.h"
 
#include <stdio.h>
#include <vector>
#include <string>
#include <string.h>
 
#include "wave_reader.h"
#include "wave_writer.h"
 
int main(int argc, char *argv[]){
    AILIAVoiceApiCallback callback = ailiaVoiceUtilGetCallback();
 
    printf("Usage : ailia_voice_sample [tacotron2/gpt-sovits/gpt-sovits-en] [input_text]\n");
 
    const char * input_text = "";
    const char * lang = "";
    const char * model = "tacotron2";
 
    if (argc >= 2){
        model = argv[1];
        if (!(strcmp(model, "tacotron2") == 0 || strcmp(model, "gpt-sovits") == 0 || strcmp(model, "gpt-sovits-en") == 0)){
            printf("model must be tacotron2 or gpt-sovits\n");
            return -1;
        }
    }
    if (argc >= 3){
        input_text = argv[2];
    }
 
    if (strcmp(model, "tacotron2") == 0 || strcmp(model, "gpt-sovits-en") == 0 ){
        if (strlen(input_text) == 0){
            input_text = u8"Hello world.";
        }
        lang = "en";
    }else{
        if (strlen(input_text) == 0){
            input_text = u8"こんにちは。今日は新しいAIエンジンであるアイリアSDKを紹介します。";
        }
        lang = "ja";
    }
 
    printf("Model : %s\n", model);
    printf("Input text : %s\n", input_text);
    printf("Language : %s\n", lang);
 
    AILIAVoice *net;
    int env_id = AILIA_ENVIRONMENT_ID_AUTO;
    int num_thread = AILIA_MULTITHREAD_AUTO;
    int memory_mode = AILIA_MEMORY_REDUCE_CONSTANT | AILIA_MEMORY_REDUCE_CONSTANT_WITH_INPUT_INITIALIZER | AILIA_MEMORY_REUSE_INTERSTAGE;
    int status = ailiaVoiceCreate(&net, env_id, num_thread, memory_mode, AILIA_VOICE_FLAG_NONE, callback, AILIA_VOICE_API_CALLBACK_VERSION);
    if (status != AILIA_STATUS_SUCCESS){
        printf("ailiaVoiceCreate error %d\n", status);
        return -1;
    }
    if (strcmp(model, "gpt-sovits") == 0 || strcmp(model, "gpt-sovits-en") == 0){
        status = ailiaVoiceOpenDictionaryFileA(net, "./open_jtalk_dic_utf_8-1.11", AILIA_VOICE_DICTIONARY_TYPE_OPEN_JTALK);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceOpenDictionaryFileA error %d\n", status);
            return -1;
        }
    }
    if (strcmp(model, "gpt-sovits-en") == 0){
        status = ailiaVoiceOpenDictionaryFileA(net, "../onnx/g2p_en", AILIA_VOICE_DICTIONARY_TYPE_G2P_EN);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceOpenDictionaryFileA error %d\n", status);
            return -1;
        }
    }
    if (strcmp(model, "tacotron2") == 0){
        status = ailiaVoiceOpenModelFileA(net, "../onnx/nvidia/encoder.onnx", "../onnx/nvidia/decoder_iter.onnx", "../onnx/nvidia/postnet.onnx", "../onnx/nvidia/waveglow.onnx", NULL, AILIA_VOICE_MODEL_TYPE_TACOTRON2, AILIA_VOICE_CLEANER_TYPE_BASIC);
    }else{
        status = ailiaVoiceOpenModelFileA(net, "../onnx/gpt-sovits/t2s_encoder.onnx", "../onnx/gpt-sovits/t2s_fsdec.onnx", "../onnx/gpt-sovits/t2s_sdec.opt3.onnx", "../onnx/gpt-sovits/vits.onnx", "../onnx/gpt-sovits/cnhubert.onnx", AILIA_VOICE_MODEL_TYPE_GPT_SOVITS, AILIA_VOICE_CLEANER_TYPE_BASIC);
    }
    if (status != AILIA_STATUS_SUCCESS){
        printf("ailiaVoiceOpenModelFileA error %d\n", status);
        return -1;
    }
 
    if (strcmp(model, "gpt-sovits") == 0 || strcmp(model, "gpt-sovits-en") == 0){
        int sampleRate, nChannels, nSamples;
        const char *ref_audio = "../onnx/gpt-sovits/reference_audio_girl.wav";
        //const char *ref_audio = "../onnx/gpt-sovits/reference_audio_man.wav";
        std::vector<float> wave = read_wave_file(ref_audio, &sampleRate, &nChannels, &nSamples);
 
        const char *ref_text = "水をマレーシアから買わなくてはならない。";
        status = ailiaVoiceGraphemeToPhoneme(net, ref_text, AILIA_VOICE_G2P_TYPE_GPT_SOVITS_JA);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceGraphemeToPhoneme error %d\n", status);
            return -1;
        }
        unsigned int len = 0;
        status = ailiaVoiceGetFeatureLength(net, &len);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceGetFeatureLength error %d\n", status);
            return -1;
        }
        std::vector<char> ref_features;
        ref_features.resize(len);
        status = ailiaVoiceGetFeatures(net, &ref_features[0], len);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceGetFeatures error %d\n", status);
            return -1;
        }
        printf("Reference Features : %s\n", &ref_features[0]);
 
        status = ailiaVoiceSetReference(net, &wave[0], wave.size() * sizeof(float), nChannels, sampleRate, &ref_features[0]);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceSetReference error %d\n", status);
            return -1;
        }
    }
 
    std::vector<char> features;
        
    if (strcmp(model, "tacotron2") == 0){
        status = ailiaVoiceInference(net, input_text);
    }else{
        if (strcmp(model, "gpt-sovits") == 0){
            status = ailiaVoiceGraphemeToPhoneme(net, input_text, AILIA_VOICE_G2P_TYPE_GPT_SOVITS_JA);
        }else{
            status = ailiaVoiceGraphemeToPhoneme(net, input_text, AILIA_VOICE_G2P_TYPE_GPT_SOVITS_EN);
        }
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceGraphemeToPhoneme error %d\n", status);
            return -1;
        }
        unsigned int len = 0;
        status = ailiaVoiceGetFeatureLength(net, &len);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceGetFeatureLength error %d\n", status);
            return -1;
        }
        features.resize(len);
        status = ailiaVoiceGetFeatures(net, &features[0], len);
        if (status != AILIA_STATUS_SUCCESS){
            printf("ailiaVoiceGetFeatures error %d\n", status);
            return -1;
        }
        printf("Features : %s\n", &features[0]);
        status = ailiaVoiceInference(net, &features[0]);
    }
 
    if (status != AILIA_STATUS_SUCCESS){
        printf("ailiaVoiceInference error %d\n", status);
        return -1;
    }
 
    unsigned int samples, channels, sampling_rate;
    status = ailiaVoiceGetWaveInfo(net, &samples, &channels, &sampling_rate);
    if (status != AILIA_STATUS_SUCCESS){
        printf("ailiaVoiceGetWaveInfo error %d\n", status);
        return -1;
    }
 
    std::vector<float> buf(samples * channels);
    status = ailiaVoiceGetWave(net, &buf[0], buf.size() * sizeof(float));
    if (status != AILIA_STATUS_SUCCESS){
        printf("ailiaVoiceGetWave error %d\n", status);
        return -1;
    }
 
    printf("Wave samples : %d\nWave channles : %d\nWave sampling rate : %d\n", samples, channels, sampling_rate);
 
    write_wave_file("output.wav", buf, sampling_rate);
 
    ailiaVoiceDestroy(net);
    return 0;
}

Using the User Dictionary

The userdic.dic created with pyopenjtalk can be loaded by executing the ailiaVoiceSetUserDictionaryFile API before the ailiaVoiceOpenDictionaryFile API.

ailiaVoiceSetUserDictionaryFileA(net, "./userdic/userdic.dic", AILIA_VOICE_DICTIONARY_TYPE_OPEN_JTALK);
ailiaVoiceOpenDictionaryFileA(net, "./open_jtalk_dic_utf_8-1.11", AILIA_VOICE_DICTIONARY_TYPE_OPEN_JTALK);

GPU usage

In order to use the GPU, pass the env_id corresponding to the GPU as the env_id argument of ailiaVoiceCreate. By default, the value AILIA_ENVIRONMENT_ID_AUTO is used, which indicates to perform the inference on the CPU. See ailia_speech_sample.cpp as an example of how to determine the GPU env_id to be passed as the env_id argument.