API Usage

Overview of ailia Tokenizer API

Basic usage

This is an example of using ailia Tokenizer. By creating an instance and giving a UTF8 string as an argument, a token can be obtained.

#include <stdio.h>
#include <vector>
#include <stdint.h>
#include <stdlib.h>
 
#include "ailia_tokenizer.h"
 
int main(int argc, char *argv[]){
 int32_t type = AILIA_TOKENIZER_TYPE_WHISPER;
 printf("Tokenizer type %d\n", type);
 AILIATokenizer *net;
 ailiaTokenizerCreate(&net, type, AILIA_TOKENIZER_FLAG_NONE);
 const char * text = u8"ハードウェア ソフトウェア";
 printf("Input Text : %s\n", text);
 ailiaTokenizerEncode(net, text);
 unsigned int count;
 ailiaTokenizerGetTokenCount(net, &count);
 std::vector<int> tokens(count);
 ailiaTokenizerGetTokens(net, &tokens[0], count);
 ailiaTokenizerDecode(net, &tokens[0], count);
 printf("Tokens : ");
 for (int i = 0; i < count; i++){
  printf("%d ", tokens[i]);
 }
 printf("\n");
 unsigned int len;
 ailiaTokenizerGetTextLength(net, &len);
 std::vector<char> out_text(len);
 char * p_text = &out_text[0];
 ailiaTokenizerGetText(net, p_text, len);
 printf("Output Text : %s\n", p_text);
 return 0;
}