ailia_tokenizer.h

Go to the documentation of this file.

 
#ifndef INCLUDED_AILIA_TOKENIZER
#define INCLUDED_AILIA_TOKENIZER
 
/* エラーコードと構造体 */
 
#include "ailia.h"
 
/* 呼び出し規約 */
 
#if defined(_WIN64) || defined(_M_X64) || defined(__amd64__) || defined(__x86_64__) || defined(__APPLE__) || \
    defined(__ANDROID__) || defined(ANDROID) || defined(__linux__) || defined(NN_NINTENDO_SDK)
#define AILIA_API
#else
#define AILIA_API __stdcall
#endif
 
#ifdef __cplusplus
extern "C" {
#endif
 
/****************************************************************
 * アルゴリズム定義
 **/
 
#define AILIA_TOKENIZER_TYPE_WHISPER (0)
 
#define AILIA_TOKENIZER_TYPE_CLIP (1)
 
#define AILIA_TOKENIZER_TYPE_XLM_ROBERTA (2)
 
#define AILIA_TOKENIZER_TYPE_MARIAN (3)
 
#define AILIA_TOKENIZER_TYPE_BERT_JAPANESE_WORDPIECE (4)
 
#define AILIA_TOKENIZER_TYPE_BERT_JAPANESE_CHARACTER (5)
 
#define AILIA_TOKENIZER_TYPE_T5 (6)
 
#define AILIA_TOKENIZER_TYPE_ROBERTA (7)
 
#define AILIA_TOKENIZER_TYPE_BERT (8)
 
#define AILIA_TOKENIZER_TYPE_GPT2 (9)
 
#define AILIA_TOKENIZER_TYPE_LLAMA (10)
 
/****************************************************************
 * フラグ定義
 **/
 
#define AILIA_TOKENIZER_FLAG_NONE (0)
 
#define AILIA_TOKENIZER_FLAG_UTF8_SAFE (1)
 
/****************************************************************
 * トークナイズオブジェクトのインスタンス
 **/
 
struct AILIATokenizer;
 
/****************************************************************
 * Tokenizer API
 **/
 
int AILIA_API ailiaTokenizerCreate(struct AILIATokenizer** net, int type, int flags);
 
int AILIA_API ailiaTokenizerOpenModelFileA(struct AILIATokenizer* net, const char *path);
int AILIA_API ailiaTokenizerOpenModelFileW(struct AILIATokenizer* net, const wchar_t *path);
 
int AILIA_API ailiaTokenizerOpenDictionaryFileA(struct AILIATokenizer* net, const char *path);
int AILIA_API ailiaTokenizerOpenDictionaryFileW(struct AILIATokenizer* net, const wchar_t *path);
 
int AILIA_API ailiaTokenizerOpenVocabFileA(struct AILIATokenizer* net, const char *path);
int AILIA_API ailiaTokenizerOpenVocabFileW(struct AILIATokenizer* net, const wchar_t *path);
 
int AILIA_API ailiaTokenizerOpenMergeFileA(struct AILIATokenizer* net, const char *path);
int AILIA_API ailiaTokenizerOpenMergeFileW(struct AILIATokenizer* net, const wchar_t *path);
 
int AILIA_API ailiaTokenizerOpenAddedTokensFileA(struct AILIATokenizer* net, const char *path);
int AILIA_API ailiaTokenizerOpenAddedTokensFileW(struct AILIATokenizer* net, const wchar_t *path);
 
int AILIA_API ailiaTokenizerOpenTokenizerConfigFileA(struct AILIATokenizer* net, const char *path);
int AILIA_API ailiaTokenizerOpenTokenizerConfigFileW(struct AILIATokenizer* net, const wchar_t *path);
 
int AILIA_API
ailiaTokenizerEncode(struct AILIATokenizer* net, const char *utf8);
 
int AILIA_API
ailiaTokenizerEncodeWithSpecialTokens(struct AILIATokenizer* net, const char *utf8);
 
int AILIA_API ailiaTokenizerGetTokenCount(struct AILIATokenizer* net, unsigned int* count);
 
int AILIA_API ailiaTokenizerGetTokens(struct AILIATokenizer* net, int* tokens, unsigned int count);
 
int AILIA_API ailiaTokenizerGetWordIDs(struct AILIATokenizer* net, int* word_ids, unsigned int count);
 
int AILIA_API ailiaTokenizerGetCharStarts(struct AILIATokenizer* net, int* char_starts, unsigned int count);
 
int AILIA_API ailiaTokenizerGetCharEnds(struct AILIATokenizer* net, int* char_ends, unsigned int count);
 
int AILIA_API
ailiaTokenizerDecode(struct AILIATokenizer* net, const int *tokens, unsigned int token_count);
 
int AILIA_API
ailiaTokenizerDecodeWithSpecialTokens(struct AILIATokenizer* net, const int *tokens, unsigned int token_count);
 
int AILIA_API ailiaTokenizerGetTextLength(struct AILIATokenizer* net, unsigned int* len);
 
int AILIA_API ailiaTokenizerGetText(struct AILIATokenizer* net, char* text, unsigned int len);
 
int AILIA_API ailiaTokenizerGetVocabSize(struct AILIATokenizer* net, unsigned int* size);
 
int AILIA_API
ailiaTokenizerGetVocab(struct AILIATokenizer* net, int token, const char **vocab);
 
int AILIA_API
ailiaTokenizerAddSpecialTokens(struct AILIATokenizer* net, const char **tokens, unsigned int count);
 
void AILIA_API ailiaTokenizerDestroy(struct AILIATokenizer* net);
 
/****************************************************************
 * Utility API
 **/
 
int AILIA_API ailiaTokenizerUtf8ToUtf32(unsigned int* utf32, unsigned int* processed_byte, const char* utf8, unsigned int utf8_len);
 
int AILIA_API ailiaTokenizerUtf32ToUtf8(char* utf8, unsigned int* processed_byte, unsigned int utf32);
 
#ifdef UNICODE
#define ailiaTokenizerOpenModelFile ailiaTokenizerOpenModelFileW
#define ailiaTokenizerOpenDictionaryFile ailiaTokenizerOpenDictionaryFileW
#define ailiaTokenizerOpenVocabFile ailiaTokenizerOpenVocabFileW
#define ailiaTokenizerOpenMergeFile ailiaTokenizerOpenMergeFileW
#define ailiaTokenizerOpenAddedTokensFile ailiaTokenizerOpenAddedTokensFileW
#define ailiaTokenizerOpenConfigFile ailiaTokenizerOpenConfigFileW
#else
#define ailiaTokenizerOpenModelFile ailiaTokenizerOpenModelFileA
#define ailiaTokenizerOpenDictionaryFile ailiaTokenizerOpenDictionaryFileA
#define ailiaTokenizerOpenVocabFile ailiaTokenizerOpenVocabFileA
#define ailiaTokenizerOpenMergeFile ailiaTokenizerOpenMergeFileA
#define ailiaTokenizerOpenAddedTokensFile ailiaTokenizerOpenAddedTokensFileA
#define ailiaTokenizerOpenTokenizerConfigFile ailiaTokenizerOpenTokenizerConfigFileA
#endif
 
#ifdef __cplusplus
}
#endif
#endif /* !defined(INCLUDED_AILIA_TOKENIZER) */