addSpecialTokens method

Implementation

void addSpecialTokens(List<String> specialTokens){
   if (!available) {
    return;
  }

  final Pointer<Pointer<Utf8>> charArray =
      calloc.allocate<Pointer<Utf8>>(specialTokens.length);

  for (int i = 0; i < specialTokens.length; i++) {
    charArray[i] = specialTokens[i].toNativeUtf8();
  }

  int status = ailiaTokenizer.ailiaTokenizerAddSpecialTokens(
    ppAiliaTokenizer!.value,
    charArray,
    specialTokens.length,
  );

  for (int i = 0; i < specialTokens.length; i++) {
    calloc.free(charArray[i]);
  }
  calloc.free(charArray);

  if (status != ailiaStatusSuccess) {
    throw Exception("ailiaTokenizerAddSpecialTokens error $status");
  }
}