encode method

Implementation

Int32List encode(String text) {
  int status;

  if (!available) {
    throw Exception("Model not opened");
  }

  Pointer<Int8> nativeText = text.toNativeUtf8().cast<Int8>();
  status = ailiaTokenizer.ailiaTokenizerEncode(
    ppAiliaTokenizer!.value,
    nativeText,
  );
  calloc.free(nativeText);
  if (status != ailiaStatusSuccess) {
    throw Exception("ailiaTokenizerEncode error $status");
  }

  final Pointer<Uint32> count = malloc<Uint32>();
  count.value = 0;
  status = ailiaTokenizer.ailiaTokenizerGetTokenCount(
    ppAiliaTokenizer!.value,
    count,
  );
  if (status != ailiaStatusSuccess) {
    throw Exception("ailiaTokenizerGetTokenCount error $status");
  }

  Pointer<Int32> tokens = malloc<Int32>(count.value);
  status = ailiaTokenizer.ailiaTokenizerGetTokens(
    ppAiliaTokenizer!.value,
    tokens,
    count.value,
  );
  if (status != ailiaStatusSuccess) {
    throw Exception("ailiaTokenizerGetTokens error $status");
  }

  Int32List tokensF = Int32List(count.value);
  for (int i = 0; i < count.value; i++) {
    tokensF[i] = tokens[i];
  }

  malloc.free(tokens);
  malloc.free(count);

  return tokensF;
}