Overview

SentencePiece is Google's language-independent subword tokenizer. Used by LLaMA, Gemma, T5, ALBERT, and most production LLMs. Implements BPE and Unigram language model tokenization. A .model file (distributed with each LLM) is required — it defines the vocabulary.

Encode & Decode

uses sentencepiece;

{ load tokenizer model }
var sp := LoadModel('/models/llama/tokenizer.model');

{ encode text to token IDs }
var ids := Encode(sp, 'Hello, world! How are you?');
var i: Integer;
for i := 0 to Length(ids) - 1 do
  Write(ids[i], ' ');   { 1 15043 29892 3186 29991 1128 526 366 29973 }
WriteLn;

{ decode back to text }
var text := Decode(sp, ids);
WriteLn(text);   { Hello, world! How are you? }

FreeModel(sp);

Pieces (Subword Tokens)

uses sentencepiece;

var sp := LoadModel('/models/tokenizer.model');

{ get token strings instead of IDs }
var pieces := EncodeAsPieces(sp, 'tokenization is powerful');
var i: Integer;
for i := 0 to Length(pieces) - 1 do
  Write('[', pieces[i], '] ');
{ [_token] [ization] [_is] [_power] [ful] }
WriteLn;

{ full result: IDs + pieces + byte offsets }
var result := EncodeEx(sp, 'Hello world');
for i := 0 to Length(result.Ids) - 1 do
  WriteLn(result.Ids[i], #9, result.Pieces[i], #9, 'offset:', result.Offsets[i]);

Special Tokens (BOS / EOS)

uses sentencepiece;

var sp     := LoadModel('/models/tokenizer.model');
var info   := GetModelInfo(sp);

WriteLn('Vocab size: ', info.VocabSize);
WriteLn('BOS id:     ', info.BOSId);
WriteLn('EOS id:     ', info.EOSId);

{ encode with BOS + EOS (required by many LLMs for prompts) }
var ids := EncodeWithSpecial(sp, 'Hello!', True, True);
{ [1, 15043, 29991, 2]  — 1=BOS, 2=EOS }

{ check if an ID is a special/control token }
WriteLn(IsControl(sp, info.BOSId));   { True }
WriteLn(IsControl(sp, 15043));        { False }

Token Counting & Budget

uses sentencepiece;

var sp := LoadModel('/models/tokenizer.model');

{ count tokens without materializing them — very fast }
var n := CountTokens(sp, longText);
WriteLn('Token count: ', n);

{ check if text fits within a limit (e.g. 4096 for LLaMA context) }
if FitsInBudget(sp, longText, 4096) then
  WriteLn('Fits in context window')
else begin
  { truncate to max tokens }
  var truncated := TruncateToTokens(sp, longText, 4000);
  WriteLn('Truncated to: ', CountTokens(sp, truncated), ' tokens');
end;

Vocabulary Lookup

uses sentencepiece;

var sp := LoadModel('/models/tokenizer.model');

{ ID to piece }
WriteLn(IdToPiece(sp, 15043));   { Hello }
WriteLn(GetPiece(sp, 1));        { <s>  (BOS) }

{ piece to ID }
var id := PieceToId(sp, 'world');
WriteLn(id);   { 3186 }

{ unknown pieces return -1 }
WriteLn(PieceToId(sp, 'xyzzyblorp'));   { -1 }

WriteLn('Vocab size: ', GetVocabSize(sp));

Batch Encoding

uses sentencepiece;

var sp := LoadModel('/models/tokenizer.model');

var texts: array of string;
SetLength(texts, 3);
texts[0] := 'The quick brown fox';
texts[1] := 'jumps over the lazy dog';
texts[2] := 'Hello world';

{ encode all at once — returns space-separated IDs per line }
var batch := EncodeBatch(sp, texts);
var i: Integer;
for i := 0 to Length(batch) - 1 do
  WriteLn('Line ', i, ': ', batch[i]);