Overview

ONNX Runtime is Microsoft's cross-platform ML inference engine. It runs any model exported to the ONNX format — PyTorch, TensorFlow, scikit-learn, XGBoost, LightGBM, and Hugging Face Transformers. No Python or framework required at inference time.

Inspect a Model

uses onnxruntime;

var session := LoadSession('/models/classifier.onnx', 0);  { 0 = all threads }

var info := GetModelInfo(session);
WriteLn('Producer: ', info.ProducerName);

var i: Integer;
for i := 0 to Length(info.Inputs) - 1 do begin
  var inp := info.Inputs[i];
  WriteLn('Input:  ', inp.Name, '  type=', Ord(inp.ElemType),
          '  rank=', inp.Rank);
end;
for i := 0 to Length(info.Outputs) - 1 do
  WriteLn('Output: ', info.Outputs[i].Name);

WriteLn('CUDA available: ', HasCUDA());
WriteLn('Providers: ', GetProviders());

FreeSession(session);

Image Classifier (single input)

uses onnxruntime;

{ model: ResNet / EfficientNet exported from PyTorch
  input: float32[1, 3, 224, 224]  (batch=1, RGB, 224x224)
  output: float32[1, 1000]         (ImageNet logits) }

var session := LoadSession('/models/resnet50.onnx', 4);

{ load and preprocess image pixels into flat float array
  (normalize: subtract mean, divide by std — done externally via libpng/libjpeg) }
var pixels: array of Single;
{ ... fill 1*3*224*224 = 150528 floats ... }

{ run inference }
var logits := RunSimpleFloat(session, pixels, [1, 3, 224, 224]);

{ top-1 prediction }
var classIdx := Argmax(logits);
WriteLn('Predicted class: ', classIdx);

{ top-5 probabilities }
var probs := Softmax(logits);
WriteLn('Confidence: ', probs[classIdx]:0:4);

FreeSession(session);

Multi-Input Model (NLP Transformer)

uses onnxruntime, sentencepiece;

{ BERT/DistilBERT sentiment classifier exported to ONNX
  inputs: input_ids int64[1,128], attention_mask int64[1,128]
  output: logits float32[1,2]  (negative / positive) }

var session := LoadSession('/models/distilbert-sentiment.onnx', 0);
var sp      := LoadModel('/models/tokenizer.model');

var text  := 'PascalAI makes ML accessible and fun!';
var ids   := Encode(sp, text);   { array of Integer }
var mask: array of Integer;

{ pad/truncate to 128 tokens }
var inputIds: array of Int64;
SetLength(inputIds, 128);
SetLength(mask, 128);
var j: Integer;
for j := 0 to 127 do begin
  if j < Length(ids) then begin
    inputIds[j] := ids[j];
    mask[j]     := 1;
  end else begin
    inputIds[j] := 0;
    mask[j]     := 0;
  end;
end;

{ build run context }
var ctx := NewRunContext(session);
AddInputInt64(ctx, 'input_ids',      inputIds, [1, 128]);
AddInputInt64(ctx, 'attention_mask', mask,     [1, 128]);
Run(ctx);

{ read output }
var logits := GetOutputFloat(ctx, 'logits');   { [neg_score, pos_score] }
var probs  := Softmax(logits);
WriteLn('Negative: ', probs[0]:0:3);
WriteLn('Positive: ', probs[1]:0:3);

FreeRunContext(ctx);
FreeSession(session);

Tabular Model (sklearn / XGBoost ONNX)

uses onnxruntime;

{ sklearn RandomForest or XGBoost model exported with skl2onnx or to_onnx()
  input:  float32[N, num_features]
  output: int64[N]   (predicted labels)
           float32[N, num_classes]  (probabilities) }

var session := LoadSession('/models/random_forest.onnx', 0);

{ 3 samples, 10 features each }
var features: array of Single;
SetLength(features, 30);
features[0]  := 2.5;  features[1]  := 0.8; { ... }
features[10] := 1.2;  features[11] := 3.1; { ... }
features[20] := 4.0;  features[21] := 0.2; { ... }

var ctx := NewRunContext(session);
AddInputFloat(ctx, 'X', features, [3, 10]);
Run(ctx);

var labels := GetOutputInt64(ctx, 'label');
var proba  := GetOutputFloat(ctx, 'probabilities');
WriteLn('Predictions: ', labels[0], ' ', labels[1], ' ', labels[2]);

FreeRunContext(ctx);
FreeSession(session);

GPU Inference

uses onnxruntime;

{ load session on GPU device 0 (falls back to CPU if CUDA unavailable) }
if HasCUDA() then begin
  var session := LoadSessionGPU('/models/model.onnx', 0);
  { ... same API as CPU ... }
  FreeSession(session);
end else
  WriteLn('CUDA not available, using CPU');

Export Models from Python

{ PyTorch export example (run in Python): }

{ import torch
  model = MyModel()
  model.eval()
  dummy = torch.randn(1, 3, 224, 224)
  torch.onnx.export(model, dummy, 'model.onnx',
      input_names=['input'], output_names=['output'],
      dynamic_axes={'input': {0: 'batch'}}) }

{ scikit-learn export with skl2onnx:
  from skl2onnx import convert_sklearn
  from skl2onnx.common.data_types import FloatTensorType
  onx = convert_sklearn(clf, 'clf',
        initial_types=[('X', FloatTensorType([None, n_features]))])
  with open('clf.onnx', 'wb') as f: f.write(onx.SerializeToString()) }