Overview

XGBoost is the most widely used gradient boosting library in ML competitions and production. It uses regularized boosted trees with level-wise growth. Excellent for structured/tabular data. Supports binary & multi-class classification, regression, and ranking. Has official C API.

Binary Classification

uses xgboost;

{ load training data from CSV (label in column 0) }
var train := MatrixFromCSV(ReadFile('train.csv'), 0);
var test  := MatrixFromCSV(ReadFile('test.csv'),  0);

{ quick start with default params }
var model := TrainDefault(train, xgbBinaryLogistic);

{ predict — returns probability for each sample }
var preds := Predict(model, test);
var i: Integer;
for i := 0 to Length(preds) - 1 do
  WriteLn('Sample ', i, ': ', preds[i]:0:3);

FreeBooster(model);
FreeMatrix(train);
FreeMatrix(test);

Custom Hyperparameters

uses xgboost;

var params := DefaultParams(xgbBinaryLogistic);
params.NumRounds  := 200;
params.MaxDepth   := 5;
params.Eta        := 0.05;    { smaller = more conservative }
params.Subsample  := 0.8;
params.ColSample  := 0.8;
params.Lambda     := 1.5;
params.EvalMetric := 'auc';

var train := MatrixFromCSV(ReadFile('train.csv'), 0);
var valid := MatrixFromCSV(ReadFile('valid.csv'), 0);

{ train with validation set (monitors eval metric) }
var result: TXGBTrainResult;
var model := TrainWithHistory(train, valid, params, result);
WriteLn('Best iteration: ', result.BestIteration);
WriteLn('Train AUC: ', result.TrainMetric:0:4);
WriteLn('Eval AUC:  ', result.EvalMetric:0:4);

Regression

uses xgboost;

var params := DefaultParams(xgbRegLinear);
params.NumRounds  := 150;
params.MaxDepth   := 4;
params.Eta        := 0.1;
params.EvalMetric := 'rmse';

var train := MatrixFromFile('housing_train.libsvm');
var model := Train(train, params, 0);

{ predict on new sample (features as float array) }
var features: array of Double;
SetLength(features, 8);
features[0] := 3.5;  { rooms }
features[1] := 2.0;  { bathrooms }
{ ... }

var price := PredictOne(model, features);
WriteLn('Predicted price: $', price:0:0);

Multi-class Classification

uses xgboost;

var params := DefaultParams(xgbMultiSoftprob);
params.NumClass  := 3;   { number of classes }
params.NumRounds := 100;

var train := MatrixFromCSV(ReadFile('iris_train.csv'), 4);
var model := Train(train, params, 0);

{ predict probabilities for each class }
var proba := PredictProba(model, MatrixFromCSV(ReadFile('iris_test.csv'), 4));
{ returns flat array: [p0_c0, p0_c1, p0_c2, p1_c0, p1_c1, p1_c2, ...] }

Feature Importance

uses xgboost;

var model := Train(train, params, 0);

{ get feature importance sorted by score }
var importance := GetFeatureImportance(model, 'gain');
WriteLn(importance);
{ feature_0  0.342
  feature_2  0.198
  feature_1  0.156
  ... }

{ top 5 most important features }
var top := GetTopFeatures(model, 5);
var i: Integer;
for i := 0 to Length(top) - 1 do
  WriteLn(top[i].Rank, '. ', top[i].Feature, '  ', top[i].Score:0:4);

Cross-Validation & Persist

uses xgboost;

{ 5-fold CV }
var cv := CrossValidate(train, params, 5);
WriteLn('CV metric: ', cv:0:4);

{ save / load }
SaveModel(model, '/models/price_predictor.json');
var loaded := LoadModel('/models/price_predictor.json');
FreeBooster(model);