Anonymous methods in testing / profiling situations

One situation in which anonymous methods are particularly useful is for ad-hoc profiling and testing scenarios. For both profiling and testing, it is code itself that one wants to work with. I.e. the question you are trying to answer is "test this code", or "time this code". Using code as a parameter is a natural fit for this problem.

I'll start with a very simple profile harness. It does little more than run a code block a number of times and report on how long it took. Here's the definition of the class:

  TBenchmarker = class
  private
    const
      DefaultIterations = 3;
      DefaultWarmups = 1;
    var
      FReportSink: TProc<string,Double>;
      FWarmups: Integer;
      FIterations: Integer;
      FOverhead: Double;
    class var
      FFreq: Int64;
    class procedure InitFreq;
  public
    constructor Create(const AReportSink: TProc<string,Double>);

    class function Benchmark(const Code: TProc; 
      Iterations: Integer = DefaultIterations; 
      Warmups: Integer = DefaultWarmups): Double; overload;

    procedure Benchmark(const Name: string; const Code: TProc); overload;
    function Benchmark<T>(const Name: string; const Code: TFunc<T>): T; overload;

    property Warmups: Integer read FWarmups write FWarmups;
    property Iterations: Integer read FIterations write FIterations;
  end;

This API has a couple of interesting features. The constructor takes method reference argument indicating where output - the results of the benchmarking - will go. This idiom makes it easy for users of the class to direct the output. It's loosely coupled, just like an event (after all, an event - a method pointer - is almost the same thing as a method reference), but it also doesn't require that the user create a whole method just for the purpose of sending the output off to whereever it ends up (on the console, in a UI, etc.).

Secondly, the Benchmark overloaded methods themselves - these take the actual code to be measured. The Benchmark overloads will execute the passed-in code a number of times for warmup, and then a number of times for the actual measurement. The return value of the class function version is the number of seconds an average iteration took; the instance function versions send off the result to the reporting sink.

These two idioms - data sinks and sources, and taking in a block of code which can be executed a carefully constructed context - are common usage patterns of anonymous methods. I'll get into them further in later posts.

Here's the Benchmark class in practice:

procedure UseIt;
const
  CallDepth = 10000;
var
  b: TBenchmarker;
  x: TSomeClass;
  intf: ISomeInterface;
begin
  b := TBenchmarker.Create(procedure(Name: string; Time: Double)
  begin
    Writeln(Format('%-20s took %15.9f ms', [Name, Time * 1000]));
  end);
  try
    b.Warmups := 100;
    b.Iterations := 100;

    x := TSomeClass.Create;
    intf := x;
      
    b.Benchmark('Static call', procedure
    begin
      x.StaticCall(x, CallDepth);
    end);

    b.Benchmark('Virtual call', procedure
    begin
      x.VirtCall(x, CallDepth);
    end);

    b.Benchmark('Interface call', procedure
    begin
      intf.IntfCall(intf, CallDepth);
    end);
    
  finally
    b.Free;
  end;
end;

The benefits of variable capture should be clear here. The segments of code to be benchmarked can freely access variables in the outer scope.

Finally, here's the complete implementation, including TBenchmark definition and use:

program am_profiler;

{$APPTYPE CONSOLE}

uses
  Windows, SysUtils;

type
  TBenchmarker = class
  private
    const
      DefaultIterations = 3;
      DefaultWarmups = 1;
    var
      FReportSink: TProc<string,Double>;
      FWarmups: Integer;
      FIterations: Integer;
      FOverhead: Double;
    class var
      FFreq: Int64;
    class procedure InitFreq;
  public
    constructor Create(const AReportSink: TProc<string,Double>);
    class function Benchmark(const Code: TProc; 
      Iterations: Integer = DefaultIterations; 
      Warmups: Integer = DefaultWarmups): Double; overload;
    procedure Benchmark(const Name: string; const Code: TProc); overload;
    function Benchmark<T>(const Name: string; const Code: TFunc<T>): T; overload;
    property Warmups: Integer read FWarmups write FWarmups;
    property Iterations: Integer read FIterations write FIterations;
  end;
  
{ TBenchmarker }

constructor TBenchmarker.Create(const AReportSink: TProc<string, Double>);
begin
  InitFreq;
  FReportSink := AReportSink;
  FWarmups := DefaultWarmups;
  FIterations := DefaultIterations;
  
  // Estimate overhead of harness
  FOverhead := Benchmark(procedure begin end, 100, 3);
end;

class procedure TBenchmarker.InitFreq;
begin
  if (FFreq = 0) and not QueryPerformanceFrequency(FFreq) then
    raise Exception.Create('No high-performance counter available.');
end;

procedure TBenchmarker.Benchmark(const Name: string; const Code: TProc);
begin
  FReportSink(Name, Benchmark(Code, Iterations, Warmups) - FOverhead);
end;

class function TBenchmarker.Benchmark(const Code: TProc; Iterations,
  Warmups: Integer): Double;
var
  start, stop: Int64;
  i: Integer;
begin
  InitFreq;
  
  for i := 1 to Warmups do
    Code;

  QueryPerformanceCounter(start);
  for i := 1 to Iterations do
    Code;
  QueryPerformanceCounter(stop);
  
  Result := (stop - start) / FFreq / Iterations;
end;

function TBenchmarker.Benchmark<T>(const Name: string; const Code: TFunc<T>): T;
var
  start, stop: Int64;
  i: Integer;
begin
  for i := 1 to FWarmups do
    Result := Code;
  
  QueryPerformanceCounter(start);
  for i := 1 to FIterations do
    Result := Code;
  QueryPerformanceCounter(stop);
  
  FReportSink(Name, (stop - start) / FFreq / Iterations - FOverhead);
end;

type
  ISomeInterface = interface
    procedure IntfCall(const Intf: ISomeInterface; depth: Integer);
  end;
  
  TSomeClass = class(TInterfacedObject, ISomeInterface)
  public
    procedure VirtCall(Inst: TSomeClass; depth: Integer); virtual;
    procedure StaticCall(Inst: TSomeClass; depth: Integer);
    procedure IntfCall(const Intf: ISomeInterface; depth: Integer);
  end;

{ TSomeClass }

procedure TSomeClass.IntfCall(const Intf: ISomeInterface; depth: Integer);
begin
  if depth > 0 then
    Intf.IntfCall(Intf, depth - 1);
end;

procedure TSomeClass.StaticCall(Inst: TSomeClass; depth: Integer);
begin
  if depth > 0 then
    StaticCall(Inst, depth - 1);
end;

procedure TSomeClass.VirtCall(Inst: TSomeClass; depth: Integer);
begin
  if depth > 0 then
    VirtCall(Inst, depth - 1);
end;

procedure UseIt;
const
  CallDepth = 10000;
var
  b: TBenchmarker;
  x: TSomeClass;
  intf: ISomeInterface;
begin
  b := TBenchmarker.Create(procedure(Name: string; Time: Double)
  begin
    Writeln(Format('%-20s took %15.9f ms', [Name, Time * 1000]));
  end);
  try
    b.Warmups := 100;
    b.Iterations := 100;

    x := TSomeClass.Create;
    intf := x;
      
    b.Benchmark('Static call', procedure
    begin
      x.StaticCall(x, CallDepth);
    end);

    b.Benchmark('Virtual call', procedure
    begin
      x.VirtCall(x, CallDepth);
    end);

    b.Benchmark('Interface call', procedure
    begin
      intf.IntfCall(intf, CallDepth);
    end);
    
  finally
    b.Free;
  end;
end;

begin
  try
    UseIt;
  except
    on E:Exception do
      Writeln(E.Classname, ': ', E.Message);
  end;
end.