Description
This RFC proposes creating a C benchmark harness which is similar to the project's current JavaScript benchmark harness.
Current Status
Currently, when we author C benchmarks, the benchmark files are self-contained apart from the API being tested. For example,
#include "stdlib/blas/ext/base/zindex_of_truthy.h"
#include "stdlib/complex/float64/ctor.h"
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <time.h>
#include <sys/time.h>
#define NAME "zindex_of_truthy"
#define ITERATIONS 10000000
#define REPEATS 3
#define MIN 1
#define MAX 6
/**
* Prints the TAP version.
*/
static void print_version( void ) {
printf( "TAP version 13\n" );
}
/**
* Prints the TAP summary.
*
* @param total total number of tests
* @param passing total number of passing tests
*/
static void print_summary( int total, int passing ) {
printf( "#\n" );
printf( "1..%d\n", total ); // TAP plan
printf( "# total %d\n", total );
printf( "# pass %d\n", passing );
printf( "#\n" );
printf( "# ok\n" );
}
/**
* Prints benchmarks results.
*
* @param iterations number of iterations
* @param elapsed elapsed time in seconds
*/
static void print_results( int iterations, double elapsed ) {
double rate = (double)iterations / elapsed;
printf( " ---\n" );
printf( " iterations: %d\n", iterations );
printf( " elapsed: %0.9f\n", elapsed );
printf( " rate: %0.9f\n", rate );
printf( " ...\n" );
}
/**
* Returns a clock time.
*
* @return clock time
*/
static double tic( void ) {
struct timeval now;
gettimeofday( &now, NULL );
return (double)now.tv_sec + (double)now.tv_usec/1.0e6;
}
/**
* Generates a random number on the interval [0,1).
*
* @return random number
*/
static double rand_double( void ) {
int r = rand();
return (double)r / ( (double)RAND_MAX + 1.0 );
}
/**
* Runs a benchmark.
*
* @param iterations number of iterations
* @param len array length
* @return elapsed time in seconds
*/
static double benchmark1( int iterations, int len ) {
double elapsed;
double *x;
double t;
int idx;
int i;
x = (double *)malloc( (size_t)( len*2 ) * sizeof( double ) );
for ( i = 0; i < len*2; i++ ) {
x[ i ] = 0.0;
}
x[ (len*2)-1 ] = 1.0;
idx = -1;
t = tic();
for ( i = 0; i < iterations; i++ ) {
x[ (len*2)-4 ] = (double)( i % 4 );
idx = stdlib_strided_zindex_of_truthy( len, (const stdlib_complex128_t *)x, 1 );
if ( idx < 0 ) {
printf( "unexpected result\n" );
break;
}
}
elapsed = tic() - t;
if ( idx < 0 ) {
printf( "unexpected result\n" );
}
free( x );
return elapsed;
}
/**
* Runs a benchmark.
*
* @param iterations number of iterations
* @param len array length
* @return elapsed time in seconds
*/
static double benchmark2( int iterations, int len ) {
double elapsed;
double *x;
double t;
int idx;
int i;
x = (double *)malloc( (size_t)( len*2 ) * sizeof( double ) );
for ( i = 0; i < len*2; i++ ) {
x[ i ] = 0.0;
}
x[ (len*2)-1 ] = 1.0;
idx = -1;
t = tic();
for ( i = 0; i < iterations; i++ ) {
x[ (len*2)-4 ] = (double)( i % 4 );
idx = stdlib_strided_zindex_of_truthy_ndarray( len, (const stdlib_complex128_t *)x, 1, 0 );
if ( idx < 0 ) {
printf( "unexpected result\n" );
break;
}
}
elapsed = tic() - t;
if ( idx < 0 ) {
printf( "unexpected result\n" );
}
free( x );
return elapsed;
}
/**
* Main execution sequence.
*/
int main( void ) {
double elapsed;
int count;
int iter;
int len;
int i;
int j;
// Use the current time to seed the random number generator:
srand( time( NULL ) );
print_version();
count = 0;
for ( i = MIN; i <= MAX; i++ ) {
len = pow( 10, i );
iter = ITERATIONS / pow( 10, i-1 );
for ( j = 0; j < REPEATS; j++ ) {
count += 1;
printf( "# c::%s:len=%d\n", NAME, len );
elapsed = benchmark1( iter, len );
print_results( iter, elapsed );
printf( "ok %d benchmark finished\n", count );
}
for ( j = 0; j < REPEATS; j++ ) {
count += 1;
printf( "# c::%s:ndarray:len=%d\n", NAME, len );
elapsed = benchmark2( iter, len );
print_results( iter, elapsed );
printf( "ok %d benchmark finished\n", count );
}
}
print_summary( count, count );
}In this benchmark, you'll notice that the file includes
- logic for writing TAP output.
- random value generation, including initial seeding.
- loop logic for managing the array length, number of iterations, and repeats.
- logic for starting and stopping the benchmark timers.
- imported headers to support 1-4.
In short, benchmark files include a fair amount of additional logic apart from the logic which forms the core benchmark. This is both a benefit--the benchmark files are fairly transparent and it is straightforward to deduce the runner logic--and a curse--the benchmark files are primarily comprised of scaffold code, leading to increased risk of drift, copy-paste mistakes, and increased maintenance overhead.
Proposal
This RFC proposes the creation of various C macros to remove the boilerplate C code and shrink the above benchmark by roughly half. Here is an example:
#include "stdlib/blas/ext/base/zindex_of_truthy.h"
#include "stdlib/complex/float64/ctor.h"
#include "stdlib/bench.h"
#ifndef NAME
#define NAME "zindex_of_truthy"
#endif
/**
* Runs a benchmark.
*
* @param iterations number of iterations
* @param len array length
* @return elapsed time in seconds
*/
STDLIB_BENCHMARK( benchmark1 ) {
int idx = -1;
STDLIB_BENCHMARK_MALLOC_ARRAY_FLOAT64( x, len*2 );
STDLIB_BENCHMARK_FILL_STRIDED_ARRAY_FLOAT64( x, len*2, 0.0 );
x[ (len*2)-1 ] = 1.0;
STDLIB_BENCHMARK_LOOP_PREAMBLE {
x[ (len*2)-4 ] = (double)( i % 4 );
idx = stdlib_strided_zindex_of_truthy( len, (const stdlib_complex128_t *)x, 1 );
if ( idx < 0 ) {
printf( "unexpected result\n" );
break;
}
}
STDLIB_BENCHMARK_LOOP_EPILOGUE;
if ( idx < 0 ) {
printf( "unexpected result\n" );
}
STDLIB_BENCHMARK_FREE( x );
STDLIB_BENCHMARK_EPILOGUE;
}
/**
* Runs a benchmark.
*
* @param iterations number of iterations
* @param len array length
* @return elapsed time in seconds
*/
STDLIB_BENCHMARK( benchmark2 ) {
int idx = -1;
STDLIB_BENCHMARK_MALLOC_ARRAY_FLOAT64( x, len*2 );
STDLIB_BENCHMARK_FILL_STRIDED_ARRAY_FLOAT64( x, len*2, 0.0 );
x[ (len*2)-1 ] = 1.0;
STDLIB_BENCHMARK_LOOP_PREAMBLE {
x[ (len*2)-4 ] = (double)( i % 4 );
idx = stdlib_strided_zindex_of_truthy_ndarray( len, (const stdlib_complex128_t *)x, 1, 0 );
if ( idx < 0 ) {
printf( "unexpected result\n" );
break;
}
}
STDLIB_BENCHMARK_LOOP_EPILOGUE;
if ( idx < 0 ) {
printf( "unexpected result\n" );
}
STDLIB_BENCHMARK_FREE( x );
STDLIB_BENCHMARK_EPILOGUE;
}
/**
* Main execution sequence.
*/
STDLIB_BENCH {
int iterations = 10000000;
int repeats = 3;
int min = 1;
int max = 6;
STDLIB_BENCH_LENGTH_PREAMBLE( repeats, iterations, min, max ) {
printf( "# c::%s:len=%d\n", NAME, len );
STDLIB_RUN_BENCHMARK( benchmark1 );
}
STDLIB_BENCH_LENGTH_PREAMBLE( repeats, iterations, min, max ) {
printf( "# c::%s:ndarray:len=%d\n", NAME, len );
STDLIB_RUN_BENCHMARK( benchmark2 );
}
STDLIB_BENCH_EPILOGUE;
}A few things to notice:
- Similar to elsewhere, such as
ndarray/base/unary and other packages which make heavy use of C macros, the above code utilizes C macros for injecting boilerplate code.
- Moving to macros does increase the risk of the benchmarks feeling more "magical", as it is not always obvious what the code behind the macro is actually doing, including what variables it is creating. This can increase the risk of mysterious compilation errors, especially for newcomers to the project.
- Building on (2), the macros will expose certain variables (e.g.,
len and i above) and will operate similar to how STDLIB_NAPI_ARGV_ macros work where certain macros will support defining userland variables (e.g., x above in the STDLIB_BENCHMARK_MALLOC_ARRAY_FLOAT64 macro).
- There is still going to be custom setup (e.g., array allocation, initialization, etc) and teardown logic (e.g.,
STDLIB_BENCHMARK_FREE), as that is hard to standardize given that benchmarks should be fairly unique (i.e., APIs being benchmarked are typically different and have different signatures and input argument assumptions).
- The naming conventions (
STDLIB_BENCH and STDLIB_BENCHMARK) were chosen to mirror our JavaScript naming conventions (e.g., var bench = require( '@stdlib/bench' ); and function benchmark(...) {...}).
- Not clear to me how we get around needing the custom
printf( "# c::..." ); lines. Those are going to be unique to each benchmark and is just some more "magic" that folks would be expected to remember.
- I'd like to move away from hardcoding benchmark names (e.g., via the
NAME macro). In JavaScript, we resolve the package name. I'd like that to be the same in C. Currently, we rely on explicitly defining a NAME macro in the benchmark source file. This is not great, as we vary in terms of convention, often using the package basename, but this runs the risk of name conflicts where other packages have the same basename. We use the package name in JavaScript benchmarks to ensure unique naming. It would be good to carry that forward to C benchmarks. In the code above, we still define a NAME macro, but as a backup when the -DNAME definition is not passed to the compiler by the build system. And in our make recipes and tooling for compiling and running benchmarks, we will update to resolve the corresponding package name and then explicitly set the corresponding compiler flag. This should be possible at the cost of a bit of extra compilation time due to need to resolve the nearest package name prior to compiling C benchmarks.
- Moving to
stdlib/bench.h, a header file which is likely to pull in several other dependencies, such as macros utilizing BLAS functions for initializing arrays, PRNGs for random number generation, etc, it is likely that this will result in additional dependencies, and thus source files, being pulled in, resulting in longer compilation times, regardless of whether the benchmark file in question actually needs the transitive dependencies. There is a risk of scope creep for what gets included in stdlib/bench.h and that will bring hidden compilation costs to every downstream consumer within stdlib. UPDATE: one way around this is to compile a shared library and then link against the *.so file. This would require some updates to tooling and how we perform compilation.
Related Issues
None.
Questions
- What are people's thoughts? Love it? Hate it?
- What could we do differently? Are the C macros too magical?
- What could we do instead? (code samples preferred)
Other
Prior Art
Checklist
Description
This RFC proposes creating a C benchmark harness which is similar to the project's current JavaScript benchmark harness.
Current Status
Currently, when we author C benchmarks, the benchmark files are self-contained apart from the API being tested. For example,
In this benchmark, you'll notice that the file includes
In short, benchmark files include a fair amount of additional logic apart from the logic which forms the core benchmark. This is both a benefit--the benchmark files are fairly transparent and it is straightforward to deduce the runner logic--and a curse--the benchmark files are primarily comprised of scaffold code, leading to increased risk of drift, copy-paste mistakes, and increased maintenance overhead.
Proposal
This RFC proposes the creation of various C macros to remove the boilerplate C code and shrink the above benchmark by roughly half. Here is an example:
A few things to notice:
ndarray/base/unaryand other packages which make heavy use of C macros, the above code utilizes C macros for injecting boilerplate code.lenandiabove) and will operate similar to howSTDLIB_NAPI_ARGV_macros work where certain macros will support defining userland variables (e.g.,xabove in theSTDLIB_BENCHMARK_MALLOC_ARRAY_FLOAT64macro).STDLIB_BENCHMARK_FREE), as that is hard to standardize given that benchmarks should be fairly unique (i.e., APIs being benchmarked are typically different and have different signatures and input argument assumptions).STDLIB_BENCHandSTDLIB_BENCHMARK) were chosen to mirror our JavaScript naming conventions (e.g.,var bench = require( '@stdlib/bench' );andfunction benchmark(...) {...}).printf( "# c::..." );lines. Those are going to be unique to each benchmark and is just some more "magic" that folks would be expected to remember.NAMEmacro). In JavaScript, we resolve the package name. I'd like that to be the same in C. Currently, we rely on explicitly defining aNAMEmacro in the benchmark source file. This is not great, as we vary in terms of convention, often using the package basename, but this runs the risk of name conflicts where other packages have the same basename. We use the package name in JavaScript benchmarks to ensure unique naming. It would be good to carry that forward to C benchmarks. In the code above, we still define aNAMEmacro, but as a backup when the-DNAMEdefinition is not passed to the compiler by the build system. And in ourmakerecipes and tooling for compiling and running benchmarks, we will update to resolve the corresponding package name and then explicitly set the corresponding compiler flag. This should be possible at the cost of a bit of extra compilation time due to need to resolve the nearest package name prior to compiling C benchmarks.stdlib/bench.h, a header file which is likely to pull in several other dependencies, such as macros utilizing BLAS functions for initializing arrays, PRNGs for random number generation, etc, it is likely that this will result in additional dependencies, and thus source files, being pulled in, resulting in longer compilation times, regardless of whether the benchmark file in question actually needs the transitive dependencies. There is a risk of scope creep for what gets included instdlib/bench.hand that will bring hidden compilation costs to every downstream consumer within stdlib. UPDATE: one way around this is to compile a shared library and then link against the*.sofile. This would require some updates to tooling and how we perform compilation.Related Issues
None.
Questions
Other
Prior Art
Checklist
RFC:.