Thread.Queue testsuite_queue;
/* General class to manage a high-score list (e.g. of slow tests, tests
void add_score(float score, string name)
int num = sizeof(scores);
if (num != sizeof(names)) {
write("This should not happen!!");
else if (scores[0] < score) {
for (int i = 0; i < sizeof(scores); i++)
write(s, names[i], scores[i]);
Highscorelist slow_moves;
mapping(int:string) correct_results = ([]);
multiset expected_failures = (<>);
Process.create_process engine;
Thread.Queue writing_finished;
Thread.Queue reading_finished;
Thread.Queue write_queue;
// Write nothing if no test from the file was run at all.
if (sizeof(pass) + sizeof(fail) + sizeof(PASS) + sizeof(FAIL) > 0)
write("%-37s %7.2f %9d %7d %8d\n", name, cputime, reading_nodes,
owl_nodes, connection_nodes);
static void program_reader(object f)
werror("Waiting for writing to be finished.\n");
writing_finished->read();
werror("Finished waiting for writing to be finished.\n");
string s = f->gets() - "\r";
float current_time = time(timebase);
werror("Recv: " + s + "\n");
if (sscanf(s, "=%d %s", number, answer)) {
if (number < 10000 || number > 10005) {
test_number = (int) number;
string correct = correct_results[test_number];
correct = "correct result missing, check the test suite";
correct_results[test_number] = correct;
correct = "^" + correct + "$";
object re = Regexp(correct);
string result = (negate ^ re->match(answer)) ? "pass" : "fail";
if (result == "pass" && expected_failures[test_number]) {
if (result == "fail" && !expected_failures[test_number]) {
this_object()[result] += ({test_number});
walltime += (current_time - last_time);
slow_moves->add_score(current_time - last_time,
name + ":" + test_number);
if (result == "PASS" || result == "FAIL" || verbose)
write("%-15s %s %s [%s]\n", name + ":" + test_number,
result, answer, correct_results[test_number]);
last_time = current_time;
else if (number == 10000)
reading_nodes += (int) answer;
else if (number == 10001)
owl_nodes += (int) answer;
else if (number == 10002)
connection_nodes += (int) answer;
else if (number == 10003)
cputime = (float) answer;
else if (number == 10005)
uncertainty += (float) answer;
else if (number == 10004)
else if (sscanf(s, "?%s", answer)) {
sscanf(answer, "%d", number);
write("%-15s ?%s\n", name + ":", answer);
werror("Reader closing down.\n");
reading_finished->write("\n");
static void program_writer(object f)
string s = write_queue->read();
if (has_value(s, "quit"))
werror("Writer closed down\n");
static void program_monitor()
while (!quit_has_been_sent) {
if (engine->status() != 0 && !quit_has_been_sent) {
write("engine crashed in test suite %s.\n", name);
werror("Finishing sending.\n");
werror("Sent: " + s + "\n");
write_queue->write(s + "\n");
void run_testsuite(string suite_name, string command,
array(string) engine_options,
mapping(string:mixed) options,
array(int)|void test_numbers)
array(string) program_start_array = ({command}) + engine_options;
string testsuite = Stdio.read_file(suite_name);
werror("Couldn't find " + suite_name + "\n");
program_start_array = ({"valgrind"}) + program_start_array;
if (options["check-unoccupied-answers"])
testsuite = modify_testsuite(testsuite);
writing_finished = Thread.Queue();
reading_finished = Thread.Queue();
write_queue = Thread.Queue();
object f1 = Stdio.File();
object pipe1 = f1->pipe();
object f2 = Stdio.FILE();
object pipe2 = f2->pipe();
engine = Process.create_process(program_start_array,
(["stdin":pipe1, "stdout":pipe2]));
thread_create(program_reader, f2);
thread_create(program_writer, f1);
thread_create(program_monitor);
last_time = time(timebase);
expected_failures = (<>);
if (test_numbers && sizeof(test_numbers) == 0)
foreach (testsuite/"\n", string s) {
if (sscanf(s, "%d %s", number, command) == 2) {
command = (command / " ")[0];
if (number >= 10000 && number <= 10003)
if (test_numbers && !has_value(test_numbers, number))
if (sizeof(allowed_commands) > 0 && !allowed_commands[command])
if (correct_results[(int) number])
write("Repeated test number " + number + ".\n");
send("reset_reading_node_counter");
send("reset_owl_node_counter");
send("reset_connection_node_counter");
if (sscanf(s, "%*sreg_genmove%*s") == 2)
send("10005 move_uncertainty");
send("10000 get_reading_node_counter");
send("10001 get_owl_node_counter");
send("10002 get_connection_node_counter");
else if (sscanf(s, "#? [%[^]]]%s", answer, expected)) {
correct_results[(int)number] = answer;
expected_failures[(int)number] = 1;
werror("Signalling finish of writing\n");
writing_finished->write("\n");
reading_finished->read();
array(Testsuite) testsuites = ({});
multiset(string) allowed_commands = (<>);
// Replace all tests in the testsuite with new tests checking whether
// the given answers are unoccupied vertices.
string modify_testsuite(string testsuite)
Regexp re = Regexp("[^A-T]([A-T][0-9]+)(.*)");
foreach (testsuite / "\n", string row) {
else if (row[0..1] != "#?")
[coord, row] = re->split(row);
s += sprintf("%d color %s\n", 100 * test_number + n, coord);
float total_cputime = 0.0;
float total_uncertainty = 0.0;
int connection_nodes = 0;
int number_unexpected_pass = 0;
int number_unexpected_fail = 0;
foreach (testsuites, Testsuite t) {
total_time += t->walltime;
total_cputime += t->cputime;
total_uncertainty += t->uncertainty;
reading_nodes += t->reading_nodes;
owl_nodes += t->owl_nodes;
connection_nodes += t->connection_nodes;
number_unexpected_pass += sizeof(t->PASS);
number_unexpected_fail += sizeof(t->FAIL);
write("Total nodes: %d %d %d\n", reading_nodes, owl_nodes,
write("Total time: %.2f (%.2f)\n", total_cputime, total_time);
write("Total uncertainty: %.2f\n", total_uncertainty);
if (number_unexpected_pass > 0)
write("%d PASS\n", number_unexpected_pass);
if (number_unexpected_fail > 0)
write("%d FAIL\n", number_unexpected_fail);
write("Slowest moves:\n");
slow_moves->report("%s: %f seconds\n");
string parse_tests(mapping(string:array(int)) partial_testsuites,
if (sscanf(tests, "%[^ :]:%s", suite, numbers) != 2) {
if (has_value(suite, " "))
if (!has_suffix(suite, ".tst"))
if (!partial_testsuites[suite])
partial_testsuites[suite] = ({});
else if (sizeof(partial_testsuites[suite]) == 0)
foreach (numbers / ",", string interval) {
if (sscanf(interval, "%d-%d", start, stop) == 2)
for (int k = start; k <= stop; k++)
partial_testsuites[suite] |= ({k});
partial_testsuites[suite] |= ({(int) interval});
partial_testsuites[suite] = ({});
int main(int argc, array(string) argv)
array(string) testsuites = ({});
mapping(string:array(int)) partial_testsuites = ([]);
array(array(mixed)) all_options;
all_options = ({ ({"help", Getopt.NO_ARG, ({"-h", "--help"})}),
({"verbose", Getopt.NO_ARG, ({"-v", "--verbose"})}),
({"valgrind", Getopt.NO_ARG, "--valgrind"}),
({"check-unoccupied-answers", Getopt.NO_ARG,
({"slow_moves", Getopt.HAS_ARG, ({"-s", "--slow-moves"})}),
({"engine", Getopt.HAS_ARG, ({"-e", "--engine"})}),
({"options", Getopt.HAS_ARG, ({"-o", "--options"})}),
({"file", Getopt.HAS_ARG, ({"-f", "--file"})}),
({"jobs", Getopt.HAS_ARG, ({"-j", "--jobs"})}),
({"limit-commands", Getopt.HAS_ARG, ({"-l", "--limit-commands"})})});
mapping(string:mixed) options = ([]);
array(string) engine_options = ({});
foreach (Getopt.find_all_options(argv, all_options), array(mixed) option) {
[string name, mixed value] = option;
write(help_message, basename(argv[0]));
case "check-unoccupied-answers":
options["check-unoccupied-answers"] = 1;
engine_options += value / " ";
slow_moves = Highscorelist((int) value);
string testlist = Stdio.read_file(value);
werror("Couldn't find %s\n", value);
foreach ((testlist / "\n") - ({""}), string tests)
testsuites |= ({parse_tests(partial_testsuites, tests)});
foreach (value / ",", string command)
allowed_commands[command] = 1;
engine = "../interface/gnugo";
engine_options |= "--quiet --mode gtp" / " ";
argv = Getopt.get_args(argv)[1..];
foreach (argv, string tests)
testsuites |= ({parse_tests(partial_testsuites, tests)});
if (sizeof(testsuites) == 0) {
string makefile = Stdio.read_file("Makefile.am");
foreach (makefile / "\n", string s) {
if (sscanf(s, "%*sregress.sh $(srcdir) %s ", filename) == 2)
testsuites += ({filename});
testsuite_queue = Thread.Queue();
for (int j = 0; j < jobs; j++)
thread_create(run_testsuites, engine, engine_options,
options, partial_testsuites);
foreach(testsuites, string testsuite)
testsuite_queue->write(testsuite);
for (int j = 0; j < jobs; j++)
testsuite_queue->write("");
while (testsuite_queue->size() > 0)
void run_testsuites(string engine, array(string) engine_options,
mapping(string:mixed) options,
mapping(string:array(int)) partial_testsuites)
string suite_name = testsuite_queue->read();
Testsuite current_testsuite = Testsuite(suite_name - ".tst");
testsuites += ({current_testsuite});
current_testsuite->run_testsuite(suite_name, engine, engine_options,
options, partial_testsuites[suite_name]);
"Usage: %s [OPTIONS]... [TESTS]...\n"
"Run all regressions or a selection of them.\n"
" -h, --help Display this help and exit.\n"
" -v, --verbose Show also expected results.\n"
" --valgrind Run regressions under valgrind (very slow).\n"
" --check-unoccupied Do not run regressions. Instead check that\n"
" the listed answers are not occupied.\n"
" -e, --engine=ENGINE Engine to run regressions on. Default is\n"
" -o, --options=OPTIONS Options passed to the engine.\n"
" -f, --file=FILE File containing a list of tests to run.\n"
" -j, --jobs=JOBS Number of testsuites being run in parallel.\n"
" -l, --limit-commands=COMMANDS Only run tests having certain GTP commands.\n"
"Tests are listed on the command line in one of the following forms:\n"
"reading Run all tests in the testsuite reading.tst.\n"
"reading:4 Run test number 4 in reading.tst.\n"
"reading:4,17,30 Run tests with numbers 4, 7, and 30 in reading.tst\n"
"reading:4-17 Run tests with numbers between 4 and 17 in reading.tst\n"
"It is also allowed to include the suffix \".tst\" above and more complex\n"
"lists like \"reading.tst:1-3,15,17,30-50,52\" are also understood.\n"
"The format of files used with --file is the same, with one testsuite on\n"
"If no test suite is listed on the command line or read from file, then all\n"
"regressions listed in Makefile.am will be run.\n"
"The --limit-commands option takes a comma separated list of GTP commands,\n"
"e.g. '--limit-commands=attack,defend' to only run tactical reading tests.\n";