selftest: Fix handling of testsuite, reintroduce progress indication.

6 files changed, 145 insertions, 33 deletions

diff --git a/lib/subunit/tap2subunit b/lib/subunit/tap2subunit
index 45f515540b..9e335168f5 100755
--- a/lib/subunit/tap2subunit
+++ b/lib/subunit/tap2subunit
@@ -33,4 +33,3 @@ while(<STDIN>) {
 	}
 }
 exit $error;
-
diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index 5ee8743c6c..4fddeec8b1 100644
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -35,11 +35,11 @@ sub parse_results($$$$)
 	while(<$fh>) {
 		if (/^test: (.+)\n/) {
 			$msg_ops->control_msg($_);
-			$msg_ops->start_test($open_tests, $1);
+			$msg_ops->start_test($1);
 			push (@$open_tests, $1);
 		} elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
 			$msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
-		} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
+		} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail|skip-testsuite|testsuite-failure|testsuite-success|testsuite-error): (.*?)( \[)?([ \t]*)\n/) {
 			$msg_ops->control_msg($_);
 			my $result = $1;
 			my $testname = $2;
@@ -55,7 +55,7 @@ sub parse_results($$$$)
 				
 				unless ($terminated) {
 					$statistics->{TESTS_ERROR}++;
-					$msg_ops->end_test($testname, $result, 1, "reason interrupted");
+					$msg_ops->end_test($testname, "error", 1, "reason ($result) interrupted");
 					return 1;
 				}
 			}
@@ -85,14 +85,26 @@ sub parse_results($$$$)
 				pop(@$open_tests); #FIXME: Check that popped value == $testname
 				$msg_ops->end_test($testname, $result, 1, $reason);
 				$unexpected_err++;
-			}
+			} elsif ($result eq "skip-testsuite") {
+				$msg_ops->skip_testsuite($testname);
+			} elsif ($result eq "testsuite-success") {
+				$msg_ops->end_testsuite($testname, "success", $reason);
+			} elsif ($result eq "testsuite-failure") {
+				$msg_ops->end_testsuite($testname, "failure", $reason);
+			} elsif ($result eq "testsuite-error") {
+				$msg_ops->end_testsuite($testname, "error", $reason);
+			} 
+		} elsif (/^testsuite: (.*)\n/) {
+			$msg_ops->start_testsuite($1);
+		} elsif (/^testsuite-count: (\d+)\n/) {
+			$msg_ops->testsuite_count($1);
 		} else {
 			$msg_ops->output_msg($_);
 		}
 	}
 
 	while ($#$open_tests > $orig_open_len) {
-		$msg_ops->end_test($open_tests, pop(@$open_tests), "error", 1,
+		$msg_ops->end_test(pop(@$open_tests), "error", 1,
 				   "was started but never finished!");
 		$statistics->{TESTS_ERROR}++;
 		$unexpected_err++;
@@ -118,7 +130,7 @@ sub end_test($$;$)
 	my $result = shift;
 	my $reason = shift;
 	if ($reason) {
-		print "$result: $name [";
+		print "$result: $name [\n";
 		print "$reason";
 		print "]\n";
 	} else {
@@ -126,6 +138,34 @@ sub end_test($$;$)
 	}
 }
 
+sub skip_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "skip", $reason);
+}
+
+sub fail_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "fail", $reason);
+}
+
+sub success_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "success", $reason);
+}
+
+sub xfail_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "xfail", $reason);
+}
+
 sub report_time($)
 {
 	my ($time) = @_;
@@ -133,4 +173,42 @@ sub report_time($)
 	printf "time: %04d-%02d-%02d %02d:%02d:%02dZ\n", $year+1900, $mon, $mday, $hour, $min, $sec;
 }
 
+# The following are Samba extensions:
+
+sub start_testsuite($)
+{
+	my ($name) = @_;
+	print "testsuite: $name\n";
+}
+
+sub skip_testsuite($;$)
+{
+	my ($name, $reason) = @_;
+	if ($reason) {
+		print "skip-testsuite: $name [$reason]\n";
+	} else {
+		print "skip-testsuite: $name\n";
+	}
+}
+
+sub end_testsuite($$;$)
+{
+	my $name = shift;
+	my $result = shift;
+	my $reason = shift;
+	if ($reason) {
+		print "testsuite-$result: $name [";
+		print "$reason";
+		print "]\n";
+	} else {
+		print "$result: $name\n";
+	}
+}
+
+sub testsuite_count($)
+{
+	my ($count) = @_;
+	print "testsuite-count: $count\n";
+}
+
 1;
diff --git a/selftest/filter-subunit.pl b/selftest/filter-subunit.pl
index 9a2c6f556c..b7a72217f3 100755
--- a/selftest/filter-subunit.pl
+++ b/selftest/filter-subunit.pl
@@ -147,9 +147,9 @@ sub output_msg($$)
 	print $msg;
 }
 
-sub start_test($$$)
+sub start_test($$)
 {
-	my ($self, $parents, $testname) = @_;
+	my ($self, $testname) = @_;
 
 	if (defined($opt_prefix)) {
 		$testname = $opt_prefix.$testname;
@@ -160,7 +160,7 @@ sub start_test($$$)
 
 sub end_test($$$$$)
 {
-	my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
+	my ($self, $testname, $result, $unexpected, $reason) = @_;
 
 	if (defined($opt_prefix)) {
 		$testname = $opt_prefix.$testname;
@@ -174,6 +174,29 @@ sub end_test($$$$$)
 	Subunit::end_test($testname, $result, $reason);
 }
 
+sub skip_testsuite($;$)
+{
+	Subunit::skip_testsuite(@_);
+}
+
+sub start_testsuite($;$)
+{
+	my ($self, $name) = @_;
+	Subunit::start_testsuite($name);
+}
+
+sub end_testsuite($$;$)
+{
+	my ($self, $name, $result, $reason) = @_;
+	Subunit::end_testsuite($name, $result, $reason);
+}
+
+sub testsuite_count($$)
+{
+	my ($self, $count) = @_;
+	Subunit::testsuite_count($count);
+}
+
 my $msg_ops = {};
 bless $msg_ops;
 
diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm
index 2427136073..bfdced2936 100644
--- a/selftest/output/plain.pm
+++ b/selftest/output/plain.pm
@@ -1,6 +1,6 @@
 #!/usr/bin/perl
 # Plain text output for selftest
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -41,6 +41,12 @@ sub new($$$$$$$) {
 	bless($self, $class);
 }
 
+sub testsuite_count($$)
+{
+	my ($self, $count) = @_;
+	$self->{totalsuites} = $count;
+}
+
 sub report_time($$)
 {
 	my ($self, $time) = @_;
@@ -101,23 +107,21 @@ sub control_msg($$)
 	#$self->output_msg($output);
 }
 
-sub end_testsuite($$$$$)
+sub end_testsuite($$$$)
 {
-	my ($self, $name, $result, $unexpected, $reason) = @_;
+	my ($self, $name, $result, $reason) = @_;
 	my $out = "";
+	my $unexpected = 0;
 
-	if ($unexpected) {
-		if ($result eq "success" and not defined($reason)) {
-			$reason = "Expected negative exit code, got positive exit code";
-		} 
+	if ($result eq "success" or $result eq "xfail") {
+		$self->{suites_ok}++;
+	} else {
 		$self->output_msg("ERROR: $reason\n");
 		push (@{$self->{suitesfailed}}, $name);
-	} else {
-		$self->{suites_ok}++;
-	}
-
-	if ($unexpected and $self->{immediate} and not $self->{verbose}) {
-		$out .= $self->{test_output}->{$name};
+		if ($self->{immediate} and not $self->{verbose}) {
+			$out .= $self->{test_output}->{$name};
+		}
+		$unexpected = 1;
 	}
 
 	if (not $self->{immediate}) {
diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index d98c5ae370..da25943957 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -166,7 +166,7 @@ sub find_in_list($$)
 	foreach (@$list) {
 		if ($fullname =~ /$$_[0]/) {
 			 return ($$_[1]) if ($$_[1]);
-			 return "NO REASON SPECIFIED";
+			 return "";
 		}
 	}
 
@@ -217,11 +217,15 @@ sub run_testsuite($$$$$)
 	my $pcap_file = setup_pcap($name);
 
 	Subunit::report_time(time());
-	Subunit::start_test($name);
+	Subunit::start_testsuite($name);
 
-	my $ret = system("$cmd | $RealBin/filter-subunit.pl --prefix \"$name.\" 2>&1");
+	system("$cmd 2>&1 | $RealBin/filter-subunit.pl --prefix \"$name.\"");
+	my $ret = $?;
 	if ($ret == -1) {
-		Subunit::end_test($name, "error", "Unable to run $cmd: $!");
+		Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
+		return 0;
+	} elsif ($ret & 127) {
+		Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
 		return 0;
 	}
 	my $envlog = getlog_env($envname);
@@ -241,7 +245,7 @@ sub run_testsuite($$$$$)
 	} else {
 		$result = "failure";
 	}
-	Subunit::end_test($name, $result, $reason);
+	Subunit::end_testsuite($name, $result, $reason);
 
 	cleanup_pcap($pcap_file, $exitcode);
 
@@ -616,18 +620,19 @@ my @available = ();
 foreach my $fn (@testlists) {
 	foreach (read_testlist($fn)) {
 		my $name = $$_[0];
-		next if (@includes and not find_in_list(\@includes, $name));
+		next if (@includes and not defined(find_in_list(\@includes, $name)));
 		push (@available, $_);
 	}
 }
 
+Subunit::testsuite_count($#available+1);
 Subunit::report_time(time());
 
 foreach (@available) {
 	my $name = $$_[0];
 	my $skipreason = skip($name);
-	if ($skipreason) {
-		Subunit::end_test($name, "skip", $skipreason);
+	if (defined($skipreason)) {
+		Subunit::skip_testsuite($name, $skipreason);
 	} else {
 		push(@todo, $_); 
 	}
@@ -815,7 +820,7 @@ $envvarstr
 		
 		my $envvars = setup_env($envname);
 		if (not defined($envvars)) {
-			Subunit::end_test($name, "skip", 
+			Subunit::skip_testsuite($name, 
 				"unable to set up environment $envname");
 			next;
 		}
diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 3271e70311..328ae09330 100644
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -11,7 +11,7 @@ SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selfte
 FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(srcdir)/selftest/knownfail
 FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
 
-subunittest:: everything
+test-subunit:: everything
 	$(SELFTEST) --socket-wrapper $(TESTS)
 
 slowtest:: everything
@@ -46,6 +46,9 @@ test-noswrap:: everything
 quicktest:: all
 	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
 
+quicktest-subunit:: all
+	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS)
+
 quicktestone:: all
 	$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT)