17 files changed, 1052 insertions, 507 deletions

diff --git a/selftest/BuildFarm.pm b/selftest/BuildFarm.pm
new file mode 100644
index 0000000000..80a91ac154
--- /dev/null
+++ b/selftest/BuildFarm.pm
@@ -0,0 +1,65 @@
+#!/usr/bin/perl
+# Convenience functions for writing output expected by the buildfarm
+# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later
+
+package BuildFarm;
+
+use Exporter;
+@ISA = qw(Exporter);
+@EXPORT_OK = qw(start_testsuite end_testsuite skip_testsuite summary);
+
+use strict;
+
+sub start_testsuite($$)
+{
+	my ($name, $duration) = @_;
+	my $out = "";
+
+	$out .= "--==--==--==--==--==--==--==--==--==--==--\n";
+	$out .= "Running test $name (level 0 stdout)\n";
+	$out .= "--==--==--==--==--==--==--==--==--==--==--\n";
+	$out .= scalar(localtime())."\n";
+	$out .= "SELFTEST RUNTIME: " . $duration . "s\n";
+	$out .= "NAME: $name\n";
+
+	print $out;
+}
+
+sub end_testsuite($$$$$)
+{
+	my ($name, $duration, $ok, $output, $reason) = @_;
+	my $out = "";
+
+	$out .= "TEST RUNTIME: " . $duration . "s\n";
+	if ($ok) {
+		$out .= "ALL OK\n";
+	} else {
+		$out .= "ERROR: $reason\n";
+	}
+	$out .= "==========================================\n";
+	if ($ok) {
+		$out .= "TEST PASSED: $name\n";
+	} else {
+		$out .= "TEST FAILED: $name (status $reason)\n";
+	}
+	$out .= "==========================================\n";
+
+	print $out;
+}
+
+sub skip_testsuite($)
+{
+	my ($name) = @_;
+
+	print "SKIPPED: $name\n";
+}
+
+sub summary($)
+{
+	my ($duration) = @_;
+
+	print "DURATION: " . $duration . " seconds\n";
+}
+
+1;
diff --git a/selftest/README b/selftest/README
index f8be20a569..4ae043692d 100644
--- a/selftest/README
+++ b/selftest/README
@@ -3,6 +3,15 @@
 This directory contains test scripts that are useful for running a
 bunch of tests all at once. 
 
+There are two parts to this: 
+
+ * The test runner (selftest/selftest.pl)
+ * The test formatter
+
+selftest.pl simply outputs subunit, which can then be formatted or analyzed 
+by tools that understand the subunit protocol. One of these tools is 
+format-subunit.pl, which is used by default as part of "make test".
+
 Available testsuites
 ====================
 The available testsuites are obtained from a script, usually 
@@ -11,7 +20,7 @@ the name of the test, the command to run and the environment that should be
 provided. Use the included "plantest" function to generate the required output.
 
 Testsuite behaviour
-================================
+===================
 
 Exit code
 ------------
@@ -31,41 +40,83 @@ for the last announced test.
 Accepted commands are:
 
 test
-~~~~~~~~~~~~
+~~~~
 test: <NAME>
 
 Announce that a new test with the specified name is starting
 
 success
-~~~~~~~~~~~~~~~
+~~~~~~~
 success: <NAME> 
 
 Announce that the test with the specified name is done and ran successfully.
 
 failure
-~~~~~~~~~~~~~~~
+~~~~~~~
 failure: <NAME> 
 failure: <NAME> [ REASON ]
 
 Announce that the test with the specified name failed. Optionally, it is 
 possible to specify a reason it failed.
 
+The alias "fail" will also work.
+
+xfail
+~~~~~
+xfail: <NAME>
+xfail: <NAME> [ REASON ]
+
+Announce that the test with the specified name failed but that the failure
+was expected, e.g. it's a test for a known bug that hasn't been fixed yet.
+Alternatively it is also possible to simply return "failure:" here but 
+specify in the samba4-knownfailures file that it is failing.
+
 skip
-~~~~~~~~~~~~
+~~~~
 skip: <NAME>
 skip: <NAME> [ REASON ]
 
 Announce that the test with the specified name was skipped. Optionally a 
 reason can be specified.
 
-knownfail
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-knownfail: <NAME>
-knownfail: <NAME> [ REASON ]
+time
+~~~~
+time: YYYY-MM-DD HH:mm:ssZ
 
-Announce that the test with the specified name was run and failed as expected.
-Alternatively it is also possible to simply return "failure:" here but 
-specify in the samba4-knownfailures file that it is failing.
+Announce the current time. This may be used to calculate the duration of 
+various tests.
+
+The following are Samba extensions to Subunit:
+
+testsuite-count
+~~~~~~~~~~~~~~~
+testsuite-count: number
+
+Announce the number of tests that is going to be run.
+
+start-testsuite
+~~~~~~~~~
+start-testsuite: name
+
+The testsuite name is used as prefix for all containing tests.
+
+skip-testsuite
+~~~~~~~~~~~~~~
+skip-testsuite: name
+
+Mark the testsuite with the specified name as skipped.
+
+testsuite-success
+~~~~~~~~~~~~~~~~~
+testsuite-success: name
+
+Indicate that the testsuite has succeeded successfully.
+
+testsuite-fail
+~~~~~~~~~~~~~~
+testsuite-fail: name
+
+Indicate that a testsuite has failed.
 
 Environments
 ============
diff --git a/selftest/SocketWrapper.pm b/selftest/SocketWrapper.pm
index e63605b8df..ef8058da79 100644
--- a/selftest/SocketWrapper.pm
+++ b/selftest/SocketWrapper.pm
@@ -1,7 +1,21 @@
 #!/usr/bin/perl
 # Bootstrap Samba and run a number of tests against it.
 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
-# Published under the GNU GPL, v3 or later.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
 
 package SocketWrapper;
 
diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index 05e51da541..9d67c8137a 100644
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -1,4 +1,21 @@
+# Perl module for parsing and generating the Subunit protocol
+# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package Subunit;
+use POSIX;
 
 require Exporter;
 @ISA = qw(Exporter);
@@ -6,22 +23,25 @@ require Exporter;
 
 use strict;
 
-sub parse_results($$$$$)
+sub parse_results($$$)
 {
-	my ($msg_ops, $statistics, $fh, $expecting_failure, $open_tests) = @_;
-	my $unexpected_ok = 0;
+	my ($msg_ops, $statistics, $fh) = @_;
 	my $expected_fail = 0;
 	my $unexpected_fail = 0;
 	my $unexpected_err = 0;
-	my $orig_open_len = $#$open_tests;
+	my $open_tests = [];
 
 	while(<$fh>) {
 		if (/^test: (.+)\n/) {
 			$msg_ops->control_msg($_);
-			$msg_ops->start_test($open_tests, $1);
+			$msg_ops->start_test($1);
 			push (@$open_tests, $1);
-		} elsif (/^(success|successful|failure|skip|knownfail|error): (.*?)( \[)?([ \t]*)\n/) {
+		} elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)\n/) {
+			$msg_ops->report_time(mktime($6, $5, $4, $3, $2-1, $1-1900));
+		} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail|skip-testsuite|testsuite-failure|testsuite-xfail|testsuite-success|testsuite-error): (.*?)( \[)?([ \t]*)\n/) {
 			$msg_ops->control_msg($_);
+			my $result = $1;
+			my $testname = $2;
 			my $reason = undef;
 			if ($3) {
 				$reason = "";
@@ -34,53 +54,60 @@ sub parse_results($$$$$)
 				
 				unless ($terminated) {
 					$statistics->{TESTS_ERROR}++;
-					$msg_ops->end_test($open_tests, $2, $1, 1, "reason interrupted");
+					$msg_ops->end_test($testname, "error", 1, 
+						               "reason ($result) interrupted");
 					return 1;
 				}
 			}
-			my $result = $1;
-			if ($1 eq "success" or $1 eq "successful") {
-				pop(@$open_tests); #FIXME: Check that popped value == $2
-				if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
-					$statistics->{TESTS_UNEXPECTED_OK}++;
-					$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
-					$unexpected_ok++;
-				} else {
-					$statistics->{TESTS_EXPECTED_OK}++;
-					$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-				}
-			} elsif ($1 eq "failure") {
-				pop(@$open_tests); #FIXME: Check that popped value == $2
-				if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
-					$statistics->{TESTS_EXPECTED_FAIL}++;
-					$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-					$expected_fail++;
-				} else {
-					$statistics->{TESTS_UNEXPECTED_FAIL}++;
-					$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
-					$unexpected_fail++;
-				}
-			} elsif ($1 eq "knownfail") {
-				pop(@$open_tests); #FIXME: Check that popped value == $2
+			if ($result eq "success" or $result eq "successful") {
+				pop(@$open_tests); #FIXME: Check that popped value == $testname 
+				$statistics->{TESTS_EXPECTED_OK}++;
+				$msg_ops->end_test($testname, "success", 0, $reason);
+			} elsif ($result eq "xfail" or $result eq "knownfail") {
+				pop(@$open_tests); #FIXME: Check that popped value == $testname
 				$statistics->{TESTS_EXPECTED_FAIL}++;
-				$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-			} elsif ($1 eq "skip") {
+				$msg_ops->end_test($testname, "xfail", 0, $reason);
+				$expected_fail++;
+			} elsif ($result eq "failure" or $result eq "fail") {
+				pop(@$open_tests); #FIXME: Check that popped value == $testname
+				$statistics->{TESTS_UNEXPECTED_FAIL}++;
+				$msg_ops->end_test($testname, "failure", 1, $reason);
+				$unexpected_fail++;
+			} elsif ($result eq "skip") {
 				$statistics->{TESTS_SKIP}++;
-				pop(@$open_tests); #FIXME: Check that popped value == $2
-				$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
-			} elsif ($1 eq "error") {
+				# Allow tests to be skipped without prior announcement of test
+				my $last = pop(@$open_tests);
+				if (defined($last) and $last ne $testname) {
+					push (@$open_tests, $testname);
+				}
+				$msg_ops->end_test($testname, "skip", 0, $reason);
+			} elsif ($result eq "error") {
 				$statistics->{TESTS_ERROR}++;
-				pop(@$open_tests); #FIXME: Check that popped value == $2
-				$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
+				pop(@$open_tests); #FIXME: Check that popped value == $testname
+				$msg_ops->end_test($testname, "error", 1, $reason);
 				$unexpected_err++;
-			}
+			} elsif ($result eq "skip-testsuite") {
+				$msg_ops->skip_testsuite($testname);
+			} elsif ($result eq "testsuite-success") {
+				$msg_ops->end_testsuite($testname, "success", $reason);
+			} elsif ($result eq "testsuite-failure") {
+				$msg_ops->end_testsuite($testname, "failure", $reason);
+			} elsif ($result eq "testsuite-xfail") {
+				$msg_ops->end_testsuite($testname, "xfail", $reason);
+			} elsif ($result eq "testsuite-error") {
+				$msg_ops->end_testsuite($testname, "error", $reason);
+			} 
+		} elsif (/^testsuite: (.*)\n/) {
+			$msg_ops->start_testsuite($1);
+		} elsif (/^testsuite-count: (\d+)\n/) {
+			$msg_ops->testsuite_count($1);
 		} else {
 			$msg_ops->output_msg($_);
 		}
 	}
 
-	while ($#$open_tests > $orig_open_len) {
-		$msg_ops->end_test($open_tests, pop(@$open_tests), "error", 1,
+	while ($#$open_tests+1 > 0) {
+		$msg_ops->end_test(pop(@$open_tests), "error", 1,
 				   "was started but never finished!");
 		$statistics->{TESTS_ERROR}++;
 		$unexpected_err++;
@@ -88,10 +115,100 @@ sub parse_results($$$$$)
 
 	return 1 if $unexpected_err > 0;
 	return 1 if $unexpected_fail > 0;
-	return 1 if $unexpected_ok > 0 and $expected_fail > 0;
-	return 0 if $unexpected_ok > 0 and $expected_fail == 0;
-	return 0 if $expected_fail > 0;
-	return 1;
+	return 0;
+}
+
+sub start_test($)
+{
+	my ($testname) = @_;
+	print "test: $testname\n";
+}
+
+sub end_test($$;$)
+{
+	my $name = shift;
+	my $result = shift;
+	my $reason = shift;
+	if ($reason) {
+		print "$result: $name [\n";
+		print "$reason";
+		print "]\n";
+	} else {
+		print "$result: $name\n";
+	}
+}
+
+sub skip_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "skip", $reason);
+}
+
+sub fail_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "fail", $reason);
+}
+
+sub success_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "success", $reason);
+}
+
+sub xfail_test($;$)
+{
+	my $name = shift;
+	my $reason = shift;
+	end_test($name, "xfail", $reason);
+}
+
+sub report_time($)
+{
+	my ($time) = @_;
+	my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime($time);
+	printf "time: %04d-%02d-%02d %02d:%02d:%02d\n", $year+1900, $mon+1, $mday, $hour, $min, $sec;
+}
+
+# The following are Samba extensions:
+
+sub start_testsuite($)
+{
+	my ($name) = @_;
+	print "testsuite: $name\n";
+}
+
+sub skip_testsuite($;$)
+{
+	my ($name, $reason) = @_;
+	if ($reason) {
+		print "skip-testsuite: $name [\n$reason\n]\n";
+	} else {
+		print "skip-testsuite: $name\n";
+	}
+}
+
+sub end_testsuite($$;$)
+{
+	my $name = shift;
+	my $result = shift;
+	my $reason = shift;
+	if ($reason) {
+		print "testsuite-$result: $name [\n";
+		print "$reason\n";
+		print "]\n";
+	} else {
+		print "testsuite-$result: $name\n";
+	}
+}
+
+sub testsuite_count($)
+{
+	my ($count) = @_;
+	print "testsuite-count: $count\n";
 }
 
 1;
diff --git a/selftest/Subunit/Diff.pm b/selftest/Subunit/Diff.pm
new file mode 100644
index 0000000000..de251b37b3
--- /dev/null
+++ b/selftest/Subunit/Diff.pm
@@ -0,0 +1,80 @@
+#!/usr/bin/perl
+# Diff two subunit streams
+# Copyright (C) Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later
+
+package Subunit::Diff;
+
+use strict;
+
+use Subunit qw(parse_results);
+
+sub control_msg() { }
+sub report_time($$) { }
+
+sub output_msg($$)
+{
+	my ($self, $msg) = @_;
+
+	# No output for now, perhaps later diff this as well ?
+}
+
+sub start_test($$)
+{
+	my ($self, $testname) = @_;
+}
+
+sub end_test($$$$$)
+{
+	my ($self, $testname, $result, $unexpected, $reason) = @_;
+
+	$self->{$testname} = $result;
+}
+
+sub skip_testsuite($;$) { } 
+sub start_testsuite($;$) { }
+sub end_testsuite($$;$) { }
+sub testsuite_count($$) { }
+
+sub new {
+	my ($class) = @_;
+
+	my $self = { 
+	};
+	bless($self, $class);
+}
+
+sub from_file($)
+{
+	my ($path) = @_;
+	my $statistics = {
+		TESTS_UNEXPECTED_OK => 0,
+		TESTS_EXPECTED_OK => 0,
+		TESTS_UNEXPECTED_FAIL => 0,
+		TESTS_EXPECTED_FAIL => 0,
+		TESTS_ERROR => 0,
+		TESTS_SKIP => 0,
+	};
+
+	my $ret = new Subunit::Diff();
+	open(IN, $path) or return;
+	parse_results($ret, $statistics, *IN);
+	close(IN);
+	return $ret;
+}
+
+sub diff($$)
+{
+	my ($old, $new) = @_;
+	my $ret = {};
+
+	foreach my $testname (keys %$old) {
+		if ($new->{$testname} ne $old->{$testname}) {
+			$ret->{$testname} = [$old->{$testname}, $new->{$testname}];
+		}
+	}
+
+	return $ret;
+}
+
+1;
diff --git a/selftest/Subunit/Filter.pm b/selftest/Subunit/Filter.pm
new file mode 100644
index 0000000000..93b690df8e
--- /dev/null
+++ b/selftest/Subunit/Filter.pm
@@ -0,0 +1,149 @@
+#!/usr/bin/perl
+# Filter a subunit stream
+# Copyright (C) Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later
+
+package Subunit::Filter;
+
+use strict;
+
+sub read_test_regexes($)
+{
+	my ($name) = @_;
+	my @ret = ();
+	open(LF, "<$name") or die("unable to read $name: $!");
+	while (<LF>) { 
+		chomp; 
+		next if (/^#/);
+		if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
+			push (@ret, [$1, $4]);
+		} else {
+			s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
+			push (@ret, [$_, undef]); 
+		}
+	}
+	close(LF);
+	return @ret;
+}
+
+sub find_in_list($$)
+{
+	my ($list, $fullname) = @_;
+
+	foreach (@$list) {
+		if ($fullname =~ /$$_[0]/) {
+			 return ($$_[1]) if ($$_[1]);
+			 return "";
+		}
+	}
+
+	return undef;
+}
+
+sub control_msg()
+{
+	# We regenerate control messages, so ignore this
+}
+
+sub report_time($$)
+{
+	my ($self, $time) = @_;
+	Subunit::report_time($time);
+}
+
+sub output_msg($$)
+{
+	my ($self, $msg) = @_;
+	unless(defined($self->{output})) {
+		print $msg;
+	} else {
+		$self->{output}.=$msg;
+	}
+}
+
+sub start_test($$)
+{
+	my ($self, $testname) = @_;
+
+	if (defined($self->{prefix})) {
+		$testname = $self->{prefix}.$testname;
+	}
+
+	if ($self->{strip_ok_output}) {
+		$self->{output} = "";
+	}
+
+	Subunit::start_test($testname);
+}
+
+sub end_test($$$$$)
+{
+	my ($self, $testname, $result, $unexpected, $reason) = @_;
+
+	if (defined($self->{prefix})) {
+		$testname = $self->{prefix}.$testname;
+	}
+
+	if (($result eq "fail" or $result eq "failure") and not $unexpected) {
+		$result = "xfail";
+		$self->{xfail_added}++;
+	}
+	my $xfail_reason = find_in_list($self->{expected_failures}, $testname);
+	if (defined($xfail_reason) and ($result eq "fail" or $result eq "failure")) {
+		$result = "xfail";
+		$self->{xfail_added}++;
+		$reason .= $xfail_reason;
+	}
+
+	if ($self->{strip_ok_output}) {
+		unless ($result eq "success" or $result eq "xfail" or $result eq "skip") {
+			print $self->{output}
+		}
+	}
+	$self->{output} = undef;
+
+	Subunit::end_test($testname, $result, $reason);
+}
+
+sub skip_testsuite($;$)
+{
+	my ($self, $name, $reason) = @_;
+	Subunit::skip_testsuite($name, $reason);
+}
+
+sub start_testsuite($;$)
+{
+	my ($self, $name) = @_;
+	Subunit::start_testsuite($name);
+	$self->{xfail_added} = 0;
+}
+
+sub end_testsuite($$;$)
+{
+	my ($self, $name, $result, $reason) = @_;
+	if ($self->{xfail_added} and ($result eq "fail" or $result eq "failure")) {
+		$result = "xfail";
+	}
+		
+	Subunit::end_testsuite($name, $result, $reason);
+}
+
+sub testsuite_count($$)
+{
+	my ($self, $count) = @_;
+	Subunit::testsuite_count($count);
+}
+
+sub new {
+	my ($class, $prefix, $expected_failures, $strip_ok_output) = @_;
+
+	my $self = { 
+		prefix => $prefix,
+		expected_failures => $expected_failures,
+		strip_ok_output => $strip_ok_output,
+		xfail_added => 0,
+	};
+	bless($self, $class);
+}
+
+1;
diff --git a/selftest/diff-subunit.pl b/selftest/diff-subunit.pl
new file mode 100755
index 0000000000..fc80c371f8
--- /dev/null
+++ b/selftest/diff-subunit.pl
@@ -0,0 +1,21 @@
+#!/usr/bin/perl
+# Diff two subunit streams
+# Copyright (C) Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later
+
+use Getopt::Long;
+use strict;
+use FindBin qw($RealBin $Script);
+use lib "$RealBin";
+use Subunit::Diff;
+
+my $old = Subunit::Diff::from_file($ARGV[0]);
+my $new = Subunit::Diff::from_file($ARGV[1]);
+
+my $ret = Subunit::Diff::diff($old, $new);
+
+foreach my $e (sort(keys %$ret)) {
+	printf "%s: %s -> %s\n", $e, $ret->{$e}[0], $ret->{$e}[1];
+}
+
+0;
diff --git a/selftest/filter-subunit.pl b/selftest/filter-subunit.pl
new file mode 100755
index 0000000000..9ebc67778f
--- /dev/null
+++ b/selftest/filter-subunit.pl
@@ -0,0 +1,97 @@
+#!/usr/bin/perl
+# Filter a subunit stream
+# Copyright (C) Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later
+
+=pod
+
+=head1 NAME
+
+filter-subunit - Filter a subunit stream
+
+=head1 SYNOPSIS
+
+filter-subunit --help
+
+filter-subunit --prefix=PREFIX --known-failures=FILE < in-stream > out-stream
+
+=head1 DESCRIPTION
+
+Simple Subunit stream filter that will change failures to known failures 
+based on a list of regular expressions.
+
+=head1 OPTIONS
+
+=over 4
+
+=item I<--prefix>
+
+Add the specified prefix to all test names.
+
+=item I<--expected-failures>
+
+Specify a file containing a list of tests that are expected to fail. Failures 
+for these tests will be counted as successes, successes will be counted as 
+failures.
+
+The format for the file is, one entry per line:
+
+TESTSUITE-NAME.TEST-NAME
+
+The reason for a test can also be specified, by adding a hash sign (#) and the reason 
+after the test name.
+
+=head1 LICENSE
+
+selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
+
+
+=head1 AUTHOR
+
+Jelmer Vernooij
+
+=cut
+
+use Getopt::Long;
+use strict;
+use FindBin qw($RealBin $Script);
+use lib "$RealBin";
+use Subunit qw(parse_results);
+use Subunit::Filter;
+
+my $opt_expected_failures = undef;
+my $opt_help = 0;
+my $opt_prefix = undef;
+my $opt_strip_ok_output = 0;
+my @expected_failures = ();
+
+my $result = GetOptions(
+		'expected-failures=s' => \$opt_expected_failures,
+		'strip-passed-output' => \$opt_strip_ok_output,
+		'prefix=s' => \$opt_prefix,
+		'help' => \$opt_help,
+	);
+exit(1) if (not $result);
+
+if ($opt_help) {
+	print "Usage: filter-subunit [--prefix=PREFIX] [--expected-failures=FILE]... < instream > outstream\n";
+	exit(0);
+}
+
+if (defined($opt_expected_failures)) {
+	@expected_failures = Subunit::Filter::read_test_regexes($opt_expected_failures);
+}
+
+my $statistics = {
+	TESTS_UNEXPECTED_OK => 0,
+	TESTS_EXPECTED_OK => 0,
+	TESTS_UNEXPECTED_FAIL => 0,
+	TESTS_EXPECTED_FAIL => 0,
+	TESTS_ERROR => 0,
+	TESTS_SKIP => 0,
+};
+
+my $msg_ops = new Subunit::Filter($opt_prefix, \@expected_failures, 
+	                              $opt_strip_ok_output);
+
+exit(parse_results($msg_ops, $statistics, *STDIN));
diff --git a/selftest/format-subunit.pl b/selftest/format-subunit.pl
new file mode 100755
index 0000000000..9d4e0c0785
--- /dev/null
+++ b/selftest/format-subunit.pl
@@ -0,0 +1,88 @@
+#!/usr/bin/perl
+# Pretty-format subunit output
+# Copyright (C) Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later
+
+=pod
+
+=head1 NAME
+
+format-subunit [--format=<NAME>] [--immediate] < instream > outstream
+
+=head1 SYNOPSIS
+
+Format the output of a subunit stream.
+
+=head1 OPTIONS
+
+=over 4
+
+=item I<--immediate>
+
+Show errors as soon as they happen rather than at the end of the test run.
+
+=item I<--format>=FORMAT
+
+Choose the format to print. Currently supported are plain or html.
+
+=head1 LICENSE
+
+GNU General Public License, version 3 or later.
+
+=head1 AUTHOR
+
+Jelmer Vernooij <jelmer@samba.org>
+		
+=cut
+
+use Getopt::Long;
+use strict;
+use FindBin qw($RealBin $Script);
+use lib "$RealBin";
+use Subunit qw(parse_results);
+
+my $opt_format = "plain";
+my $opt_help = undef;
+my $opt_verbose = 0;
+my $opt_immediate = 0;
+my $opt_prefix = ".";
+
+my $result = GetOptions (
+		'help|h|?' => \$opt_help,
+		'format=s' => \$opt_format,
+		'verbose' => \$opt_verbose,
+		'immediate' => \$opt_immediate,
+		'prefix:s' => \$opt_prefix,
+	    );
+
+exit(1) if (not $result);
+
+my $msg_ops;
+
+my $statistics = {
+	SUITES_FAIL => 0,
+
+	TESTS_UNEXPECTED_OK => 0,
+	TESTS_EXPECTED_OK => 0,
+	TESTS_UNEXPECTED_FAIL => 0,
+	TESTS_EXPECTED_FAIL => 0,
+	TESTS_ERROR => 0,
+	TESTS_SKIP => 0,
+};
+
+if ($opt_format eq "plain") {
+	require output::plain;
+	$msg_ops = new output::plain("$opt_prefix/summary", $opt_verbose, $opt_immediate, $statistics, undef);
+} elsif ($opt_format eq "html") {
+	require output::html;
+	mkdir("test-results", 0777);
+	$msg_ops = new output::html("test-results", $statistics);
+} else {
+	die("Invalid output format '$opt_format'");
+}
+
+my $expected_ret = parse_results($msg_ops, $statistics, *STDIN);
+
+$msg_ops->summary();
+
+exit($expected_ret);
diff --git a/selftest/output/buildfarm.pm b/selftest/output/buildfarm.pm
deleted file mode 100644
index cee6c1e63a..0000000000
--- a/selftest/output/buildfarm.pm
+++ /dev/null
@@ -1,120 +0,0 @@
-#!/usr/bin/perl
-
-package output::buildfarm;
-
-use Exporter;
-@ISA = qw(Exporter);
-
-use FindBin qw($RealBin);
-use lib "$RealBin/..";
-
-use Subunit qw(parse_results);
-
-use strict;
-
-sub new($$$) {
-	my ($class) = @_;
-	my $self = {
-		test_output => {},
-		start_time => time()
-	};
-	bless($self, $class);
-}
-
-sub start_testsuite($$)
-{
-	my ($self, $name) = @_;
-	my $out = "";
-
-	$self->{NAME} = $name;
-	$self->{START_TIME} = time();
-
-	my $duration = $self->{START_TIME} - $self->{start_time};
-	$out .= "--==--==--==--==--==--==--==--==--==--==--\n";
-	$out .= "Running test $name (level 0 stdout)\n";
-	$out .= "--==--==--==--==--==--==--==--==--==--==--\n";
-	$out .= scalar(localtime())."\n";
-	$out .= "SELFTEST RUNTIME: " . $duration . "s\n";
-	$out .= "NAME: $name\n";
-
-	$self->{test_output}->{$name} = "";
-
-	print $out;
-}
-
-sub output_msg($$)
-{
-	my ($self, $output) = @_;
-
-	$self->{test_output}->{$self->{NAME}} .= $output;
-}
-
-sub control_msg($$)
-{
-	my ($self, $output) = @_;
-
-	$self->{test_output}->{$self->{NAME}} .= $output;
-}
-
-sub end_testsuite($$$$$$)
-{
-	my ($self, $name, $result, $unexpected, $reason) = @_;
-	my $out = "";
-
-	$out .= "TEST RUNTIME: " . (time() - $self->{START_TIME}) . "s\n";
-
-	if (not $unexpected) {
-		$out .= "ALL OK\n";
-	} else {
-		$out .= "ERROR: $reason\n";
-		$out .= $self->{test_output}->{$name};
-	}
-
-	$out .= "==========================================\n";
-	if (not $unexpected) {
-		$out .= "TEST PASSED: $name\n";
-	} else {
-		$out .= "TEST FAILED: $name (status $reason)\n";
-	}
-	$out .= "==========================================\n";
-
-	print $out;
-}
-
-sub start_test($$$)
-{
-	my ($self, $parents, $testname) = @_;
-
-	if ($#$parents == -1) {
-		$self->start_testsuite($testname);
-	}
-}
-
-sub end_test($$$$$)
-{
-	my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
-
-	if ($unexpected) {
-		$self->{test_output}->{$self->{NAME}} .= "UNEXPECTED($result): $testname\n";
-	}
-
-	if ($#$parents == -1) {
-		$self->end_testsuite($testname, $result, $unexpected, $reason); 
-	}
-}
-
-sub summary($)
-{
-	my ($self) = @_;
-
-	print "DURATION: " . (time() - $self->{start_time}) . " seconds\n";
-}
-
-sub skip_testsuite($$$$)
-{
-	my ($self, $name, $reason) = @_;
-
-	print "SKIPPED: $name\n";
-}
-
-1;
diff --git a/selftest/output/html.pm b/selftest/output/html.pm
index 1049527129..8e42b65649 100644
--- a/selftest/output/html.pm
+++ b/selftest/output/html.pm
@@ -1,5 +1,19 @@
 #!/usr/bin/perl
-
+# HTML output for selftest
+# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package output::html;
 use Exporter;
 @ISA = qw(Exporter);
@@ -10,8 +24,6 @@ use warnings;
 use FindBin qw($RealBin);
 use lib "$RealBin/..";
 
-use Subunit qw(parse_results);
-
 sub new($$$) {
 	my ($class, $dirname, $statistics) = @_;
 	my $self = { 
@@ -49,6 +61,10 @@ sub new($$$) {
 	return $self;
 }
 
+sub testsuite_count($$)
+{
+}
+
 sub print_html_header($$$)
 {
 	my ($self, $title, $fh) = @_;
@@ -80,6 +96,8 @@ sub start_testsuite($$)
 {
 	my ($self, $name) = @_;
 
+	$self->{START_TIME} = $self->{last_time};
+
 	$self->{local_statistics} = {
 		success => 0,
 		skip => 0,
@@ -104,6 +122,7 @@ sub control_msg($$)
 {
 	my ($self, $output) = @_;
 
+	# Perhaps the CSS should hide this by default?
 	$self->{msg} .=  "<span class=\"control\">$output<br/></span>\n";
 }
 
@@ -112,19 +131,21 @@ sub output_msg($$)
 	my ($self, $output) = @_;
 
 	unless (defined($self->{active_test})) {
-		print TEST "$output<br/>";
+		if (defined($self->{NAME})) {
+			print TEST "$output<br/>";
+		}
 	} else {
 		$self->{msg} .= "$output<br/>";
 	}
 }
 
-sub end_testsuite($$$$)
+sub end_testsuite($$$)
 {
-	my ($self, $name, $result, $unexpected, $reason) = @_;
+	my ($self, $name, $result, $reason) = @_;
 
 	print TEST "</table>\n";
 
-	print TEST "<div class=\"duration\">Duration: " . (time() - $self->{START_TIME}) . "s</div>\n";
+	print TEST "<div class=\"duration\">Duration: " . ($self->{last_time} - $self->{START_TIME}) . "s</div>\n";
 
 	$self->print_html_footer(*TEST);
 
@@ -134,12 +155,10 @@ sub end_testsuite($$$$)
 	print INDEX "  <td class=\"testSuite\"><a href=\"$self->{HTMLFILE}\">$name</a></td>\n";
 	my $st = $self->{local_statistics};
 
-	if (not $unexpected) {
-		if ($result eq "failure") {
-			print INDEX "  <td class=\"resultExpectedFailure\">";
-		} else {
-			print INDEX "  <td class=\"resultOk\">";
-		}
+	if ($result eq "xfail") {
+		print INDEX "  <td class=\"resultExpectedFailure\">";
+	} elsif ($result eq "success") {
+		print INDEX "  <td class=\"resultOk\">";
 	} else {
 		print INDEX "  <td class=\"resultFailure\">";
 	}
@@ -166,40 +185,33 @@ sub end_testsuite($$$$)
 	}
 
 	if ($l == 0) {
-		if (not $unexpected) {
-			print INDEX "OK";
-		} else {
-			print INDEX "FAIL";
-		}
+		print INDEX uc($result);
 	}
 
 	print INDEX "</td>";
 		
 	print INDEX "</tr>\n";
+
+	$self->{NAME} = undef;
 }
 
-sub start_test($$)
+sub report_time($$)
 {
-	my ($self, $parents, $testname) = @_;
+	my ($self, $time) = @_;
+	$self->{last_time} = $time;
+}
 
-	if ($#$parents == -1) {
-		$self->{START_TIME} = time();
-		$self->start_testsuite($testname);
-		return;
-	}
+sub start_test($$)
+{
+	my ($self, $testname) = @_;
 
 	$self->{active_test} = $testname;
 	$self->{msg} = "";
 }
 
-sub end_test($$$$$$)
+sub end_test($$$$)
 {
-	my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
-
-	if ($#$parents == -1) {
-		$self->end_testsuite($testname, $result, $unexpected, $reason);
-		return;
-	}
+	my ($self, $testname, $result, $unexpected, $reason) = @_;
 
 	print TEST "<tr>";
 
diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm
index 82a73ab932..5e75ce9511 100644
--- a/selftest/output/plain.pm
+++ b/selftest/output/plain.pm
@@ -1,5 +1,19 @@
 #!/usr/bin/perl
-
+# Plain text output for selftest
+# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package output::plain;
 use Exporter;
 @ISA = qw(Exporter);
@@ -15,7 +29,7 @@ sub new($$$$$$$) {
 		verbose => $verbose, 
 		immediate => $immediate, 
 		statistics => $statistics,
-		start_time => time(),
+		start_time => undef,
 		test_output => {},
 		suitesfailed => [],
 		suites_ok => 0,
@@ -27,6 +41,21 @@ sub new($$$$$$$) {
 	bless($self, $class);
 }
 
+sub testsuite_count($$)
+{
+	my ($self, $count) = @_;
+	$self->{totalsuites} = $count;
+}
+
+sub report_time($$)
+{
+	my ($self, $time) = @_;
+	unless ($self->{start_time}) {
+		$self->{start_time} = $time;
+	}
+	$self->{last_time} = $time;
+}
+
 sub output_msg($$);
 
 sub start_testsuite($$)
@@ -35,14 +64,18 @@ sub start_testsuite($$)
 
 	$self->{index}++;
 	$self->{NAME} = $name;
-	$self->{START_TIME} = time();
+	$self->{START_TIME} = $self->{last_time};
 
 	my $duration = $self->{START_TIME} - $self->{start_time};
 
 	$self->{test_output}->{$name} = "" unless($self->{verbose});
 
 	my $out = "";
-	$out .= "[$self->{index}/$self->{totalsuites} in ".$duration."s";
+	$out .= "[$self->{index}";
+	if ($self->{totalsuites}) {
+		$out .= "/$self->{totalsuites}";
+	}
+	$out.= " in ".$duration."s";
 	$out .= sprintf(", %d errors", ($#{$self->{suitesfailed}}+1)) if ($#{$self->{suitesfailed}} > -1);
 	$out .= "] $name"; 
 	if ($self->{immediate}) {
@@ -60,8 +93,10 @@ sub output_msg($$)
 		require FileHandle;
 		print $output;
 		STDOUT->flush();
-	} else {
+	} elsif (defined($self->{NAME})) {
 		$self->{test_output}->{$self->{NAME}} .= $output;
+	} else {
+		print $output;
 	}
 }
 
@@ -69,30 +104,28 @@ sub control_msg($$)
 {
 	my ($self, $output) = @_;
 
-	$self->output_msg($output);
+	#$self->output_msg($output);
 }
 
-sub end_testsuite($$$$$)
+sub end_testsuite($$$$)
 {
-	my ($self, $name, $result, $unexpected, $reason) = @_;
+	my ($self, $name, $result, $reason) = @_;
 	my $out = "";
+	my $unexpected = 0;
 
-	if ($unexpected) {
-		if ($result eq "success" and not defined($reason)) {
-			$reason = "Expected negative exit code, got positive exit code";
-		} 
+	if ($result eq "success" or $result eq "xfail") {
+		$self->{suites_ok}++;
+	} else {
 		$self->output_msg("ERROR: $reason\n");
 		push (@{$self->{suitesfailed}}, $name);
-	} else {
-		$self->{suites_ok}++;
-	}
-
-	if ($unexpected and $self->{immediate} and not $self->{verbose}) {
-		$out .= $self->{test_output}->{$name};
+		if ($self->{immediate} and not $self->{verbose}) {
+			$out .= $self->{test_output}->{$name};
+		}
+		$unexpected = 1;
 	}
 
 	if (not $self->{immediate}) {
-		if (not $unexpected) {
+		unless($unexpected) {
 			$out .= " ok\n";
 		} else {
 			$out .= " " . uc($result) . "\n";
@@ -104,28 +137,20 @@ sub end_testsuite($$$$$)
 
 sub start_test($$$)
 {
-	my ($self, $parents, $testname) = @_;
-
-	if ($#$parents == -1) {
-		$self->start_testsuite($testname);
-	}
+	my ($self, $testname) = @_;
 }
 
 sub end_test($$$$$)
 {
-	my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
+	my ($self, $testname, $result, $unexpected, $reason) = @_;
 	
-	if ($#$parents == -1) {
-		$self->end_testsuite($testname, $result, $unexpected, $reason);
-		return;
-	}
-
 	my $append = "";
 
 	unless ($unexpected) {
 		$self->{test_output}->{$self->{NAME}} = "";
 		if (not $self->{immediate}) {
 			if ($result eq "failure") { print "f"; }
+			elsif ($result eq "xfail") { print "X"; }
 			elsif ($result eq "skip") { print "s"; }
 			elsif ($result eq "success") { print "."; }
 			else { print "?($result)"; }
@@ -133,9 +158,7 @@ sub end_test($$$$$)
 		return;
 	}
 
-	my $fullname = join(".", @$parents).".$testname";
-
-	$append = "UNEXPECTED($result): $testname ($fullname)\n";
+	$append = "UNEXPECTED($result): $testname\n";
 
 	$self->{test_output}->{$self->{NAME}} .= $append;
 
@@ -200,13 +223,18 @@ sub summary($)
 
 }
 
-sub skip_testsuite($$)
+sub skip_testsuite($$$)
 {
 	my ($self, $name, $reason) = @_;
 
+	unless (defined($reason)) {
+		$reason = "UNKNOWN";
+	}
 	push (@{$self->{skips}->{$reason}}, $name);
 
-	$self->{totalsuites}--;
+	if ($self->{totalsuites}) {
+		$self->{totalsuites}--;
+	}
 }
 
 1;
diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index ef4c385d33..1cae9eaa7d 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -1,8 +1,20 @@
 #!/usr/bin/perl
 # Bootstrap Samba and run a number of tests against it.
-# Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
-# Published under the GNU GPL, v3 or later.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 =pod
 
@@ -14,7 +26,7 @@ selftest - Samba test runner
 
 selftest --help
 
-selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
+selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
 
 =head1 DESCRIPTION
 
@@ -44,10 +56,6 @@ Executable extention
 
 Change directory to run tests in. Default is 'st'.
 
-=item I<--immediate>
-
-Show errors as soon as they happen rather than at the end of the test run.
-		
 =item I<--target samba4|samba3|win|kvm>
 
 Specify test target against which to run. Default is 'samba4'.
@@ -65,23 +73,10 @@ when the server is running locally.
 Will prevent TCP and UDP ports being opened on the local host but 
 (transparently) redirects these calls to use unix domain sockets.
 
-=item I<--expected-failures>
-
-Specify a file containing a list of tests that are expected to fail. Failures for 
-these tests will be counted as successes, successes will be counted as failures.
-
-The format for the file is, one entry per line:
-
-TESTSUITE-NAME.TEST-NAME
-
-The reason for a test can also be specified, by adding a hash sign (#) and the reason 
-after the test name.
-
 =item I<--exclude>
 
 Specify a file containing a list of tests that should be skipped. Possible 
-candidates are tests that segfault the server, flip or don't end. The format of this file is the same as 
-for the --expected-failures flag.
+candidates are tests that segfault the server, flip or don't end. 
 
 =item I<--include>
 
@@ -135,6 +130,7 @@ use POSIX;
 use Cwd qw(abs_path);
 use lib "$RealBin";
 use Subunit qw(parse_results);
+use Subunit::Filter;
 use SocketWrapper;
 
 my $opt_help = 0;
@@ -144,8 +140,6 @@ my $opt_socket_wrapper = 0;
 my $opt_socket_wrapper_pcap = undef;
 my $opt_socket_wrapper_keep_pcap = undef;
 my $opt_one = 0;
-my $opt_immediate = 0;
-my $opt_expected_failures = undef;
 my @opt_exclude = ();
 my @opt_include = ();
 my $opt_verbose = 0;
@@ -156,7 +150,6 @@ my $opt_analyse_cmd = undef;
 my $opt_resetup_env = undef;
 my $opt_bindir = undef;
 my $opt_no_lazy_setup = undef;
-my $opt_format = "plain";
 my @testlists = ();
 
 my $srcdir = ".";
@@ -164,21 +157,9 @@ my $builddir = ".";
 my $exeext = "";
 my $prefix = "./st";
 
-my @expected_failures = ();
 my @includes = ();
 my @excludes = ();
 
-my $statistics = {
-	SUITES_FAIL => 0,
-
-	TESTS_UNEXPECTED_OK => 0,
-	TESTS_EXPECTED_OK => 0,
-	TESTS_UNEXPECTED_FAIL => 0,
-	TESTS_EXPECTED_FAIL => 0,
-	TESTS_ERROR => 0,
-	TESTS_SKIP => 0,
-};
-
 sub find_in_list($$)
 {
 	my ($list, $fullname) = @_;
@@ -186,19 +167,13 @@ sub find_in_list($$)
 	foreach (@$list) {
 		if ($fullname =~ /$$_[0]/) {
 			 return ($$_[1]) if ($$_[1]);
-			 return "NO REASON SPECIFIED";
+			 return "";
 		}
 	}
 
 	return undef;
 }
 
-sub expecting_failure($)
-{
-	my ($name) = @_;
-	return find_in_list(\@expected_failures, $name);
-}
-
 sub skip($)
 {
 	my ($name) = @_;
@@ -225,63 +200,82 @@ sub setup_pcap($)
 	return $pcap_file;
 }
 
-sub cleanup_pcap($$$)
+sub cleanup_pcap($$)
 {
-	my ($pcap_file, $expected_ret, $ret) = @_;
+	my ($pcap_file, $exitcode) = @_;
 
 	return unless ($opt_socket_wrapper_pcap);
 	return if ($opt_socket_wrapper_keep_pcap);
-	return unless ($expected_ret == $ret);
+	return unless ($exitcode == 0);
 	return unless defined($pcap_file);
 
 	unlink($pcap_file);
 }
 
-sub run_testsuite($$$$$$)
+sub run_testsuite($$$$$)
 {
-	my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
+	my ($envname, $name, $cmd, $i, $totalsuites) = @_;
 	my $pcap_file = setup_pcap($name);
 
-	$msg_ops->start_test([], $name);
+	Subunit::start_testsuite($name);
+	Subunit::report_time(time());
 
-	unless (open(RESULT, "$cmd 2>&1|")) {
-		$statistics->{TESTS_ERROR}++;
-		$msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
-		$statistics->{SUITES_FAIL}++;
-		return 0;
-	}
+	open(RESULTS, "$cmd 2>&1|");
+	my $statistics = {
+		TESTS_UNEXPECTED_OK => 0,
+		TESTS_EXPECTED_OK => 0,
+		TESTS_UNEXPECTED_FAIL => 0,
+		TESTS_EXPECTED_FAIL => 0,
+		TESTS_ERROR => 0,
+		TESTS_SKIP => 0,
+	};
 
-	my $expected_ret = parse_results(
-		$msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
+	my $msg_ops = new Subunit::Filter("$name\.", []);
 
-	my $envlog = getlog_env($envname);
-	$msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
+	parse_results($msg_ops, $statistics, *RESULTS);
+
+	my $ret = 0;
+
+	unless (close(RESULTS)) {
+		if ($!) {
+			Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
+			return 0;
+		} else {
+			$ret = $?;
+		}
+	} 
 
-	$msg_ops->output_msg("CMD: $cmd\n");
+	if ($ret & 127) {
+		Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
+		return 0;
+	}
+	my $envlog = getlog_env($envname);
+	if ($envlog ne "") {
+		print "envlog: $envlog\n";
+	}
 
-	my $ret = close(RESULT);
-	$ret = 0 unless $ret == 1;
+	print "command: $cmd\n";
 
-	my $exitcode = $? >> 8;
+	my $exitcode = $ret >> 8;
 
-	if ($ret == 1) {
-		$msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef); 
+	Subunit::report_time(time());
+	if ($exitcode == 0) {
+		Subunit::end_testsuite($name, "success");
 	} else {
-		$msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
+		Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
 	}
 
-	cleanup_pcap($pcap_file, $expected_ret, $ret);
+	cleanup_pcap($pcap_file, $exitcode);
 
 	if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
-		$msg_ops->output_msg("PCAP FILE: $pcap_file\n");
+		print "PCAP FILE: $pcap_file\n";
 	}
 
-	if ($ret != $expected_ret) {
-		$statistics->{SUITES_FAIL}++;
+	if ($exitcode != 0) {
 		exit(1) if ($opt_one);
 	}
 
-	return ($ret == $expected_ret);
+	return $exitcode;
 }
 
 sub ShowHelp()
@@ -309,7 +303,6 @@ Target Specific:
                             failed
  --socket-wrapper           enable socket wrapper
  --bindir=PATH              path to target binaries
- --expected-failures=FILE   specify list of tests that is guaranteed to fail
 
 Samba4 Specific:
  --ldap=openldap|fedora-ds  back samba onto specified ldap server
@@ -320,7 +313,6 @@ Kvm Specific:
 Behaviour:
  --quick                    run quick overall test
  --one                      abort when the first test fails
- --immediate                print test output for failed tests during run
  --verbose                  be verbose
  --analyse-cmd CMD          command to run after each test
 ";
@@ -336,8 +328,6 @@ my $result = GetOptions (
 		'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 		'quick' => \$opt_quick,
 		'one' => \$opt_one,
-		'immediate' => \$opt_immediate,
-		'expected-failures=s' => \$opt_expected_failures,
 		'exclude=s' => \@opt_exclude,
 		'include=s' => \@opt_include,
 		'srcdir=s' => \$srcdir,
@@ -350,7 +340,6 @@ my $result = GetOptions (
 		'no-lazy-setup' => \$opt_no_lazy_setup,
 		'resetup-environment' => \$opt_resetup_env,
 		'bindir:s' => \$opt_bindir,
-		'format=s' => \$opt_format,
 		'image=s' => \$opt_image,
 		'testlist=s' => \@testlists
 	    );
@@ -359,7 +348,7 @@ exit(1) if (not $result);
 
 ShowHelp() if ($opt_help);
 
-my $tests = shift;
+my @tests = @ARGV;
 
 # quick hack to disable rpc validation when using valgrind - its way too slow
 unless (defined($ENV{VALGRIND})) {
@@ -410,11 +399,6 @@ $ENV{BUILDDIR} = $builddir;
 $ENV{BUILDDIR_ABS} = $builddir_abs;
 $ENV{EXEEXT} = $exeext;
 
-if (defined($ENV{RUN_FROM_BUILD_FARM}) and 
-	($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
-	$opt_format = "buildfarm";
-}
-
 my $tls_enabled = not $opt_quick;
 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
@@ -523,10 +507,6 @@ sub read_test_regexes($)
 	return @ret;
 }
 
-if (defined($opt_expected_failures)) {
-	@expected_failures = read_test_regexes($opt_expected_failures);
-}
-
 foreach (@opt_exclude) {
 	push (@excludes, read_test_regexes($_));
 }
@@ -557,12 +537,18 @@ sub write_clientconf($$)
 	        mkdir("$prefix/client/private", 0777);
 	}
 
-	if ( -d "$prefix/client/lock" ) {
+	if ( -d "$prefix/client/lockdir" ) {
 	        unlink <$prefix/client/lockdir/*>;
 	} else {
 	        mkdir("$prefix/client/lockdir", 0777);
 	}
 
+	if ( -d "$prefix_abs/client/ncalrpcdir" ) {
+	        unlink <$prefix/client/ncalrpcdir/*>;
+	} else {
+	        mkdir("$prefix/client/ncalrpcdir", 0777);
+	}
+
 	open(CF, ">$conffile");
 	print CF "[global]\n";
 	if (defined($ENV{VALGRIND})) {
@@ -583,6 +569,7 @@ sub write_clientconf($$)
 	print CF "
 	private dir = $prefix_abs/client/private
 	lock dir = $prefix_abs/client/lockdir
+	ncalrpc dir = $prefix_abs/client/ncalrpcdir
 	name resolve order = bcast
 	panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
 	max xmit = 32K
@@ -604,6 +591,20 @@ my $testsdir = "$srcdir/selftest";
 
 my %required_envs = ();
 
+sub should_run_test($)
+{
+	my $name = shift;
+	if ($#tests == -1) {
+		return 1;
+	}
+	for (my $i=0; $i <= $#tests; $i++) {
+		if ($name =~ /$tests[$i]/i) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
 sub read_testlist($)
 {
 	my ($filename) = @_;
@@ -619,7 +620,7 @@ sub read_testlist($)
 			$env =~ s/\n//g;
 			my $cmdline = <IN>;
 			$cmdline =~ s/\n//g;
-			if (not defined($tests) or $name =~ /$tests/) {
+			if (should_run_test($name) == 1) {
 				$required_envs{$env} = 1;
 				push (@ret, [$name, $env, $cmdline]);
 			}
@@ -658,32 +659,19 @@ my @available = ();
 foreach my $fn (@testlists) {
 	foreach (read_testlist($fn)) {
 		my $name = $$_[0];
-		next if (@includes and not find_in_list(\@includes, $name));
+		next if (@includes and not defined(find_in_list(\@includes, $name)));
 		push (@available, $_);
 	}
 }
 
-my $msg_ops;
-if ($opt_format eq "buildfarm") {
-	require output::buildfarm;
-	$msg_ops = new output::buildfarm($statistics);
-} elsif ($opt_format eq "plain") {
-	require output::plain;
-	$msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
-} elsif ($opt_format eq "html") {
-	require output::html;
-	mkdir("test-results", 0777);
-	$msg_ops = new output::html("test-results", $statistics);
-} else {
-	die("Invalid output format '$opt_format'");
-}
-
+Subunit::testsuite_count($#available+1);
+Subunit::report_time(time());
 
 foreach (@available) {
 	my $name = $$_[0];
 	my $skipreason = skip($name);
-	if ($skipreason) {
-		$msg_ops->skip_testsuite($name, $skipreason);
+	if (defined($skipreason)) {
+		Subunit::skip_testsuite($name, $skipreason);
 	} else {
 		push(@todo, $_); 
 	}
@@ -871,12 +859,12 @@ $envvarstr
 		
 		my $envvars = setup_env($envname);
 		if (not defined($envvars)) {
-			$msg_ops->skip_testsuite($name, "unable to set up environment $envname");
+			Subunit::skip_testsuite($name, 
+				"unable to set up environment $envname");
 			next;
 		}
 
-		run_testsuite($envname, $name, $cmd, $i, $suitestotal, 
-		              $msg_ops);
+		run_testsuite($envname, $name, $cmd, $i, $suitestotal);
 
 		if (defined($opt_analyse_cmd)) {
 			system("$opt_analyse_cmd \"$name\"");
@@ -892,8 +880,6 @@ teardown_env($_) foreach (keys %running_envs);
 
 $target->stop();
 
-$msg_ops->summary();
-
 my $failed = 0;
 
 # if there were any valgrind failures, show them
@@ -906,9 +892,4 @@ foreach (<$prefix/valgrind.log*>) {
 	    system("cat $_");
 	}
 }
-
-if ($opt_format eq "buildfarm") {
-	print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
-}
-
-exit $statistics->{SUITES_FAIL};
+exit 0;
diff --git a/selftest/target/Samba3.pm b/selftest/target/Samba3.pm
index c636be22f1..e1e5210696 100644
--- a/selftest/target/Samba3.pm
+++ b/selftest/target/Samba3.pm
@@ -41,6 +41,9 @@ sub teardown_env($$)
 	$self->stop_sig_term($smbdpid);
 	$self->stop_sig_term($nmbdpid);
 	$self->stop_sig_term($winbinddpid);
+
+	sleep(2);
+
 	$self->stop_sig_kill($smbdpid);
 	$self->stop_sig_kill($nmbdpid);
 	$self->stop_sig_kill($winbinddpid);
@@ -124,9 +127,8 @@ sub setup_dc($$)
 				    $dc_options);
 
 	$self->check_or_start($vars,
-			      ($ENV{NMBD_MAXTIME} or 2700),
-			      ($ENV{WINBINDD_MAXTIME} or 2700),
-			      ($ENV{SMBD_MAXTIME} or 2700));
+			      ($ENV{SMBD_MAXTIME} or 2700),
+			       "yes", "yes", "yes");
 
 	$self->wait_for_start($vars);
 
@@ -143,6 +145,7 @@ sub setup_member($$$)
 
 	my $member_options = "
 	security = domain
+	server signing = on
 ";
 	my $ret = $self->provision($prefix,
 				   "LOCALMEMBER3",
@@ -161,9 +164,8 @@ sub setup_member($$$)
 	system($cmd) == 0 or die("Join failed\n$cmd");
 
 	$self->check_or_start($ret,
-			      ($ENV{NMBD_MAXTIME} or 2700),
-			      ($ENV{WINBINDD_MAXTIME} or 2700),
-			      ($ENV{SMBD_MAXTIME} or 2700));
+			      ($ENV{SMBD_MAXTIME} or 2700),
+			       "yes", "yes", "yes");
 
 	$self->wait_for_start($ret);
 
@@ -188,7 +190,7 @@ sub stop_sig_term($$) {
 
 sub stop_sig_kill($$) {
 	my ($self, $pid) = @_;
-	kill("KILL", $pid) or warn("Unable to kill $pid: $!");
+	kill("ALRM", $pid) or warn("Unable to kill $pid: $!");
 }
 
 sub write_pid($$$)
@@ -210,8 +212,8 @@ sub read_pid($$)
 	return $pid;
 }
 
-sub check_or_start($$$$) {
-	my ($self, $env_vars, $nmbd_maxtime, $winbindd_maxtime, $smbd_maxtime) = @_;
+sub check_or_start($$$$$) {
+	my ($self, $env_vars, $maxtime, $nmbd, $winbindd, $smbd) = @_;
 
 	unlink($env_vars->{NMBD_TEST_LOG});
 	print "STARTING NMBD...";
@@ -226,14 +228,15 @@ sub check_or_start($$$$) {
 
 		$ENV{NSS_WRAPPER_PASSWD} = $env_vars->{NSS_WRAPPER_PASSWD};
 		$ENV{NSS_WRAPPER_GROUP} = $env_vars->{NSS_WRAPPER_GROUP};
+		$ENV{NSS_WRAPPER_WINBIND_SO_PATH} = $env_vars->{NSS_WRAPPER_WINBIND_SO_PATH};
 
-		if ($nmbd_maxtime eq "skip") {
+		if ($nmbd ne "yes") {
 			$SIG{USR1} = $SIG{ALRM} = $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
 				my $signame = shift;
 				print("Skip nmbd received signal $signame");
 				exit 0;
 			};
-			sleep(999999);
+			sleep($maxtime);
 			exit 0;
 		}
 
@@ -244,7 +247,7 @@ sub check_or_start($$$$) {
 
 		$ENV{MAKE_TEST_BINARY} = $self->binpath("nmbd");
 
-		my @preargs = ($self->binpath("timelimit"), $nmbd_maxtime);
+		my @preargs = ($self->binpath("timelimit"), $maxtime);
 		if(defined($ENV{NMBD_VALGRIND})) { 
 			@preargs = split(/ /, $ENV{NMBD_VALGRIND});
 		}
@@ -267,14 +270,15 @@ sub check_or_start($$$$) {
 
 		$ENV{NSS_WRAPPER_PASSWD} = $env_vars->{NSS_WRAPPER_PASSWD};
 		$ENV{NSS_WRAPPER_GROUP} = $env_vars->{NSS_WRAPPER_GROUP};
+		$ENV{NSS_WRAPPER_WINBIND_SO_PATH} = $env_vars->{NSS_WRAPPER_WINBIND_SO_PATH};
 
-		if ($winbindd_maxtime eq "skip") {
+		if ($winbindd ne "yes") {
 			$SIG{USR1} = $SIG{ALRM} = $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
 				my $signame = shift;
 				print("Skip winbindd received signal $signame");
 				exit 0;
 			};
-			sleep(999999);
+			sleep($maxtime);
 			exit 0;
 		}
 
@@ -285,7 +289,7 @@ sub check_or_start($$$$) {
 
 		$ENV{MAKE_TEST_BINARY} = $self->binpath("winbindd");
 
-		my @preargs = ($self->binpath("timelimit"), $winbindd_maxtime);
+		my @preargs = ($self->binpath("timelimit"), $maxtime);
 		if(defined($ENV{WINBINDD_VALGRIND})) {
 			@preargs = split(/ /, $ENV{WINBINDD_VALGRIND});
 		}
@@ -308,14 +312,15 @@ sub check_or_start($$$$) {
 
 		$ENV{NSS_WRAPPER_PASSWD} = $env_vars->{NSS_WRAPPER_PASSWD};
 		$ENV{NSS_WRAPPER_GROUP} = $env_vars->{NSS_WRAPPER_GROUP};
+		$ENV{NSS_WRAPPER_WINBIND_SO_PATH} = $env_vars->{NSS_WRAPPER_WINBIND_SO_PATH};
 
-		if ($smbd_maxtime eq "skip") {
+		if ($smbd ne "yes") {
 			$SIG{USR1} = $SIG{ALRM} = $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
 				my $signame = shift;
 				print("Skip smbd received signal $signame");
 				exit 0;
 			};
-			sleep(999999);
+			sleep($maxtime);
 			exit 0;
 		}
 
@@ -324,7 +329,7 @@ sub check_or_start($$$$) {
 		if (defined($ENV{SMBD_OPTIONS})) {
 			@optargs = split(/ /, $ENV{SMBD_OPTIONS});
 		}
-		my @preargs = ($self->binpath("timelimit"), $smbd_maxtime);
+		my @preargs = ($self->binpath("timelimit"), $maxtime);
 		if(defined($ENV{SMBD_VALGRIND})) {
 			@preargs = split(/ /,$ENV{SMBD_VALGRIND});
 		}
@@ -540,6 +545,7 @@ $unix_name:x:$unix_uid:$unix_gids[0]:$unix_name gecos:$prefix_abs:/bin/false
 	open(GROUP, ">$nss_wrapper_group") or die("Unable to open $nss_wrapper_group");
 	print GROUP "nobody:x:65533:
 nogroup:x:65534:nobody
+root:x:65532:
 $unix_name-group:x:$unix_gids[0]:
 ";
 	close(GROUP);
@@ -576,6 +582,7 @@ $unix_name-group:x:$unix_gids[0]:
 	$ret{SOCKET_WRAPPER_DEFAULT_IFACE} = $swiface;
 	$ret{NSS_WRAPPER_PASSWD} = $nss_wrapper_passwd;
 	$ret{NSS_WRAPPER_GROUP} = $nss_wrapper_group;
+	$ret{NSS_WRAPPER_WINBIND_SO_PATH} = $ENV{NSS_WRAPPER_WINBIND_SO_PATH};
 
 	return \%ret;
 }
diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm
index 71dddf6939..3c0c4f5152 100644
--- a/selftest/target/Samba4.pm
+++ b/selftest/target/Samba4.pm
@@ -31,34 +31,27 @@ sub bindir_path($$) {
 }
 
 sub openldap_start($$$) {
-        my ($slapd_conf, $uri, $logs) = @_;
-  	my $oldpath = $ENV{PATH};
-	my $olroot = "";
-	my $olpath = "";
-	if (defined $ENV{OPENLDAP_ROOT}) {
-	    $olroot = "$ENV{OPENLDAP_ROOT}";
-	    $olpath = "$olroot/libexec:$olroot/sbin:";
-	}
-	$ENV{PATH} = "$olpath/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
-        system("slapd -d0 -f $slapd_conf -h $uri > $logs 2>&1 &");
-        $ENV{PATH} = $oldpath;
 }
 
 sub slapd_start($$)
 {
 	my $count = 0;
 	my ($self, $env_vars) = @_;
+	my $ldbsearch = $self->bindir_path("ldbsearch");
 
 	my $uri = $env_vars->{LDAP_URI};
 
+	if (system("$ldbsearch -H $uri -s base -b \"\" supportedLDAPVersion > /dev/null") == 0) {
+	    print "A SLAPD is still listening to $uri before we started the LDAP backend.  Aborting!";
+	    return 1;
+	}
 	# running slapd in the background means it stays in the same process group, so it can be
 	# killed by timelimit
 	if ($self->{ldap} eq "fedora-ds") {
 	        system("$ENV{FEDORA_DS_ROOT}/sbin/ns-slapd -D $env_vars->{FEDORA_DS_DIR} -d0 -i $env_vars->{FEDORA_DS_PIDFILE}> $env_vars->{LDAPDIR}/logs 2>&1 &");
 	} elsif ($self->{ldap} eq "openldap") {
-	        openldap_start($env_vars->{SLAPD_CONF}, $uri, "$env_vars->{LDAPDIR}/logs");
+	        system("$ENV{OPENLDAP_SLAPD} -d0 -F $env_vars->{SLAPD_CONF_D} -h $uri > $env_vars->{LDAPDIR}/logs 2>&1 &");
 	}
-	my $ldbsearch = $self->bindir_path("ldbsearch");
 	while (system("$ldbsearch -H $uri -s base -b \"\" supportedLDAPVersion > /dev/null") != 0) {
 	        $count++;
 		if ($count > 40) {
@@ -97,36 +90,51 @@ sub check_or_start($$$)
 	my $pid = fork();
 	if ($pid == 0) {
 		open STDIN, $env_vars->{SAMBA_TEST_FIFO};
-		open STDOUT, ">$env_vars->{SAMBA_TEST_LOG}";
+		# we want out from samba to go to the log file, but also
+		# to the users terminal when running 'make test' on the command
+		# line. This puts it on stderr on the terminal
+		open STDOUT, "| tee $env_vars->{SAMBA_TEST_LOG} 1>&2";
 		open STDERR, '>&STDOUT';
-		
+
 		SocketWrapper::set_default_iface($env_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
 
 		my $valgrind = "";
-		if (defined($ENV{SMBD_VALGRIND})) {
-		    $valgrind = $ENV{SMBD_VALGRIND};
+		if (defined($ENV{SAMBA_VALGRIND})) {
+		    $valgrind = $ENV{SAMBA_VALGRIND};
 		} 
 
 		$ENV{KRB5_CONFIG} = $env_vars->{KRB5_CONFIG}; 
+		$ENV{WINBINDD_SOCKET_DIR} = $env_vars->{WINBINDD_SOCKET_DIR};
 
 		$ENV{NSS_WRAPPER_PASSWD} = $env_vars->{NSS_WRAPPER_PASSWD};
 		$ENV{NSS_WRAPPER_GROUP} = $env_vars->{NSS_WRAPPER_GROUP};
 
+		$ENV{UID_WRAPPER} = "1";
+
 		# Start slapd before samba, but with the fifo on stdin
 		if (defined($self->{ldap})) {
 		    $self->slapd_start($env_vars) or 
-			die("couldn't start slapd (2nd time)");
+			die("couldn't start slapd (main run)");
 		}
 
 		my $optarg = "";
 		if (defined($max_time)) {
 			$optarg = "--maximum-runtime=$max_time ";
 		}
-		if (defined($ENV{SMBD_OPTIONS})) {
-			$optarg.= " $ENV{SMBD_OPTIONS}";
+		if (defined($ENV{SAMBA_OPTIONS})) {
+			$optarg.= " $ENV{SAMBA_OPTIONS}";
 		}
 		my $samba = $self->bindir_path("samba");
-		my $ret = system("$valgrind $samba $optarg $env_vars->{CONFIGURATION} -M single -i --leak-report-full");
+
+		# allow selection of the process model using
+		# the environment varibale SAMBA_PROCESS_MODEL
+		# that allows us to change the process model for 
+		# individual machines in the build farm
+		my $model = "single";
+		if (defined($ENV{SAMBA_PROCESS_MODEL})) {
+			$model = $ENV{SAMBA_PROCESS_MODEL};
+		}
+		my $ret = system("$valgrind $samba $optarg $env_vars->{CONFIGURATION} -M $model -i");
 		if ($? == -1) {
 			print "Unable to start $samba: $ret: $!\n";
 			exit 1;
@@ -201,83 +209,26 @@ type: 0x3
 ");
 }
 
-sub mk_fedora_ds($$$)
+sub mk_fedora_ds($$)
 {
-	my ($self, $ldapdir, $configuration) = @_;
-
-	my $fedora_ds_inf = "$ldapdir/fedorads.inf";
-	my $fedora_ds_extra_ldif = "$ldapdir/fedorads-partitions.ldif";
+	my ($self, $ldapdir) = @_;
 
 	#Make the subdirectory be as fedora DS would expect
 	my $fedora_ds_dir = "$ldapdir/slapd-samba4";
 
 	my $pidfile = "$fedora_ds_dir/logs/slapd-samba4.pid";
 
-my $dir = getcwd();
-chdir "$ENV{FEDORA_DS_ROOT}/bin" || die;
-	if (system("perl $ENV{FEDORA_DS_ROOT}/sbin/setup-ds.pl --silent --file=$fedora_ds_inf >&2") != 0) {
-            chdir $dir;
-            die("perl $ENV{FEDORA_DS_ROOT}/sbin/setup-ds.pl --silent --file=$fedora_ds_inf FAILED: $?");
-        }
-        chdir $dir || die;
-
 	return ($fedora_ds_dir, $pidfile);
 }
 
-sub mk_openldap($$$)
+sub mk_openldap($$)
 {
-	my ($self, $ldapdir, $configuration) = @_;
+	my ($self, $ldapdir) = @_;
 
-	my $slapd_conf = "$ldapdir/slapd.conf";
+	my $slapd_conf_d = "$ldapdir/slapd.d";
 	my $pidfile = "$ldapdir/slapd.pid";
-	my $modconf = "$ldapdir/modules.conf";
-
-	my $oldpath = $ENV{PATH};
-	my $olpath = "";
-	my $olroot = "";
-	if (defined $ENV{OPENLDAP_ROOT}) {
-               $olroot = "$ENV{OPENLDAP_ROOT}";
-	       $olpath = "$olroot/libexec:$olroot/sbin:";
-	}
-	$ENV{PATH} = "$olpath/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
-
-	unlink($modconf);
-
-	#This code tries to guess what modules we need to load (if any) by trying different combinations in the modules.conf
-
-	# Try without any slapd modules
-	open(CONF, ">$modconf"); close(CONF);
-
-	if (system("slaptest -u -f $slapd_conf >&2") != 0) {
-		open(CONF, ">$modconf"); 
-		# enable slapd modules
-		print CONF "
-moduleload	syncprov
-moduleload      memberof
-moduleload      refint
-moduleload      deref
-";
-		close(CONF);
-	}
-	if (system("slaptest -u -f $slapd_conf >&2") != 0) {
-		open(CONF, ">$modconf"); 
-		# enable slapd modules, and the module for back_hdb
-		print CONF "
-moduleload	back_hdb
-moduleload	syncprov
-moduleload      memberof
-moduleload      refint
-moduleload      deref
-";
-		close(CONF);
-	}
-
-	system("slaptest -u -f $slapd_conf") == 0 or die("slaptest still fails after adding modules");
-
-    
-	$ENV{PATH} = $oldpath;
 
-	return ($slapd_conf, $pidfile);
+	return ($slapd_conf_d, $pidfile);
 }
 
 sub mk_keyblobs($$)
@@ -294,6 +245,7 @@ sub mk_keyblobs($$)
 	my $adminkeyfile = "$tlsdir/adminkey.pem";
 	my $reqadmin = "$tlsdir/req-admin.der";
 	my $admincertfile = "$tlsdir/admincert.pem";
+	my $admincertupnfile = "$tlsdir/admincertupn.pem";
 
 	mkdir($tlsdir, 0777);
 
@@ -441,24 +393,51 @@ EOF
 	open(ADMINCERTFILE, ">$admincertfile");
 	print ADMINCERTFILE <<EOF;
 -----BEGIN CERTIFICATE-----
-MIIDHTCCAoagAwIBAgIUC0W5dW/N9kE+NgD0mKK34YgyqQ0wCwYJKoZIhvcNAQEFMFIxEzAR
+MIIDHTCCAoagAwIBAgIUUggzW4lLRkMKe1DAR2NKatkMDYwwCwYJKoZIhvcNAQELMFIxEzAR
 BgoJkiaJk/IsZAEZDANjb20xFzAVBgoJkiaJk/IsZAEZDAdleGFtcGxlMRUwEwYKCZImiZPy
-LGQBGQwFc2FtYmExCzAJBgNVBAMMAkNBMCIYDzIwMDgwMzAxMTMyMzAwWhgPMjAzMzAyMjQx
-MzIzMDBaMG0xEzARBgoJkiaJk/IsZAEZDANjb20xFzAVBgoJkiaJk/IsZAEZDAdleGFtcGxl
+LGQBGQwFc2FtYmExCzAJBgNVBAMMAkNBMCIYDzIwMDkwNzI3MDMzMjE1WhgPMjAzNDA3MjIw
+MzMyMTVaMG0xEzARBgoJkiaJk/IsZAEZDANjb20xFzAVBgoJkiaJk/IsZAEZDAdleGFtcGxl
 MRUwEwYKCZImiZPyLGQBGQwFc2FtYmExDjAMBgNVBAMMBXVzZXJzMRYwFAYDVQQDDA1BZG1p
 bmlzdHJhdG9yMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQD0+OL7TQBj0RejbIH1+g5G
 eRaWaM9xF43uE5y7jUHEsi5owhZF5iIoHZeeL6cpDF5y1BZRs0JlA1VqMry1jjKlzFYVEMMF
 xB6esnXhl0Jpip1JkUMMXLOP1m/0dqayuHBWozj9f/cdyCJr0wJIX1Z8Pr+EjYRGPn/MF0xd
 l3JRlwIDAQABo4HSMIHPMA4GA1UdDwEB/wQEAwIFoDAoBgNVHSUEITAfBgcrBgEFAgMEBggr
 BgEFBQcDAgYKKwYBBAGCNxQCAjBIBgNVHREEQTA/oD0GBisGAQUCAqAzMDGgExsRU0FNQkEu
-RVhBTVBMRS5DT02hGjAYoAMCAQGhETAPGw1hZG1pbmlzdHJhdG9yMB8GA1UdIwQYMBaAFMLZ
+RVhBTVBMRS5DT02hGjAYoAMCAQGhETAPGw1BZG1pbmlzdHJhdG9yMB8GA1UdIwQYMBaAFMLZ
 ufegDKLZs0VOyFXYK1L6M8oyMB0GA1UdDgQWBBQg81bLyfCA88C2B/BDjXlGuaFaxjAJBgNV
-HRMEAjAAMA0GCSqGSIb3DQEBBQUAA4GBAHsqSqul0hZCXn4t8Kfp3v/JLMiUMJihR1XOgzoa
-ufLOQ1HNzFUHKuo1JEQ1+i5gHT/arLu/ZBF4BfQol7vW27gKIEt0fkRV8EvoPxXvSokHq0Ku
-HCuPOhYNEP3wYiwB3g93NMCinWVlz0mh5aijEU7y/XrjlZxBKFFrTE+BJi1o
+HRMEAjAAMA0GCSqGSIb3DQEBCwUAA4GBAEf/OSHUDJaGdtWGNuJeqcVYVMwrfBAc0OSwVhz1
+7/xqKHWo8wIMPkYRtaRHKLNDsF8GkhQPCpVsa6mX/Nt7YQnNvwd+1SBP5E8GvwWw9ZzLJvma
+nk2n89emuayLpVtp00PymrDLRBcNaRjFReQU8f0o509kiVPHduAp3jOiy13l
 -----END CERTIFICATE-----
 EOF
 	close(ADMINCERTFILE);
+
+	# hxtool issue-certificate --ca-certificate=FILE:$CAFILE,$KEYFILE \
+	# --type="pkinit-client" \
+	# --ms-upn="administrator@samba.example.com" \
+	# --req="PKCS10:$ADMINREQFILE" --certificate="FILE:$ADMINCERTUPNFILE" \
+	# --lifetime="25 years"
+	
+	open(ADMINCERTUPNFILE, ">$admincertupnfile");
+	print ADMINCERTUPNFILE <<EOF;
+-----BEGIN CERTIFICATE-----
+MIIDDzCCAnigAwIBAgIUUp3CJMuNaEaAdPKp3QdNIwG7a4wwCwYJKoZIhvcNAQELMFIxEzAR
+BgoJkiaJk/IsZAEZDANjb20xFzAVBgoJkiaJk/IsZAEZDAdleGFtcGxlMRUwEwYKCZImiZPy
+LGQBGQwFc2FtYmExCzAJBgNVBAMMAkNBMCIYDzIwMDkwNzI3MDMzMzA1WhgPMjAzNDA3MjIw
+MzMzMDVaMG0xEzARBgoJkiaJk/IsZAEZDANjb20xFzAVBgoJkiaJk/IsZAEZDAdleGFtcGxl
+MRUwEwYKCZImiZPyLGQBGQwFc2FtYmExDjAMBgNVBAMMBXVzZXJzMRYwFAYDVQQDDA1BZG1p
+bmlzdHJhdG9yMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQD0+OL7TQBj0RejbIH1+g5G
+eRaWaM9xF43uE5y7jUHEsi5owhZF5iIoHZeeL6cpDF5y1BZRs0JlA1VqMry1jjKlzFYVEMMF
+xB6esnXhl0Jpip1JkUMMXLOP1m/0dqayuHBWozj9f/cdyCJr0wJIX1Z8Pr+EjYRGPn/MF0xd
+l3JRlwIDAQABo4HEMIHBMA4GA1UdDwEB/wQEAwIFoDAoBgNVHSUEITAfBgcrBgEFAgMEBggr
+BgEFBQcDAgYKKwYBBAGCNxQCAjA6BgNVHREEMzAxoC8GCisGAQQBgjcUAgOgIQwfYWRtaW5p
+c3RyYXRvckBzYW1iYS5leGFtcGxlLmNvbTAfBgNVHSMEGDAWgBTC2bn3oAyi2bNFTshV2CtS
++jPKMjAdBgNVHQ4EFgQUIPNWy8nwgPPAtgfwQ415RrmhWsYwCQYDVR0TBAIwADANBgkqhkiG
+9w0BAQsFAAOBgQBk42+egeUB3Ji2PC55fbt3FNKxvmm2xUUFkV9POK/YR9rajKOwk5jtYSeS
+Zd7J9s//rNFNa7waklFkDaY56+QWTFtdvxfE+KoHaqt6X8u6pqi7p3M4wDKQox+9Dx8yWFyq
+Wfz/8alZ5aMezCQzXJyIaJsCLeKABosSwHcpAFmxlQ==
+-----END CERTIFICATE-----
+EOF
 }
 
 #
@@ -487,7 +466,7 @@ sub provision_raw_prepare($$$$$$$)
 	$ctx->{kdc_ipv4} = $kdc_ipv4;
 
 	$ctx->{server_loglevel} = 1;
-	$ctx->{username} = "administrator";
+	$ctx->{username} = "Administrator";
 	$ctx->{domain} = "SAMBADOMAIN";
 	$ctx->{realm} = "SAMBA.EXAMPLE.COM";
 	$ctx->{dnsname} = "samba.example.com";
@@ -744,7 +723,6 @@ sub provision($$$$$$$)
 [tmp]
 	path = $ctx->{tmpdir}
 	read only = no
-	ntvfs handler = posix
 	posix:sharedelay = 100000
 	posix:eadb = $ctx->{lockdir}/eadb.tdb
 	posix:oplocktimeout = 3
@@ -753,7 +731,6 @@ sub provision($$$$$$$)
 [test1]
 	path = $ctx->{tmpdir}/test1
 	read only = no
-	ntvfs handler = posix
 	posix:sharedelay = 100000
 	posix:eadb = $ctx->{lockdir}/eadb.tdb
 	posix:oplocktimeout = 3
@@ -762,7 +739,6 @@ sub provision($$$$$$$)
 [test2]
 	path = $ctx->{tmpdir}/test2
 	read only = no
-	ntvfs handler = posix
 	posix:sharedelay = 100000
 	posix:eadb = $ctx->{lockdir}/eadb.tdb
 	posix:oplocktimeout = 3
@@ -807,37 +783,22 @@ sub provision($$$$$$$)
 	my $ret = $self->provision_raw_step1($ctx);
 
 	if (defined($self->{ldap})) {
-		my $configuration = "--configfile=$ctx->{smb_conf}";
-
-		$ret->{LDAP_URI} = $ctx->{ldap_uri};
-		push (@{$ctx->{provision_options}},"--ldap-backend=$ctx->{ldap_uri}");
-
-		system("$self->{setupdir}/provision-backend $configuration --ldap-admin-pass=$ctx->{password} --root=$ctx->{unix_name} --realm=$ctx->{realm} --domain=$ctx->{domain} --host-name=$ctx->{netbiosname} --ldap-backend-type=$self->{ldap}>&2") == 0 or die("backend provision failed");
-
-		push (@{$ctx->{provision_options}}, "--password=$ctx->{password}");
-
+                $ret->{LDAP_URI} = $ctx->{ldap_uri};
+		push (@{$ctx->{provision_options}}, "--ldap-backend-type=" . $self->{ldap});
 		if ($self->{ldap} eq "openldap") {
-			push (@{$ctx->{provision_options}}, "--username=samba-admin");
-			push (@{$ctx->{provision_options}}, "--ldap-backend-type=openldap");
+ 		        push (@{$ctx->{provision_options}}, "--slapd-path=" . $ENV{OPENLDAP_SLAPD});
+			($ret->{SLAPD_CONF_D}, $ret->{OPENLDAP_PIDFILE}) = $self->mk_openldap($ctx->{ldapdir}) or die("Unable to create openldap directories");
 
-			($ret->{SLAPD_CONF}, $ret->{OPENLDAP_PIDFILE}) = $self->mk_openldap($ctx->{ldapdir}, $configuration) or die("Unable to create openldap directories");
-
-		} elsif ($self->{ldap} eq "fedora-ds") {
-			push (@{$ctx->{provision_options}}, "--simple-bind-dn=cn=Manager,$ctx->{localbasedn}");
-			push (@{$ctx->{provision_options}}, "--ldap-backend-type=fedora-ds");
-
-			($ret->{FEDORA_DS_DIR}, $ret->{FEDORA_DS_PIDFILE}) = $self->mk_fedora_ds($ctx->{ldapdir}, $configuration) or die("Unable to create fedora ds directories");
+                } elsif ($self->{ldap} eq "fedora-ds") {
+ 		        push (@{$ctx->{provision_options}}, "--slapd-path=" . "$ENV{FEDORA_DS_ROOT}/sbin/ns-slapd");
+ 		        push (@{$ctx->{provision_options}}, "--setup-ds-path=" . "$ENV{FEDORA_DS_ROOT}/sbin/setup-ds.pl");
+			($ret->{FEDORA_DS_DIR}, $ret->{FEDORA_DS_PIDFILE}) = $self->mk_fedora_ds($ctx->{ldapdir}) or die("Unable to create fedora ds directories");
 		}
 
-		$self->slapd_start($ret) or die("couldn't start slapd");
 	}
 
 	$ret = $self->provision_raw_step2($ctx, $ret);
 
-	if (defined($self->{ldap})) {
-		$self->slapd_stop($ret) or die("couldn't stop slapd");
-	}
-
 	return $ret;
 }
 
diff --git a/selftest/test_samba4.pl b/selftest/test_samba4.pl
index f2935be66b..59763afdaa 100755
--- a/selftest/test_samba4.pl
+++ b/selftest/test_samba4.pl
@@ -3,13 +3,14 @@
 use Test::More tests => 3;
 use FindBin qw($RealBin);
 use lib $RealBin;
+use lib "$RealBin/target";
 use Samba4;
 
-my $s = new Samba4($RealBin."/../bin", undef, $RealBin."/../setup");
+my $s = new Samba4($RealBin."/../source4/bin", undef, $RealBin."/../setup");
 
 ok($s);
 
-is($RealBin."/../bin", $s->{bindir});
+is($RealBin."/../source4/bin", $s->{bindir});
 
 ok($s->write_ldb_file("tmpldb", "
 dn: a=b
diff --git a/selftest/test_subunit.pl b/selftest/test_subunit.pl
deleted file mode 100755
index 28522ed49f..0000000000
--- a/selftest/test_subunit.pl
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/perl
-
-use Test::More tests => 0;
-use FindBin qw($RealBin);
-use lib $RealBin;
-use Subunit qw(parse_results);
-