path: root/regress/infra-unittests/test.subr

#! /bin/sh
# $NetBSD: test.subr,v 1.15 2020/05/12 05:34:04 rillig Exp $
#
# This file defines utilities for testing Makefile fragments and shell
# programs from the pkgsrc infrastructure. While testing one part of the
# infrastructure, other parts can be mocked away.
#
# A test case is defined by the following functions:
#
#	test_case_begin
#	test_case_set_up
#	test_case_tear_down
#	test_case_end
#
# These functions form the structure for a test. To define a test, use the
# following pattern:
#
# 	if test_case_begin "description of the test"; then
#		...
#		test_case_end
#	fi
#
# The functions test_case_set_up and test_case_tear_down can be defined in the
# test file to provide a test fixture that is common to all test cases. These
# functions are called by test_case_begin and test_case_end, respectively.
#
# During a test case, the following variables are defined:
#
# tmpdir
#	a directory for creating intermediate files
#
# pkgsrcdir
#	the real pkgsrc directory
#
# mocked_pkgsrcdir
#	the directory where the mock files are installed, which override
#	the Makefile fragments from the real pkgsrc infrastructure
#
#
# Setting up a test
#
# mock_cmd
#	Returns the path to a newly created shell program whose behavior
#	(output and exit status) is specified by pairs of
#	--when-args/--then-output or --when-args/--then-exit.
#
#	Example:
#
#	hello=$(mock_cmd mock-hello \
#		--when-args "" --then-output "Hello, world!" \
#		--when-args "-t" --then-output "hello, world" \
#		--when-args "-?" --then-exit 1
#	)
#
# create_file $filename <<EOF ... EOF
#	Creates a file in the temporary directory. The filename is relative
#	to the temporary directory.
#
# create_file_lines $filename $line1 $line2 ...
#	Creates a file in the temporary directory containing the given lines.
#
# create_pkgsrc_file $filename <<EOF ... EOF
#	Creates a file in the temporary pkgsrc directory. If it is a Makefile
#	fragment, it will be included instead of the one in the real pkgsrc
#	infrastructure.
#
#	This is typically used for creating an empty file for mk/bsd.prefs.mk
#	or similar files that are included by the file that is currently under
#	test.
#
#	Example:
#
#	create_pkgsrc_file "mk/pkg-build-options.mk" <<EOF
#	# nothing
#	EOF
#
#
# Running the code to be tested
#
# run_bmake $filename $bmake_args...
#	Runs bmake with the correct environment so that it picks up the mocked
#	infrastructure files first and then the ones from the real pkgsrc
#	installation. The filename is relative to the temporary directory.
#
#
# Checking the result
#
# assert_that $actual --equals $expected
#	Complains loudly if the string $actual is not equal to $expected.
#
# assert_that $tmpfile --file-contains-exactly $content
#	Complains loudly if the file in $tmpdir does not contain the $content,
#	plus a trailing newline.
#
# assert_that $tmpfile1 --file-equals $tmpfile2
#	Complains loudly if the two files in $tmpdir differ.
#
# assert_that $tmpfile --file-is-empty
#	Complains loudly if the file in $tmpdir is not empty.
#
# assert_that $tmpfile --file-is-lines $lines...
#	Complains loudly if the file in $tmpdir does not consist of exactly
#	the given lines.
#
# assert_succeed
#	Counts one succeeded assertion, for the verbose statistics.
#
# assert_fail $format $args...
#	Marks the current test as failed but continues to execute it.
#

set -eu

cleanup='yes'
: "${make:=bmake}"
if_verbose=':'
tmpdir="${TMPDIR:-/tmp}/infra-unittests-$$"
mocked_pkgsrcdir="$tmpdir/pkgsrc"
test_case_filter='.*'

rm -rf "$tmpdir"
mkdir -p "$mocked_pkgsrcdir"

while getopts 'kvf:' opt; do
	case "$opt" in
	(f)	test_case_filter="$OPTARG";;
	(k)	cleanup='no';;
	(v)	if_verbose='';;
	(*)	printf '%s\n' 1>&2 \
			"usage: $0 [-kv] [-f filter]" \
			'  -f regex	only run matching test cases' \
			'  -k		keep the temporary files' \
			'  -v		verbose mode'
		exit 1;;
	esac
done

pkgsrcdir=""
for relative_pkgsrcdir in . .. ../.. ../../..; do
	if [ -f "$relative_pkgsrcdir/mk/bsd.pkg.mk" ]; then
		pkgsrcdir="$PWD/$relative_pkgsrcdir"
		break
	fi
done
if [ -z "$pkgsrcdir" ]; then
	printf 'error: must be run from somewhere inside the pkgsrc directory\n' 1>&2
	exit 1
fi

verbose_printf() {
	$if_verbose printf "$@"
}

test_case_name="unknown test"
test_case_begun=0
test_case_ended=0

test_case_begin() {
	test_case_name="$1"
	[ "$(expr "$test_case_name" : "$test_case_filter")" -gt 0 ] \
	|| return 1

	test_case_begun="`expr "$test_case_begun" + 1`"
	verbose_printf 'running test case "%s"\n' "$test_case_name"

	cd "$tmpdir" || exit 1
	rm -rf 'work'
	mkdir 'work' || exit 1
	cd 'work' || exit 1

	test_case_set_up
}

# Can be redefined by actual tests.
test_case_set_up() {
	:
}

# Can be redefined by actual tests.
test_case_tear_down() {
	:
}

test_case_end() {
	test_case_tear_down

	test_case_ended="`expr "$test_case_ended" + 1`"
	test "$test_case_ended" = "$test_case_begun" \
	|| assert_fail 'unbalanced test_case_begin (%d) and test_case_end (%d)\n' \
		"$test_case_begun" "$test_case_ended"

	test_case_name="unknown test"
}

test_subr_cleanup() {
	exit_status=$?
	if [ $exit_status -ne 0 ]; then
		printf 'info: the test files are in %s\n' "$tmpdir" 1>&2
		exit $exit_status
	fi

	[ "$cleanup" = "yes" ] && rm -rf "$tmpdir"

	verbose_printf '%s%d assertions succeeded, %d failed\n' \
		"$assert_fail_sep" "$assert_succeeded" "$assert_failed"

	if [ "$assert_failed" != 0 ]; then
		exit 1
	fi
}
trap "test_subr_cleanup" EXIT

mock_cmd() {
	cmdname="$1"
	shift 1

	{
	printf '#! /bin/sh\n'
	printf '\n'

	while [ $# -ge 4 ]; do
		case $1,$3 in
		(--when-args,--then-output)
			cat <<EOF
[ "x\$*" = "x$2" ] && { printf '%s\n' "$4" && exit 0; }

EOF
			shift 4
			;;
		(--when-args,--then-exit)
			cat <<EOF
[ "x\$*" = "x$2" ] && exit $4

EOF
			shift 4
			;;
		*)	printf 'error: invalid arguments to mock_cmd: %s\n' "$*" 1>&2
			exit 1
			;;
		esac
	done
	cat <<EOF
printf 'error: %s: no mock behavior defined for arguments "%s"\n' "\$0" "\$*" 1>&2
exit 1
EOF
	} > "$tmpdir/$cmdname"
	chmod +x "$tmpdir/$cmdname"

	printf '%s\n' "$tmpdir/$cmdname"
}

create_file() {
	assert_that "$#" --equals 1
	mkdir -p "$(dirname -- "$1")"
	cat > "$1"
}

create_file_lines() {
	_cfl_filename="$1"; shift
	mkdir -p "$(dirname -- "$_cfl_filename")"
	printf '%s\n' "$@" > "$_cfl_filename"
}

create_pkgsrc_file() {
	mkdir -p "$(dirname -- "$mocked_pkgsrcdir/$1")"
	cat > "$mocked_pkgsrcdir/$1"
}

run_bmake() {
	cat <<EOF > "$tmpdir/test.subr.main.mk"
PKGSRCDIR=	$pkgsrcdir
.PATH:		$mocked_pkgsrcdir
.PATH:		$pkgsrcdir
.include "$1"
EOF
	shift

	"$make" -f "$tmpdir/test.subr.main.mk" ${1+"$@"}
}

assert_succeeded=0
assert_failed=0
assert_fail_sep=''

assert_succeed() {
	assert_succeeded=`expr "$assert_succeeded" + 1`
}

assert_fail() {
	printf '%s' "$assert_fail_sep" 1>&2
	assert_fail_sep='
'

	printf 'assertion failed in "%s": ' "$test_case_name" 1>&2

	printf "$@" 1>&2
	assert_failed=`expr "$assert_failed" + 1`
}

files_equal() {
	(diff -u -- "$@" >/dev/null) || return 1
}

diff_without_timestamps() {
	(diff -u -- "$@" || true) \
	| awk '/^(---|[+][+][+]) / { print($1, $2); next } { print }' 1>&2
}

assert_that() {
	case "$2" in
	(--equals)
		if [ "x$1" = "x$3" ]; then
			assert_succeed
			return 0
		fi
		assert_fail '\n  expected: <%s>\n  but was:  <%s>\n' "$3" "$1"
		;;

	(--file-contains-exactly)
		printf '%s\n' "$3" > "$tmpdir/expected"
		if files_equal "$tmpdir/expected" "$1"; then
			assert_succeed
			return 0
		fi
		assert_fail 'file "%s" has unexpected content:\n' "$1"
		diff_without_timestamps "$tmpdir/expected" "$1"
		;;

	(--file-equals)
		if files_equal "$3" "$1"; then
			assert_succeed
			return 0
		fi
		assert_fail 'files "%s" and "%s" differ:\n' "$1" "$3"
		diff_without_timestamps "$3" "$1"
		;;

	(--file-is-empty)
		if files_equal "/dev/null" "$1"; then
			assert_succeed
			return 0
		fi
		assert_fail 'file "%s" is not empty:\n' "$1"
		diff_without_timestamps "/dev/null" "$1"
		;;

	(--file-is-lines)
		_assert_that_tmp_actual="$1"
		_assert_that_filename="$1"; shift 2

		printf '%s\n' "$@" > "$tmpdir/expected"
		if files_equal "$tmpdir/expected" "$_assert_that_tmp_actual"; then
			assert_succeed
			return 0
		fi
		assert_fail 'file "%s" has unexpected content:\n' "$_assert_that_filename"
		diff_without_timestamps "$tmpdir/expected" "$_assert_that_tmp_actual"
		;;

	(*)
		printf 'usage: assert_that <expr> --equals <expr>\n' 1>&2
		exit 1
	esac
}