Mercurial > hg > vamp-build-and-test

changeset 6:c4ee2ab22c38

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Run individual plugin tests separately; put output in reports dir. Move scripts to subdir
author Chris Cannam
date Sat, 26 Jul 2014 20:41:13 +0100
parents 57da88814766
children 5b720d5c7bc1
files .hgignore SCRIPTS/process.sh SCRIPTS/update-all.sh nondeterministic.txt process.sh update-all.sh
diffstat 6 files changed, 165 insertions(+), 136 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Sat Jul 26 10:50:24 2014 +0100
+++ b/.hgignore	Sat Jul 26 20:41:13 2014 +0100
@@ -1,2 +1,3 @@
 syntax: glob
 *~
+REPORTS
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/SCRIPTS/process.sh	Sat Jul 26 20:41:13 2014 +0100
@@ -0,0 +1,151 @@
+#!/bin/bash
+
+# Run this from the top-level vamp-build-and-test directory
+
+## Things to test:
+##   the plugin builds!
+##   plugin loads
+##   passes vamp-plugin-tester tests
+##   does not export any unnecessary symbols
+##   has valid .cat and .n3
+
+platform=linux
+bits=64
+
+reportdir="REPORTS/$platform$bits"
+mkdir -p "$reportdir" || exit 1
+
+configure() {
+    dir="$1"
+    if [ -f "$dir/configure" ] ; then
+	( cd "$dir" ; ./configure ) 2>&1 | tee "$reportdir/$dir.configure.txt"
+    fi
+}
+
+configure_maybe() {
+    dir="$1"
+    if [ ! -f "$dir/Makefile" ] ; then
+	configure "$dir"
+    fi
+}
+
+find_makefile() {
+    dir="$1"
+    for f in Makefile Makefile.$platform Makefile.$platform$bits build/$platform/Makefile build/$platform/Makefile.$platform build/$platform/Makefile.$platform$bits; do
+	if [ -f "$dir/$f" ]; then
+	    echo $f
+	    break
+	fi
+    done
+}
+
+build() {
+    dir="$1"
+    if configure_maybe "$dir"; then
+	mfile=$(find_makefile "$dir")
+	if [ -n "$mfile" ]; then
+	    make -C "$dir" -f "$mfile" 2>&1 | tee "$reportdir/$dir.build.txt"
+	else
+	    echo "Failed to find a Makefile in $dir"
+	    return 1
+	fi
+    fi
+}
+
+rebuild() {
+    dir="$1"
+    if configure "$dir"; then
+	mfile=$(find_makefile "$dir")
+	if [ -n "$mfile" ]; then
+	    make -C "$dir" -f "$mfile" clean
+	    make -C "$dir" -f "$mfile" 2>&1 | tee "$reportdir/$dir.build.txt"
+	else
+	    echo "Failed to find a Makefile in $dir"
+	    return 1
+	fi
+    fi
+}
+
+run_tester() {
+    ##!!! todo: list the plugins in each library and run the tester
+    ##!!! separately on each -- that way we can report on specific
+    ##!!! plugins rather than whole libraries (though for the
+    ##!!! "printout" report we should include the whole library test,
+    ##!!! just for information's sake?) and we can have separate
+    ##!!! nondeterministic statuses
+    dir="$1"
+    ids=$(VAMP_PATH="$dir" vamp-plugin-sdk/host/vamp-simple-host --list-ids | sed 's/^vamp://')
+    if [ -z "$ids" ]; then
+	echo 
+	echo "No plugins reported to test in $dir"
+	return 1
+    else
+	for id in $ids; do
+	    extra=""
+	    if grep -q "^$id\$" nondeterministic.txt; then
+		extra="-n"
+	    fi
+	    if VAMP_PATH="$dir" vamp-plugin-tester/vamp-plugin-tester "$extra" "$id" 2>&1 | tee "$reportdir/$dir.test.txt" ; then
+		echo "OK"
+	    else
+		echo
+		echo "Tester failed for id $id: running again with valgrind and verbose for a report..."
+		echo "$dir" >> "$testfailed"
+		VAMP_PATH="$dir" valgrind vamp-plugin-tester/vamp-plugin-tester -v "$extra" "$id" 2>&1 | tee -a "$reportdir/$dir.test.txt"
+	    fi
+	done
+    fi
+}
+
+built="/tmp/built.$$.txt"
+testfailed="/tmp/testfailed.$$.txt"
+notbuilt="/tmp/notbuilt.$$.txt"
+trap 'rm -f "$built" "$testfailed" "$notbuilt"' 0
+
+if ! build "vamp-plugin-sdk"; then 
+    echo "Failed to build Vamp plugin SDK!"
+    exit 1
+fi
+
+if ! build "vamp-plugin-tester"; then 
+    echo "Failed to build Vamp plugin tester!"
+    exit 1
+fi
+
+for dir in $(cat .hgsub | grep -v vamp-plugin-sdk | grep -v vamp-plugin-tester | awk '{ print $1; }') ; do
+    echo
+    echo "Processing: $dir"
+    if rebuild "$dir"; then
+	echo "$dir" >> "$built"
+	run_tester "$dir"
+    else
+	echo "$dir" >> "$notbuilt"
+    fi
+done
+
+truncate -s0 "$reportdir/$dir.summary.txt"
+
+echo
+echo "** Successfully built and tested:"
+cat "$built" | while read d; do
+    if ! grep -q "^$d\$" "$testfailed"; then
+	echo "$d"
+	echo "Success" >> "$reportdir/$d.summary.txt"
+    fi
+done | sort
+
+echo
+echo "** Built, but failed tests:"
+cat "$testfailed" | sort | uniq | while read d; do
+    echo "$d"
+    echo "Built successfully, but failed tests" >> "$reportdir/$d.summary.txt"
+done
+
+echo
+echo "** Failed to build:"	    
+cat "$notbuilt" | sort | while read d; do
+    echo "$d"
+    echo "Failed to build" >> "$reportdir/$d.summary.txt"
+done
+
+echo
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/SCRIPTS/update-all.sh	Sat Jul 26 20:41:13 2014 +0100
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Run this from the top-level vamp-build-and-test directory
+
+cat .hgsub | awk '{ print $1 }' | while read x; do
+    if [ -d "$x" ]; then 
+	( cd $x ; hg pull && hg update )
+    else 
+	url=$(grep "^$x " .hgsub | awk '{ print $3; }')
+	hg clone "$url" "$x"
+    fi
+done
--- a/nondeterministic.txt	Sat Jul 26 10:50:24 2014 +0100
+++ b/nondeterministic.txt	Sat Jul 26 20:41:13 2014 +0100
@@ -1,1 +1,1 @@
-silvet
+silvet:silvet
--- a/process.sh	Sat Jul 26 10:50:24 2014 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,126 +0,0 @@
-#!/bin/bash
-
-## Things to test:
-##   the plugin builds!
-##   plugin loads
-##   passes vamp-plugin-tester tests
-##   does not export any unnecessary symbols
-##   has valid .cat and .n3
-
-platform=linux
-bits=64
-
-configure() {
-    dir="$1"
-    if [ -f "$dir/configure" ] ; then
-	( cd "$dir" ; ./configure )
-    fi
-}
-
-configure_maybe() {
-    dir="$1"
-    if [ ! -f "$dir/Makefile" ] ; then
-	configure "$dir"
-    fi
-}
-
-find_makefile() {
-    dir="$1"
-    for f in Makefile Makefile.$platform Makefile.$platform$bits build/$platform/Makefile build/$platform/Makefile.$platform build/$platform/Makefile.$platform$bits; do
-	if [ -f "$dir/$f" ]; then
-	    echo $f
-	    break
-	fi
-    done
-}
-
-build() {
-    dir="$1"
-    if configure_maybe "$dir"; then
-	mfile=$(find_makefile "$dir")
-	if [ -n "$mfile" ]; then
-	    make -C "$dir" -f "$mfile"
-	else
-	    echo "Failed to find a Makefile in $dir"
-	    return 1
-	fi
-    fi
-}
-
-rebuild() {
-    dir="$1"
-    if configure "$dir"; then
-	mfile=$(find_makefile "$dir")
-	if [ -n "$mfile" ]; then
-	    make -C "$dir" -f "$mfile" clean
-	    make -C "$dir" -f "$mfile"
-	else
-	    echo "Failed to find a Makefile in $dir"
-	    return 1
-	fi
-    fi
-}
-
-run_tester() {
-    ##!!! todo: list the plugins in each library and run the tester
-    ##!!! separately on each -- that way we can report on specific
-    ##!!! plugins rather than whole libraries (though for the
-    ##!!! "printout" report we should include the whole library test,
-    ##!!! just for information's sake?) and we can have separate
-    ##!!! nondeterministic statuses
-    dir="$1"
-    extra=""
-    if grep -q "^$dir\$" nondeterministic.txt; then
-	extra="-n"
-    fi
-    if VAMP_PATH="$dir" vamp-plugin-tester/vamp-plugin-tester "$extra" -a ; then
-	echo "OK"
-    else
-	echo
-	echo "Tester failed: running again with valgrind and verbose for a report..."
-	VAMP_PATH="$dir" valgrind vamp-plugin-tester/vamp-plugin-tester -v "$extra" -a
-    fi
-}
-
-successes="/tmp/successes.$$.txt"
-partials="/tmp/successes.$$.txt"
-failures="/tmp/failures.$$.txt"
-trap 'rm -f "$successes" "$partials" "$failures"' 0
-
-if ! build "vamp-plugin-sdk"; then 
-    echo "Failed to build Vamp plugin SDK!"
-    exit 1
-fi
-
-if ! build "vamp-plugin-tester"; then 
-    echo "Failed to build Vamp plugin tester!"
-    exit 1
-fi
-
-for dir in $(cat .hgsub | grep -v vamp-plugin-sdk | grep -v vamp-plugin-tester | awk '{ print $1; }') ; do
-    echo
-    echo "Processing: $dir"
-    if build "$dir"; then
-	if run_tester "$dir"; then
-	    echo "$dir" >> "$successes"
-	else
-	    echo "$dir" >> "$partials"
-	fi
-    else
-	echo "$dir" >> "$failures"
-    fi
-done
-
-echo
-echo "** Successfully built and tested:"
-cat "$successes"
-
-echo
-echo "** Built, but failed tests:"
-cat "$partials"
-
-echo
-echo "** Failed to build:"	    
-cat "$failures"
-
-echo
--- a/update-all.sh	Sat Jul 26 10:50:24 2014 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,9 +0,0 @@
-#!/bin/bash
-cat .hgsub | awk '{ print $1 }' | while read x; do
-    if [ -d "$x" ]; then 
-	( cd $x ; hg pull && hg update )
-    else 
-	url=$(grep "^$x " .hgsub | awk '{ print $3; }')
-	hg clone "$url" "$x"
-    fi
-done