# HG changeset patch
# User Chris Cannam
# Date 1407415033 -3600
# Node ID 0036098a5edbf4d2dd981e1de22ecea0fee1287f
# Parent  5638274646286987b3ee840a03ce2233365fa5f0
Add further environmental tests; factor out logfile_for

diff -r 563827464628 -r 0036098a5edb SCRIPTS/process.sh
--- a/SCRIPTS/process.sh	Thu Aug 07 13:01:57 2014 +0100
+++ b/SCRIPTS/process.sh	Thu Aug 07 13:37:13 2014 +0100
@@ -193,8 +193,15 @@
     fi
 }
 
+logfile_for() {
+    activity="$1"
+    dir="$2"
+    echo "$reportdir/$dir.$activity.txt"
+}
+
 build() {
     dir="$1"
+    log=$(logfile_for build "$dir")
     if configure_maybe "$dir"; then
 	mfile=$(find_makefile "$dir")
 	if [ -n "$mfile" ]; then
@@ -204,10 +211,10 @@
 		LDFLAGS="-L${deplibdir} -L../vamp-plugin-sdk" \
 		ARCHFLAGS="$archflags" \
 		make -C "$dir" -f "$mfile" $target 2>&1 | \
-		tee "$reportdir/$dir.build.txt"
+		tee "$log"
 	    return ${PIPESTATUS[0]}
 	else
-	    echo "Failed to find a Makefile in $dir"
+	    echo "Failed to find a Makefile in $dir!" | tee "$log"
 	    return 1
 	fi
     fi
@@ -215,17 +222,18 @@
 
 rebuild() {
     dir="$1"
+    log=$(logfile_for build "$dir")
     if configure_maybe "$dir"; then
 	mfile=$(find_makefile "$dir")
 	if [ -n "$mfile" ]; then
 	    if make -C "$dir" -f "$mfile" clean; then
 		build "$dir"
 	    else
-		echo "Failed to 'make clean' in $dir!" | tee "$reportdir/$dir.build.txt"
+		echo "Failed to 'make clean' in $dir!" | tee "$log"
 		return 1
 	    fi
 	else
-	    echo "Failed to find a Makefile in $dir!" | tee "$reportdir/$dir.build.txt"
+	    echo "Failed to find a Makefile in $dir!" | tee "$log"
 	    return 1
 	fi
     fi
@@ -242,13 +250,13 @@
 
 have_plugin() {
     dir="$1"
+    log=$(logfile_for build "$dir")
     for x in "$dir/"*"$pluginext"; do 
 	if [ -f "$x" ]; then
 	    if file "$x" | grep -q "$identpattern" ; then
 		return 0
 	    else
-		echo "Plugin $x exists, but fails to match file type for correct platform"
-		echo "(file type is: `file $x`)"
+		echo "Plugin $x exists, but fails to match file type for correct platform (file type is: `file $x`, expected pattern is: $identpattern)" | tee "$log"
 		return 1
 	    fi
 	fi
@@ -261,44 +269,54 @@
     grep -q "^$id\$" METADATA/nondeterministic.txt
 }
 
-run_tester() {
-    ##!!! todo: timeout if the plugin takes too long and report as failure?
+plugin_ids_in() {
     dir="$1"
     # can't use sed to remove \r from DOS line endings -- BSD and GNU
-    # vary in how they interpret \r escape
-    ids=$(VAMP_PATH="$dir" $hostwrapper vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids | sed 's/^vamp://' | perl -p -e 's/\r//g' )
-    cat /dev/null > "$reportdir/$dir.test.txt"
+    # vary in how they interpret \r escape -- so we use perl for that...
+    VAMP_PATH="$dir" $hostwrapper \
+	vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids | \
+	grep '^vamp:' | \
+	sed 's/^vamp://' | \
+	perl -p -e 's/\r//g' 
+}
+
+run_tester() {
+    ##!!! todo: timeout if the plugin takes too long and report as failure
+    dir="$1"
+    log=$(logfile_for test "$dir")
+    ids=$(plugin_ids_in "$dir")
+    cat /dev/null > "$log"
     if [ -z "$ids" ]; then
 	echo 
-	echo "No plugins reported to test in $dir" | tee -a "$reportdir/$dir.test.txt"
+	echo "No plugins reported to test in $dir" | tee -a "$log"
+	echo "$dir" >> "$testfailed"
+	return 1
+    fi
+    good=yes
+    for id in $ids; do
+	extra=""
+	if is_nondeterministic "$id"; then
+	    extra="-n"
+	fi
+	echo "Running command: VAMP_PATH=\"$dir\" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext \"$extra\" \"$id\"" | tee -a "$log"
+	if ( VAMP_PATH="$dir" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext "$extra" "$id" 2>&1 | tee -a "$log" ; exit ${PIPESTATUS[0]} ) ; then
+	    echo "OK"
+	else
+	    echo
+	    echo "Tester failed for id $id: running again with valgrind (if available) and verbose for a report..." | tee -a "$log"
+	    VAMP_PATH="$dir" $valgrind $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext -v "$extra" "$id" 2>&1 | tee -a "$log"
+	    good=no
+	fi
+    done
+    if [ "$good" != "yes" ]; then
 	echo "$dir" >> "$testfailed"
 	return 1
     else
-	good=yes
-	for id in $ids; do
-	    extra=""
-	    if is_nondeterministic "$id"; then
-		extra="-n"
-	    fi
-	    echo "Running command: VAMP_PATH=\"$dir\" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext \"$extra\" \"$id\"" | tee -a "$reportdir/$dir.test.txt"
-	    if ( VAMP_PATH="$dir" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext "$extra" "$id" 2>&1 | tee -a "$reportdir/$dir.test.txt" ; exit ${PIPESTATUS[0]} ) ; then
-		echo "OK"
-	    else
-		echo
-		echo "Tester failed for id $id: running again with valgrind (if available) and verbose for a report..."
-		echo "$dir" >> "$testfailed"
-		VAMP_PATH="$dir" $valgrind $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext -v "$extra" "$id" 2>&1 | tee -a "$reportdir/$dir.test.txt"
-		good=no
-	    fi
-	done
-	if [ "$good" != "yes" ]; then
-	     return 1
-	fi
+	return 0
     fi
-    return 0
 }
 
-public_symbols() {
+public_symbols_in() {
     lib="$1"
     # nm -g prints global symbols in both OS/X and GNU tools, but
     # printing only global *defined* symbols is harder. In GNU it is
@@ -308,33 +326,135 @@
     "$toolprefix"nm -g "$lib" | grep -v ' U ' | awk '{ print $3; }'
 }
 
+env_test_exports() {
+    dir="$1"
+    log=$(logfile_for envtest "$dir")
+    good=yes
+    for lib in "$dir"/*"$pluginext"; do 
+	if [ ! -f "$lib" ]; then
+	    # This should only happen if the glob was not expanded at all
+	    echo "NOTE: no library found in $dir?" | tee -a "$log"
+	    good=no
+	    break
+	fi
+	echo 
+	echo "Testing for exported symbols in $lib..."
+	if public_symbols_in "$lib" | grep -q vampGetPluginDescriptor; then
+	    others=`public_symbols_in "$lib" | grep -v vampGetPluginDescriptor`
+	    if [ -n "$others" ]; then
+		count=`echo "$others" | wc -l`
+		echo "ERROR: library $lib exports $count extra symbols in addition to vampGetPluginDescriptor" | tee -a "$log"
+		good=no
+	    else
+		echo "GOOD: library $lib only exports vampGetPluginDescriptor" | tee -a "$log"
+	    fi
+	else
+	    echo "NOTE: found library $lib that is not a Vamp plugin library" | tee -a "$log"
+	fi
+    done
+    [ "$good" = "yes" ]
+}
+
+env_test_stdout() {
+    dir="$1"
+    log=$(logfile_for envtest "$dir")
+    good=yes
+    ids=$(VAMP_PATH="$dir" $hostwrapper vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids);
+    echo 
+    echo "Testing for any unwanted output to stdout..."
+    for id in $ids; do
+	case "$id" in
+	    vamp:*) ;;
+	    *)
+		echo "ERROR: plugin in $dir prints to stdout as it runs: found text $id (should use stderr for debug logging to avoid mixing with batch output stream)" | tee -a "$log"
+		good=no
+		;;
+	esac
+    done
+    [ "$good" = "yes" ]
+}
+
+env_test_cat() {
+    dir="$1"
+    log=$(logfile_for envtest "$dir")
+    good=yes
+    first=yes
+    echo 
+    echo "Testing some details of .cat files..."
+    for catfile in "$dir"/*".cat"; do
+	if [ ! -f "$catfile" ]; then
+	    # This should only happen if the glob was not expanded at all
+	    echo "ERROR: no .cat file found in $dir" | tee -a "$log"
+	    good=no
+	    break
+	fi
+	if [ "$first" = "yes" ]; then
+	    first=no
+	else
+	    echo "NOTE: multiple .cat files found in $dir" | tee -a "$log"
+	fi
+    done
+    if [ "$good" = "yes" ]; then
+	excess=$(VAMP_PATH="$dir" $hostwrapper \
+	    vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids | \
+	    grep '^vamp:' | \
+	    cat "$dir"/*".cat" - | \
+	    sort | \
+	    perl -e 'while (<>) { 
+                         if (/::/) {
+                             print if (!defined $plugid or !/^$plugid/);
+                         } else {
+                             chomp; $plugid = $_; 
+                         }
+                     }')
+	if [ -n "$excess" ]; then
+	    echo "ERROR: excess definitions in .cat file? $excess" | tee -a "$log"
+	    good=no
+	else
+	    echo "GOOD: no excess definitions in .cat files" | tee -a "$log"
+	fi
+    fi
+    [ "$good" = "yes" ]
+}
+
+env_test_ttl() {
+    dir="$1"
+    log=$(logfile_for envtest "$dir")
+    good=yes
+    first=yes
+    echo 
+    echo "Testing existence of RDF files..."
+    for ttlfile in "$dir"/*.{n3,ttl}; do
+	if [ ! -f "$ttlfile" ]; then
+	    # Because we have two different extensions, this isn't an
+	    # error as it is with e.g. .cat (because one or the other
+	    # of .n3 and .ttl almost certainly won't exist). But if we
+	    # drop out the bottom and first is still true, then we
+	    # know neither matched.
+	    :
+	elif [ "$first" = "yes" ]; then
+	    first=no
+	else
+	    echo "NOTE: multiple .ttl or .n3 files found in $dir" | tee -a "$log"
+	fi
+    done
+    if [ "$first" = "yes" ]; then good=no; fi
+    [ "$good" = "yes" ]
+}
+
 run_environmental_tests() {
     dir="$1"
-    good=yes
-    cat /dev/null > "$reportdir/$dir.envtest.txt"
-    for lib in $dir/*$pluginext; do 
-	if [ ! -f "$lib" ]; then
-	    echo "NOTE: no library found in $dir?" | tee -a "$reportdir/$dir.envtest.txt"
-	    good=no
-	else
-	    echo 
-	    echo "Testing for exported symbols in $lib..."
-	    if public_symbols "$lib" | grep -q vampGetPluginDescriptor; then
-		others=`public_symbols "$lib" | grep -v vampGetPluginDescriptor`
-		if [ -n "$others" ]; then
-		    count=`echo "$others" | wc -l`
-		    echo "WARNING: $count extra symbols exported by plugin library" | tee -a "$reportdir/$dir.envtest.txt"
-		    good=no
-		else
-		    echo "GOOD: library $lib only exports vampGetPluginDescriptor" | tee -a "$reportdir/$dir.envtest.txt"
-		fi
-	    else
-		echo "NOTE: found library $lib that is not a Vamp plugin library" | tee -a "$reportdir/$dir.envtest.txt"
-	    fi
-	fi
+    log=$(logfile_for envtest "$dir")
+    allgood=yes
+    cat /dev/null > "$log"
+    for test in exports stdout cat ttl; do
+	"env_test_$test" "$dir" || allgood=no
     done
-    if [ "$good" != "yes" ]; then
+    if [ "$allgood" != "yes" ]; then
 	echo "$dir" >> "$envcheckfailed"
+	return 1
+    else
+	return 0
     fi
 }
 
@@ -369,45 +489,51 @@
 	    run_tester "$dir"
 	    run_environmental_tests "$dir"
 	else 
-	    echo "Build apparently succeeded, but no resulting plugin(s) found, or plugin(s) have wrong file type or platform" | tee -a "$reportdir/$dir.build.txt"
+	    log=$(logfile_for build "$dir")
+	    echo "Build apparently succeeded, but no resulting plugin(s) found, or plugin(s) have wrong file type or platform" | tee -a "$log"
 	    echo "$dir" >> "$notbuilt"
 	fi
     else
 	echo "$dir" >> "$notbuilt"
     fi
-    cat /dev/null > "$reportdir/$dir.summary.txt"
+    slog=$(logfile_for summary "$dir")
+    cat /dev/null > "$slog"
 done
 
 echo
 echo "** Successfully built, tested, and checked:"
-cat "$built" | while read d; do
-    if ! grep -q "^$d\$" "$testfailed"; then
-	if ! grep -q "^$d\$" "$envcheckfailed"; then
-	    echo "$d"
-	    echo "$d: Success" >> "$reportdir/$d.summary.txt"
+cat "$built" | while read dir; do
+    slog=$(logfile_for summary "$dir")
+    if ! grep -q "^$dir\$" "$testfailed"; then
+	if ! grep -q "^$dir\$" "$envcheckfailed"; then
+	    echo "$dir"
+	    echo "$dir: Success" >> "$slog"
 	fi
     fi
 done | sort
 
 echo
 echo "** Failed tests:"
-cat "$testfailed" | sort | uniq | while read d; do
-    echo "$d"
-    echo "$d: Built successfully, but failed tests" >> "$reportdir/$d.summary.txt"
+cat "$testfailed" | sort | uniq | while read dir; do
+    slog=$(logfile_for summary "$dir")
+    echo "$dir"
+    echo "$dir: Built successfully, but failed tests" >> "$slog"
 done
 
 echo
 echo "** Failed environmental checks:"
-cat "$envcheckfailed" | sort | uniq | while read d; do
-    echo "$d"
-    echo "$d: Built successfully, but failed environmental checks" >> "$reportdir/$d.summary.txt"
+cat "$envcheckfailed" | sort | uniq | while read dir; do
+    slog=$(logfile_for summary "$dir")
+    echo "$dir"
+    echo "$dir: Built successfully, but failed environmental checks" >> "$slog"
 done
 
 echo
 echo "** Failed to build:"	    
-cat "$notbuilt" | sort | while read d; do
-    echo "$d"
-    echo "$d: Failed to build" >> "$reportdir/$d.summary.txt"
+cat "$notbuilt" | sort | while read dir; do
+    slog=$(logfile_for summary "$dir")
+    echo "$dir"
+    echo "$dir: Failed to build" >> "$slog"
 done
 
 echo