annotate SCRIPTS/process.sh @ 47:fe753ff3d0b5

Simpler solution to .cat test

#!/bin/bash

# Run this from the top-level vamp-build-and-test directory

## Things to test:
##   the plugin builds!
##   plugin loads
##   passes vamp-plugin-tester tests
##   does not export any unnecessary symbols
##   has valid .cat and .n3

mydir=$(dirname "$0")
case "$mydir" in /*);; *) mydir=$(pwd)/"$mydir";; esac

do_rebuild=""

usage() {
    echo 
    echo "Usage: $0 <platform> [-c] [<dir> ...]"
    echo 
    echo " <platform>  one of native, linux32, linux64, mingw32, mingw64, osx32, osx64"
    echo " -c          build from clean"
    echo " <dir>       directory to build (default is all of them)"
    echo
    echo "Platform usually should match the platform you are running this"
    echo "script on, unless you have a cross-compile toolset installed and"
    echo "this script knows how to run it. The special platform 'native'"
    echo "tries to guess the currently running platform."
    echo
    exit 2
}

platform_arg="$1"

if [ "$platform_arg" = "native" ]; then
    case `uname -a` in
	Linux*x86_64*) platform_arg=linux64;;
	Linux*) platform_arg=linux32;;
	Darwin*) platform_arg=osx64;;
	CYG*) platform_arg=mingw32;;
	MINGW*) platform_arg=mingw32;;
    esac
fi

case "$platform_arg" in
    linux32)
	platform=linux
	bits=32
	toolprefix=
	pluginext=.so
	hostwrapper=
	hostext=
	valgrind=valgrind
	archflags=
	identpattern='ELF 32'
	;;
    linux64)
	platform=linux
	bits=64
	toolprefix=
	pluginext=.so
	hostwrapper=
	hostext=
	valgrind=valgrind
	archflags=
	identpattern='ELF 64'
	;;
    mingw32)
	platform=mingw
	bits=32
	toolprefix=i686-w64-mingw32- 
	pluginext=.dll
	hostwrapper=wine
	hostext=.exe
	valgrind=
	archflags=
	identpattern='PE32.*386.*Windows'
	;;
    mingw64)
	platform=mingw
	bits=64
	toolprefix=x86_64-w64-mingw32- 
	pluginext=.dll
	hostwrapper=wine
	hostext=.exe
	valgrind=
	archflags=
	identpattern='not known yet' ##!!!
	;;
    osx32)
	platform=osx
	bits=32
	toolprefix=
	pluginext=.dylib
	hostwrapper=
	hostext=
	valgrind=
	archflags="-arch i386"
	identpattern='Mach-O .*i386'
	;;
    osx64)
	platform=osx
	bits=64
	toolprefix=
	pluginext=.dylib
	hostwrapper=
	hostext=
	valgrind=
	# This is a difficult choice for various reasons... have to ponder
	archflags="-mmacosx-version-min=10.6 -arch x86_64 -arch i386"
	identpattern='Mach-O 64-bit .*x86_64'
	;;
esac;

shift

if [ -z "$platform" ]; then
    usage
else
    echo "(Platform is $platform, $bits bits)"
fi

if [ t"$1" = t"-c" ]; then
    echo "(Building from clean)"
    do_rebuild=yes
    shift
fi

depincdir="$mydir"/../DEPENDENCIES/$platform$bits/include
deplibdir="$mydir"/../DEPENDENCIES/$platform$bits/lib

depincdir_generic="$mydir"/../DEPENDENCIES/generic/include

plugindirs="$@"
if [ -z "$plugindirs" ]; then
    plugindirs=$(cat .hgsub | grep -v vamp-plugin-sdk | grep -v vamp-plugin-tester | awk '{ print $1; }')
fi

reportdir="REPORTS/$platform$bits"
mkdir -p "$reportdir" || exit 1

built="/tmp/built.$$.txt"
testfailed="/tmp/testfailed.$$.txt"
envcheckfailed="/tmp/envcheckfailed.$$.txt"
notbuilt="/tmp/notbuilt.$$.txt"
trap 'rm -f "$built" "$envcheckfailed" "$testfailed" "$notbuilt"' 0
touch "$built" "$envcheckfailed" "$testfailed" "$notbuilt"

configure() {
    dir="$1"
    if [ -f "$dir/configure" ] ; then
	( cd "$dir" ; ./configure ) 2>&1 | tee "$reportdir/$dir.configure.txt"
    fi
}

find_makefile() {
    dir="$1"
    for f in \
	build/$platform$bits/Makefile.$platform$bits \
	build/$platform/Makefile.$platform$bits \
	build/$platform$bits/Makefile.$platform \
	build/$platform$bits/Makefile \
	build/Makefile.$platform$bits \
	Makefile.$platform$bits \
	build/$platform/Makefile.$platform \
	build/$platform/Makefile \
	build/Makefile.$platform \
	Makefile.$platform \
	Makefile ; do
	if [ -f "$dir/$f" ]; then
	    echo $f
	    break
	fi
    done
}

configure_maybe() {
    dir="$1"
    if [ t"$do_rebuild" = t"yes" ]; then
        configure "$dir"
    else
        mfile=$(find_makefile "$dir")
        if [ -z "$mfile" ]; then
	    configure "$dir"
        fi
    fi
}

target_for() {
    dir="$1"
    if grep -q "^$dir: " METADATA/maketarget.txt ; then
	grep "^$dir: " METADATA/maketarget.txt | head -1 | sed 's/^[^:]*: //'
    fi
}

logfile_for() {
    activity="$1"
    dir="$2"
    echo "$reportdir/$dir.$activity.txt"
}

build() {
    dir="$1"
    log=$(logfile_for build "$dir")
    if configure_maybe "$dir"; then
	mfile=$(find_makefile "$dir")
	if [ -n "$mfile" ]; then
	    target=$(target_for "$dir")
	    TOOLPREFIX="$toolprefix" \
		CXXFLAGS="-I${depincdir} -I${depincdir_generic} -I../vamp-plugin-sdk" \
		LDFLAGS="-L${deplibdir} -L../vamp-plugin-sdk" \
		ARCHFLAGS="$archflags" \
		make -C "$dir" -f "$mfile" $target 2>&1 | \
		tee "$log"
	    return ${PIPESTATUS[0]}
	else
	    echo "Failed to find a Makefile in $dir!" | tee "$log"
	    return 1
	fi
    fi
}

rebuild() {
    dir="$1"
    log=$(logfile_for build "$dir")
    if configure_maybe "$dir"; then
	mfile=$(find_makefile "$dir")
	if [ -n "$mfile" ]; then
	    if make -C "$dir" -f "$mfile" clean; then
		build "$dir"
	    else
		echo "Failed to 'make clean' in $dir!" | tee "$log"
		return 1
	    fi
	else
	    echo "Failed to find a Makefile in $dir!" | tee "$log"
	    return 1
	fi
    fi
}

build_or_rebuild() {
    dir="$1"
    if [ -n "$do_rebuild" ]; then
	rebuild "$dir"
    else
	build "$dir"
    fi
}

have_plugin() {
    dir="$1"
    log=$(logfile_for build "$dir")
    for x in "$dir/"*"$pluginext"; do 
	if [ -f "$x" ]; then
	    if file "$x" | grep -q "$identpattern" ; then
		return 0
	    else
		echo "Plugin $x exists, but fails to match file type for correct platform (file type is: `file $x`, expected pattern is: $identpattern)" | tee "$log"
		return 1
	    fi
	fi
    done
    return 1
}

is_nondeterministic() {
    plugin_id="$1"
    grep -q "^$id\$" METADATA/nondeterministic.txt
}

plugin_ids_in() {
    dir="$1"
    # can't use sed to remove \r from DOS line endings -- BSD and GNU
    # vary in how they interpret \r escape -- so we use perl for that...
    VAMP_PATH="$dir" $hostwrapper \
	vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids | \
	grep '^vamp:' | \
	sed 's/^vamp://' | \
	perl -p -e 's/\r//g' 
}

run_tester() {
    ##!!! todo: timeout if the plugin takes too long and report as failure
    dir="$1"
    log=$(logfile_for test "$dir")
    ids=$(plugin_ids_in "$dir")
    cat /dev/null > "$log"
    if [ -z "$ids" ]; then
	echo 
	echo "No plugins reported to test in $dir" | tee -a "$log"
	echo "$dir" >> "$testfailed"
	return 1
    fi
    good=yes
    for id in $ids; do
	extra=""
	if is_nondeterministic "$id"; then
	    extra="-n"
	fi
	echo "Running command: VAMP_PATH=\"$dir\" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext \"$extra\" \"$id\"" | tee -a "$log"
	if ( VAMP_PATH="$dir" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext "$extra" "$id" 2>&1 | tee -a "$log" ; exit ${PIPESTATUS[0]} ) ; then
	    echo "OK"
	else
	    echo
	    echo "Tester failed for id $id: running again with valgrind (if available) and verbose for a report..." | tee -a "$log"
	    VAMP_PATH="$dir" $valgrind $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext -v "$extra" "$id" 2>&1 | tee -a "$log"
	    good=no
	fi
    done
    if [ "$good" != "yes" ]; then
	echo "$dir" >> "$testfailed"
	return 1
    else
	return 0
    fi
}

public_symbols_in() {
    lib="$1"
    # nm -g prints global symbols in both OS/X and GNU tools, but
    # printing only global *defined* symbols is harder. In GNU it is
    # nm -g --defined-only; the OS/X docs suggest nm -gu should work,
    # but it doesn't. What I think will work with both is simply
    # grepping out the undefineds:
    "$toolprefix"nm -g "$lib" | grep -v ' U ' | awk '{ print $3; }'
}

env_test_exports() {
    dir="$1"
    log=$(logfile_for envtest "$dir")
    good=yes
    for lib in "$dir"/*"$pluginext"; do 
	if [ ! -f "$lib" ]; then
	    # This should only happen if the glob was not expanded at all
	    echo "NOTE: no library found in $dir?" | tee -a "$log"
	    good=no
	    break
	fi
	echo 
	echo "Testing for exported symbols in $lib..."
	if public_symbols_in "$lib" | grep -q vampGetPluginDescriptor; then
	    others=`public_symbols_in "$lib" | grep -v vampGetPluginDescriptor`
	    if [ -n "$others" ]; then
		count=`echo "$others" | wc -l`
		echo "ERROR: library $lib exports $count extra symbols in addition to vampGetPluginDescriptor" | tee -a "$log"
		good=no
	    else
		echo "GOOD: library $lib only exports vampGetPluginDescriptor" | tee -a "$log"
	    fi
	else
	    echo "NOTE: found library $lib that is not a Vamp plugin library" | tee -a "$log"
	fi
    done
    [ "$good" = "yes" ]
}

env_test_stdout() {
    dir="$1"
    log=$(logfile_for envtest "$dir")
    good=yes
    ids=$(VAMP_PATH="$dir" $hostwrapper vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids);
    echo 
    echo "Testing for any unwanted output to stdout..."
    for id in $ids; do
	case "$id" in
	    vamp:*) ;;
	    *)
		echo "ERROR: plugin in $dir prints to stdout as it runs: found text $id (should use stderr for debug logging to avoid mixing with batch output stream)" | tee -a "$log"
		good=no
		;;
	esac
    done
    [ "$good" = "yes" ]
}

env_test_cat() {
    dir="$1"
    log=$(logfile_for envtest "$dir")
    good=yes
    first=yes
    echo 
    echo "Testing some details of .cat files..."
    for catfile in "$dir"/*".cat"; do
	if [ ! -f "$catfile" ]; then
	    # This should only happen if the glob was not expanded at all
	    echo "ERROR: no .cat file found in $dir" | tee -a "$log"
	    good=no
	    break
	fi
	if [ "$first" = "yes" ]; then
	    first=no
	else
	    echo "NOTE: multiple .cat files found in $dir" | tee -a "$log"
	fi
    done
    if [ "$good" = "yes" ]; then
	excess=$(plugin_ids_in "$dir" | sed 's/^/vamp:/' | \
	    cat "$dir"/*".cat" - | \
	    sed 's/::.*//' | \
            sort | \
            uniq -u)
	if [ -n "$excess" ]; then
	    echo "ERROR: excess or missing definitions in .cat file? $excess" | tee -a "$log"
	    good=no
	else
	    echo "GOOD: no excess or missing definitions in .cat files" | tee -a "$log"
	fi
    fi
    [ "$good" = "yes" ]
}

env_test_ttl() {
    dir="$1"
    log=$(logfile_for envtest "$dir")
    good=yes
    first=yes
    echo 
    echo "Testing existence of RDF files..."
    for ttlfile in "$dir"/*.{n3,ttl}; do
	if [ ! -f "$ttlfile" ]; then
	    # Because we have two different extensions, this isn't an
	    # error as it is with e.g. .cat (because one or the other
	    # of .n3 and .ttl almost certainly won't exist). But if we
	    # drop out the bottom and first is still true, then we
	    # know neither matched.
	    :
	elif [ "$first" = "yes" ]; then
	    first=no
	else
	    echo "NOTE: multiple .ttl or .n3 files found in $dir" | tee -a "$log"
	fi
    done
    if [ "$first" = "yes" ]; then 
	good=no; 
    else
	echo "GOOD: found one or more .ttl or .n3 files (we don't actually test their validity or content here though)" | tee -a "$log"
    fi
    [ "$good" = "yes" ]
}

env_test_accompaniments() {
    dir="$1"
    log=$(logfile_for envtest "$dir")
    good=yes
    echo 
    echo "Testing existence of README and accompanying files..."
    if ! ls -1 "$dir" | egrep -qi "^readme(.txt|)$"; then
	echo "ERROR: no README file found" | tee -a "$log"
	good=no
    fi
    if ! ls -1 "$dir" | egrep -qi "^(copying|licen[cs]e)(.txt|)$"; then
	echo "ERROR: no COPYING or LICEN[CS]E file found" | tee -a "$log"
	good=no
    fi
    if ! ls -1 "$dir" | egrep -qi "^citation(.txt|)$"; then
	echo "NOTE: no CITATION file found" | tee -a "$log"
    fi
    [ "$good" = "yes" ]
}

run_environmental_tests() {
    dir="$1"
    log=$(logfile_for envtest "$dir")
    allgood=yes
    cat /dev/null > "$log"
    for test in exports stdout cat ttl accompaniments; do
	"env_test_$test" "$dir" || allgood=no
    done
    if [ "$allgood" != "yes" ]; then
	echo "$dir" >> "$envcheckfailed"
	return 1
    else
	return 0
    fi
}

if ! build_or_rebuild "vamp-plugin-sdk"; then 
    echo "Failed to build Vamp plugin SDK!"
    exit 1
fi

# Ensure we can only link statically against these
for x in vamp-hostsdk vamp-sdk; do 
    for y in dylib dll so; do
	rm -f "vamp-plugin-sdk/lib$x.$y"
	rm -f "vamp-plugin-sdk/$x.$y"
    done
done

if ! build_or_rebuild "vamp-plugin-tester"; then 
    echo "Failed to build Vamp plugin tester!"
    exit 1
fi

for dir in $plugindirs ; do
    dir=${dir%/*}
    echo
    echo "Processing: $dir"
    if [ ! -d "$dir" ]; then
	echo "Directory $dir not found!"
	echo "$dir" >> "$notbuilt"
    elif build_or_rebuild "$dir"; then
	if have_plugin "$dir" ; then
	    echo "$dir" >> "$built"
	    run_tester "$dir"
	    run_environmental_tests "$dir"
	else 
	    log=$(logfile_for build "$dir")
	    echo "Build apparently succeeded, but no resulting plugin(s) found, or plugin(s) have wrong file type or platform" | tee -a "$log"
	    echo "$dir" >> "$notbuilt"
	fi
    else
	echo "$dir" >> "$notbuilt"
    fi
    slog=$(logfile_for summary "$dir")
    cat /dev/null > "$slog"
done

echo
echo "** Successfully built, tested, and checked:"
cat "$built" | while read dir; do
    slog=$(logfile_for summary "$dir")
    if ! grep -q "^$dir\$" "$testfailed"; then
	if ! grep -q "^$dir\$" "$envcheckfailed"; then
	    echo "$dir"
	    echo "$dir: Success" >> "$slog"
	fi
    fi
done | sort

echo
echo "** Failed tests:"
cat "$testfailed" | sort | uniq | while read dir; do
    slog=$(logfile_for summary "$dir")
    echo "$dir"
    echo "$dir: Built successfully, but failed tests" >> "$slog"
done

echo
echo "** Failed environmental checks:"
cat "$envcheckfailed" | sort | uniq | while read dir; do
    slog=$(logfile_for summary "$dir")
    echo "$dir"
    echo "$dir: Built successfully, but failed environmental checks" >> "$slog"
done

echo
echo "** Failed to build:"	    
cat "$notbuilt" | sort | while read dir; do
    slog=$(logfile_for summary "$dir")
    echo "$dir"
    echo "$dir: Failed to build" >> "$slog"
done

echo