annotate SCRIPTS/process.sh @ 120:4729c8589274 emscripten-piper

Add piper builds

#!/bin/bash

# Run this from the top-level vamp-build-and-test directory

## Things to test:
##   the plugin builds!
##   plugin loads
##   passes vamp-plugin-tester tests
##   does not export any unnecessary symbols
##   has valid .cat and .n3

set -e

mydir=$(dirname "$0")
case "$mydir" in /*);; *) mydir=$(pwd)/"$mydir";; esac

. "$mydir"/include.sh

have_all=true
for program in make perl git svn hg zip tar ; do
    if ! $program --version >/dev/null 2>&1; then
	if ! $program -v 2>&1 | grep -q version; then
	    echo " ** $program program not found"
	    have_all=false
	fi
    fi
done
if [ "$have_all" != "true" ]; then
    echo "Not all dependencies found, exiting"
    exit 1
fi

do_rebuild=""

usage() {
    echo 
    echo "Usage: $0 <platform> [-c] [<dir> ...]"
    echo 
    echo " <platform> one of:"
    echo "            native, linux32, linux64, mingw32, mingw64, osx32, osx64, piper"
    echo " -c         build from clean"
    echo " <dir>      directory to build (default is all of them)"
    echo
    echo "Platform usually should match the platform you are running this"
    echo "script on, unless you have a cross-compile toolset installed and"
    echo "this script knows how to run it. The 'piper' platform builds a"
    echo "portable Piper API Javascript module using Emscripten. The special"
    echo "platform 'native' tries to guess the currently running platform."
    echo
    exit 2
}

platform_arg="$1"
platform_native="unknown"
case `uname -a` in
    Linux*x86_64*) platform_native=linux64;;
    Linux*) platform_native=linux32;;
    Darwin*) platform_native=osx64;;
    CYG*) platform_native=mingw32;;
    MINGW*) platform_native=mingw32;;
esac

if [ "$platform_arg" = "native" ]; then
    platform_arg="$platform_native"
fi

platform=
platform_defines=("UNUSED_=")   # Avoid passing empty arg to make as if target
pubtag=
bits=32
altbits=
toolprefix=
pluginext=
hostwrapper=
hostext=
valgrind=
archflags=
identpattern=
sep=":"

case "$platform_arg" in
    linux32)
	platform=linux
	pubtag=linux32
	if [ "$platform_native" = "linux64" ]; then
	    toolprefix=x86_64-unknown-linux-gnu-
	    platform_defines=("CXX=${toolprefix}g++ -m32" "CC=${toolprefix}gcc -m32" "LD=${toolprefix}g++ -m32")
	fi
	pluginext=.so
	valgrind=valgrind
	identpattern='ELF 32'
	;;
    linux64)
	platform=linux
	bits=64
	pubtag=linux64
	pluginext=.so
	valgrind=valgrind
	identpattern='ELF 64'
	;;
    mingw32)
	platform=mingw
	pubtag=win32
	toolprefix=i686-w64-mingw32- 
	pluginext=.dll
	hostwrapper=wine
	hostext=.exe
	identpattern='PE32.*386.*Windows'
	sep=";"
	;;
    mingw64)
	platform=mingw
	bits=64
	pubtag=win64
	altbits=32 # We can usually use a mingw32 Makefile if toolprefix is OK
	toolprefix=x86_64-w64-mingw32- 
	pluginext=.dll
	hostwrapper=wine
	hostext=.exe
	identpattern='PE32.*x86-64.*Windows'
	sep=";"
	;;
    osx32)
	platform=osx
	pubtag=osx32
	pluginext=.dylib
	archflags="-arch i386"
	identpattern='Mach-O .*i386'
	;;
    osx64)
	platform=osx
	bits=64
	pubtag=osx
	pluginext=.dylib
	# This is a difficult choice for various reasons... have to ponder
	archflags="-mmacosx-version-min=10.7 -arch x86_64 -arch i386 -stdlib=libc++"
	identpattern='Mach-O 64-bit .*x86_64'
	;;
    piper)
	shift
	exec "$mydir"/process-piper.sh "$@"
	;;
    *)
	usage
	;;
esac;

shift

if [ -z "$platform" ]; then
    usage
else
    echo "(Platform is $platform, $bits bits, package platform tag $pubtag)"
fi

if [ -z "$pluginext" ]; then
    echo "Internal error: pluginext not set for platform $platform" 1>&2
    exit 2
fi

if [ -z "$identpattern" ]; then
    echo "Internal error: identpattern not set for platform $platform" 1>&2
    exit 2
fi

if [ t"$1" = t"-c" ]; then
    echo "(Building from clean)"
    do_rebuild=yes
    shift
fi

metadir="$mydir"/../METADATA

depincdir="$mydir"/../DEPENDENCIES/$platform$bits/include
deplibdir="$mydir"/../DEPENDENCIES/$platform$bits/lib
depincdir_generic="$mydir"/../DEPENDENCIES/generic/include

pyver=27
pyincdir="$mydir"/../DEPENDENCIES/$platform$bits/Python$pyver/include
numpyincdir="$mydir"/../DEPENDENCIES/$platform$bits/Python$pyver/Lib/site-packages/numpy/core/include
pylibdir="$mydir"/../DEPENDENCIES/$platform$bits/Python$pyver/libs

plugindirs="$@"
if [ -z "$plugindirs" ]; then
    plugindirs=$(cat METADATA/repos.txt |
		     grep -v vamp-plugin-sdk |
		     grep -v vamp-plugin-tester |
		     grep -v '^piper' |
		     awk '{ print $1; }')
else 
    for dir in $plugindirs ; do
	if [ ! -d "$dir" ]; then
	    echo "ERROR: Directory $dir not found"
	    usage
	fi
    done
fi

set -u

reportdir="REPORTS/$platform$bits"
packagedir="PACKAGES/$platform$bits"

mkdir -p "$reportdir" "$packagedir" || exit 1

built="/tmp/built.$$.txt"
testfailed="/tmp/testfailed.$$.txt"
envcheckfailed="/tmp/envcheckfailed.$$.txt"
notbuilt="/tmp/notbuilt.$$.txt"

trap 'rm -f "$built" "$envcheckfailed" "$testfailed" "$notbuilt"' 0
touch "$built" "$envcheckfailed" "$testfailed" "$notbuilt"

target_for() {
    local dir="$1"
    if grep -q "^$dir: " ${metadir}/maketarget.txt ; then
	grep "^$dir: " ${metadir}/maketarget.txt | head -1 | sed 's/^[^:]*: //'
    fi
}

custom_plugin_dir_for() {
    local dir="$1"
    if grep -q "^$dir: " ${metadir}/plugindir.txt ; then
	echo "${mydir}/../${dir}"/$(grep "^$dir: " ${metadir}/plugindir.txt | head -1 | sed 's/^[^:]*: //')
    fi
}

plugin_dir_for() {
    local dir="$1"
    local pdir=$(custom_plugin_dir_for "$dir")
    if [ -z "$pdir" ]; then
	case "$dir" in
	    /*) pdir="$dir" ;;
	    *) pdir="${mydir}/../${dir}";;
	esac
    fi
    echo "$pdir"
}

configure() {
    local dir="$1"
    if [ -f "$dir/configure" ] ; then
	( cd "$dir" ; ./configure ) 2>&1 | tee "$reportdir/$dir.configure.txt"
    fi
}

find_makefile() {
    local dir="$1"
    for f in \
	build/$platform$bits/Makefile.$platform$bits \
	build/$platform/Makefile.$platform$bits \
	build/$platform$bits/Makefile.$platform \
	build/$platform$bits/Makefile \
	build/Makefile.$platform$bits \
	Makefile.$platform$bits \
	build/$platform/Makefile.$platform \
	build/$platform/Makefile \
	build/Makefile.$platform \
	Makefile.$platform \
	build/$platform$altbits/Makefile.$platform$altbits \
	build/$platform/Makefile.$platform$altbits \
	build/$platform$altbits/Makefile.$platform \
	build/$platform$altbits/Makefile \
	build/Makefile.$platform$altbits \
	Makefile.$platform$altbits \
	Makefile ; do
	if [ -f "$dir/$f" ]; then
	    echo $f
	    break
	fi
    done
}

find_vampy_plugins_in() {
    local dir="$1"
    local pdir=$(plugin_dir_for "$dir")
    find "$pdir" -name \*.py -print0 | xargs -0 grep -l 'import.*\bvampy\b'
}

have_vampy_plugins() {
    local dir="$1"
    local plugs=$(find_vampy_plugins_in "$dir")
    if [ -z "$plugs" ]; then
	return 1
    else
	return 0
    fi
}

configure_maybe() {
    local dir="$1"
    if [ t"$do_rebuild" = t"yes" ]; then
        configure "$dir"
    else
        mfile=$(find_makefile "$dir")
        if [ -z "$mfile" ]; then
	    configure "$dir"
        fi
    fi
}

logfile_for() {
    local activity="$1"
    local dir="$2"
    echo "$reportdir/$dir.$activity.txt"
}

build() {
    local dir="$1"
    local log=$(logfile_for build "$dir")
    if configure_maybe "$dir"; then
	mfile=$(find_makefile "$dir")
	if [ -n "$mfile" ]; then
	    target=$(target_for "$dir")
	    TOOLPREFIX="$toolprefix" \
		CXXFLAGS="-I${depincdir} -I${pyincdir} -I${numpyincdir} -I${depincdir_generic} -I${mydir}/../vamp-plugin-sdk" \
		LDFLAGS="-L${deplibdir} -L${pylibdir} -L${mydir}/../vamp-plugin-sdk" \
		ARCHFLAGS="$archflags" \
		make -C "$dir" -f "$mfile" $target "${platform_defines[@]}" 2>&1 | \
		tee "$log"
	    return ${PIPESTATUS[0]}
	elif have_vampy_plugins "$dir"; then
	    return 0
	else
	    echo "*** Failed to find a Makefile in $dir!" | tee "$log"
	    return 1
	fi
    fi
}

rebuild() {
    local dir="$1"
    local log=$(logfile_for build "$dir")
    if configure_maybe "$dir"; then
	mfile=$(find_makefile "$dir")
	if [ -n "$mfile" ]; then
	    if make -C "$dir" -f "$mfile" distclean; then
		build "$dir"
	    elif make -C "$dir" -f "$mfile" clean; then
		build "$dir"
	    else
		echo "*** Failed to 'make clean' in $dir!" | tee "$log"
		return 1
	    fi
	elif have_vampy_plugins "$dir"; then
	    return 0
	else
	    echo "*** Failed to find a Makefile in $dir!" | tee "$log"
	    return 1
	fi
    fi
}

build_or_rebuild() {
    local dir="$1"
    if [ -n "$do_rebuild" ]; then
	rebuild "$dir"
    else
	build "$dir"
    fi
}

have_plugin() {
    local dir="$1"
    local log=$(logfile_for build "$dir")
    for x in "$dir/"*"$pluginext"; do 
	if [ -f "$x" ]; then
	    if file "$x" | grep -q "$identpattern" ; then
		return 0
	    else
		echo "Plugin $x exists, but fails to match file type for correct platform (file type is: `file $x`, expected pattern is: $identpattern)" | tee "$log"
		return 1
	    fi
	fi
    done
    have_vampy_plugins "$dir"
}

is_nondeterministic() {
    plugin_id="$1"
    grep -q "^$id\$" ${metadir}/nondeterministic.txt
}

plugin_ids_in() {
    local dir="$1"
    local pdir=$(plugin_dir_for "$dir")
    local vampydir=""
    if have_vampy_plugins "$pdir"; then
	vampydir="./vampy"
    fi
    # can't use sed to remove \r from DOS line endings -- BSD and GNU
    # vary in how they interpret \r escape -- so we use perl for that...
    VAMP_PATH="$pdir$sep$vampydir" $hostwrapper \
	vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids | \
	grep '^vamp:' | \
	sed 's/^vamp://' | \
	perl -p -e 's/\r//g' 
}

run_tester() {
    ##!!! todo: timeout if the plugin takes too long and report as failure
    local dir="$1"
    local log=$(logfile_for test "$dir")
    local ids=$(plugin_ids_in "$dir")
    local pdir=$(plugin_dir_for "$dir")
    cat /dev/null > "$log"
    if [ -z "$ids" ]; then
	echo 
	echo "No plugins reported to test in $dir" | tee -a "$log"
	echo "$dir" >> "$testfailed"
	return 1
    fi
    good=yes
    for id in $ids; do
	extra=""
	if is_nondeterministic "$id"; then
	    extra="-n"
	fi
	echo "Running command: VAMP_PATH=\"$pdir\" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext \"$extra\" \"$id\"" | tee -a "$log"
	if ( VAMP_PATH="$pdir$sep./vampy" $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext "$extra" "$id" 2>&1 | tee -a "$log" ; exit ${PIPESTATUS[0]} ) ; then
	    echo "OK" | tee -a "$log"
	else
	    if have_vampy_plugins "$pdir"; then # (don't attempt vampy+valgrind)
		echo | tee -a "$log"
		echo "Tester failed for id $id (not restarting with valgrind, as this is a VamPy plugin)" | tee -a "$log"
	    else
		echo | tee -a "$log"
		echo "Tester failed for id $id: running again with valgrind (if available) and verbose for a report..." | tee -a "$log"
		VAMP_PATH="$pdir$sep./vampy" $valgrind $hostwrapper vamp-plugin-tester/vamp-plugin-tester$hostext -v "$extra" "$id" 2>&1 | tee -a "$log"
	    fi
	    good=no
	fi
    done
    if [ "$good" != "yes" ]; then
	echo "$dir" >> "$testfailed"
	return 1
    else
	return 0
    fi
}

public_symbols_in() {
    lib="$1"
    # nm -g prints global symbols in both OS/X and GNU tools, but
    # printing only global *defined* symbols is harder. In GNU it is
    # nm -g --defined-only; the OS/X docs suggest nm -gu should work,
    # but it doesn't. What I think will work with both is simply
    # grepping out the undefineds:
    nm="${toolprefix}nm"
    if ! type -path "$nm" >/dev/null; then nm=nm; fi
    "$nm" -g "$lib" | grep -v ' U ' | awk '{ print $3; }'
}

env_test_exports() {
    local dir="$1"
    local log=$(logfile_for envtest "$dir")
    local good=yes
    for lib in "$dir"/*"$pluginext"; do 
	if [ ! -f "$lib" ]; then
	    # This should only happen if the glob was not expanded at all
	    echo "NOTE: no library found in $dir?" | tee -a "$log"
	    good=no
	    break
	fi
	echo 
	echo "Testing for exported symbols in $lib..."
	if public_symbols_in "$lib" | grep -q vampGetPluginDescriptor; then
	    others=`public_symbols_in "$lib" | grep -v vampGetPluginDescriptor`
	    if [ -n "$others" ]; then
		count=`echo "$others" | wc -l`
		echo "ERROR: library $lib exports $count extra symbols in addition to vampGetPluginDescriptor" | tee -a "$log"
		good=no
	    else
		echo "GOOD: library $lib only exports vampGetPluginDescriptor" | tee -a "$log"
	    fi
	else
	    echo "NOTE: found library $lib that is not a Vamp plugin library" | tee -a "$log"
	fi
    done
    [ "$good" = "yes" ]
}

env_test_stdout() {
    local dir="$1"
    local log=$(logfile_for envtest "$dir")
    local pdir=$(plugin_dir_for "$dir")
    local good=yes
    local ids=$(VAMP_PATH="$pdir" $hostwrapper vamp-plugin-sdk/host/vamp-simple-host$hostext --list-ids);
    echo 
    echo "Testing for any unwanted output to stdout..."
    for id in $ids; do
	case "$id" in
	    vamp:*) ;;
	    *)
		echo "ERROR: plugin in $dir prints to stdout as it runs: found text $id (should use stderr for debug logging to avoid mixing with batch output stream)" | tee -a "$log"
		good=no
		;;
	esac
    done
    [ "$good" = "yes" ]
}

env_test_cat() {
    local dir="$1"
    local log=$(logfile_for envtest "$dir")
    local pdir=$(custom_plugin_dir_for "$dir")
    local cdir=$(if test -n "$pdir" ; then echo "$pdir" ; else echo "$dir" ; fi)
    local good=yes
    local first=yes
    echo 
    echo "Testing some details of .cat files..."
    for catfile in "$cdir"/*".cat"; do
	if [ ! -f "$catfile" ]; then
	    # This should only happen if the glob was not expanded at all
	    echo "ERROR: no .cat file found in $dir" | tee -a "$log"
	    good=no
	    break
	fi
	if [ "$first" = "yes" ]; then
	    first=no
	else
	    echo "NOTE: multiple .cat files found in $cdir" | tee -a "$log"
	fi
    done
    if [ "$good" = "yes" ]; then
	excess=$(plugin_ids_in "$dir" | sed 's/^/vamp:/' | \
	    cat - "$cdir"/*".cat" | \
	    sed 's/::.*//' | \
            sort | \
            uniq -u)
	if [ -n "$excess" ]; then
	    echo "ERROR: excess or missing definitions in .cat file? $excess" | tee -a "$log"
	    good=no
	else
	    echo "GOOD: no excess or missing definitions in .cat files" | tee -a "$log"
	fi
    fi
    [ "$good" = "yes" ]
}

env_test_ttl() {
    local dir="$1"
    local log=$(logfile_for envtest "$dir")
    local pdir=$(custom_plugin_dir_for "$dir")
    local rdir=$(if test -n "$pdir" ; then echo "$pdir" ; else echo "$dir" ; fi)
    local good=yes
    local first=yes
    echo 
    echo "Testing existence of RDF files..."
    for ttlfile in "$rdir"/*.{n3,ttl}; do
	if [ ! -f "$ttlfile" ]; then
	    # Because we have two different extensions, this isn't an
	    # error as it is with e.g. .cat (because one or the other
	    # of .n3 and .ttl almost certainly won't exist). But if we
	    # drop out the bottom and first is still true, then we
	    # know neither matched.
	    :
	elif [ "$first" = "yes" ]; then
	    first=no
	else
	    echo "NOTE: multiple .ttl or .n3 files found in $rdir" | tee -a "$log"
	fi
    done
    if [ "$first" = "yes" ]; then
	echo "ERROR: no .ttl or .n3 files found in $rdir" | tee -a "$log"
	good=no; 
    else
	echo "GOOD: found one or more .ttl or .n3 files (we don't actually test their validity or content here though)" | tee -a "$log"
    fi
    [ "$good" = "yes" ]
}

env_test_accompaniments() {
    local dir="$1"
    local log=$(logfile_for envtest "$dir")
    local good=yes
    echo 
    echo "Testing existence of README and accompanying files..."
    if ! ls -1 "$dir" | egrep -qi "^readme(.txt)?$"; then
	echo "ERROR: no README file found" | tee -a "$log"
	good=no
    fi
    if ! ls -1 "$dir" | egrep -qi "^(copying|licen[cs]e)(.txt)?$"; then
	echo "ERROR: no COPYING or LICEN[CS]E file found" | tee -a "$log"
	good=no
    fi
    if ! ls -1 "$dir" | egrep -qi "^citation(.txt)?$"; then
	echo "NOTE: no CITATION file found" | tee -a "$log"
    fi
    [ "$good" = "yes" ]
}

run_environmental_tests() {
    local dir="$1"
    local log=$(logfile_for envtest "$dir")
    local allgood=yes
    cat /dev/null > "$log"
    for test in exports stdout cat ttl accompaniments; do
	"env_test_$test" "$dir" || allgood=no
    done
    if [ "$allgood" != "yes" ]; then
	echo "$dir" >> "$envcheckfailed"
	return 1
    else
	return 0
    fi
}

package() {
    local dir="$1"
    local id=$(vcs_id "$dir")
    local pstub="$dir-$pubtag-$id"
    local pdir="$packagedir/$pstub"
    mkdir -p "$pdir"
    ( cd "$dir" ; 
	cp -av \
	    *"$pluginext" \
	    *.cat \
	    *.n3 \
	    *.ttl \
	    [Rr][Ee][Aa][Dd][Mm][Ee]* \
	    [Cc][Oo][Pp][Yy][Ii][Nn][Gg]* \
	    [Ll][Ii][Cc][Ee][Nn][CcSs][Ee]* \
	    [Cc][Ii][Tt][Aa][Tt][Ii][Oo][Nn]* \
	    [Cc][Hh][Aa][Nn][Gg][Ee][Ll][Oo][Gg]* \
	    ../"$pdir"/ || true
	if have_vampy_plugins "$dir"; then
	    find_vampy_plugins_in "$dir" | while read p; do
					       cp -av "$p" ../"$pdir"/
					   done
	fi
	)
    ( cd "$packagedir";
	if [ "$platform" = "mingw" ]; then
	    zip -r "$pstub".zip "$pstub"
	else
	    tar cvjf "$pstub".tar.bz2 "$pstub"
	fi;
	rm -rf "$pstub"
    )
}

if ! build_or_rebuild "vamp-plugin-sdk"; then 
    echo "*** Failed to build Vamp plugin SDK!"
    exit 1
fi

# Ensure we can only link statically against these
for x in vamp-hostsdk vamp-sdk; do 
    for y in dylib dll so; do
	rm -f "vamp-plugin-sdk/lib$x.$y"
	rm -f "vamp-plugin-sdk/$x.$y"
    done
done

if ! build_or_rebuild "vamp-plugin-tester"; then 
    echo "*** Failed to build Vamp plugin tester!"
    exit 1
fi

need_vampy=no
for dir in $plugindirs ; do
    if have_vampy_plugins "$dir"; then
	need_vampy=yes
	break
    fi
done
if [ "$need_vampy" = "yes" ] && [ "$plugindirs" != "vampy" ]; then
    if ! build_or_rebuild "vampy"; then 
	echo "*** Failed to build VamPy! (needed for other plugins)"
	exit 1
    fi
fi

for dir in $plugindirs ; do
    dir=${dir%/*}
    echo
    echo "Processing: $dir"
    if [ ! -d "$dir" ]; then
	echo "Directory $dir not found!"
	echo "$dir" >> "$notbuilt"
    elif build_or_rebuild "$dir"; then
	if have_plugin "$dir" ; then
	    echo "$dir" >> "$built"
	    if ! run_tester "$dir"; then
		echo "Tester failed for $dir"
	    fi
	    if ! run_environmental_tests "$dir"; then
		echo "Environmental tests failed for $dir"
	    fi
	else 
	    log=$(logfile_for build "$dir")
	    echo "Build apparently succeeded, but no resulting plugin(s) found, or plugin(s) have wrong file type or platform" | tee -a "$log"
	    echo "$dir" >> "$notbuilt"
	fi
    else
	echo "$dir" >> "$notbuilt"
    fi
    slog=$(logfile_for summary "$dir")
    echo "initialising logfile $slog for $dir"
    cat /dev/null > "$slog"
done

cat "$built" | while read dir; do
    package "$dir"
done

echo
echo "** Built, tested, checked, and packaged:"
cat "$built" | while read dir; do
    slog=$(logfile_for summary "$dir")
    if ! grep -q "^$dir\$" "$testfailed"; then
	if ! grep -q "^$dir\$" "$envcheckfailed"; then
	    echo "$dir"
	    echo "$dir: OK: Success" >> "$slog"
	fi
    fi
done | sort

echo
echo "** Failed tests:"
cat "$testfailed" | sort | uniq | while read dir; do
    slog=$(logfile_for summary "$dir")
    echo "$dir"
    echo "$dir: TEST_FAILED: Built successfully, but failed tests" >> "$slog"
done

echo
echo "** Failed environmental checks:"
cat "$envcheckfailed" | sort | uniq | while read dir; do
    slog=$(logfile_for summary "$dir")
    echo "$dir"
    echo "$dir: ENV_FAILED: Built successfully, but failed environmental checks" >> "$slog"
done

echo
echo "** Failed to build:"	    
cat "$notbuilt" | sort | while read dir; do
    slog=$(logfile_for summary "$dir")
    echo "$dir"
    echo "$dir: BUILD_FAILED: Failed to compile" >> "$slog"
done

echo