Mercurial > hg > webaudioevaluationtool
changeset 332:8207063052bf WAC2016
Merged. Updated parts RE Brechts email. browser compatibility, manipulation of audio: volume slider, loudness equalisation, cross-fade.
| author | Nicholas Jillings <nicholas.jillings@eecs.qmul.ac.uk> |
|---|---|
| date | Wed, 14 Oct 2015 20:59:21 +0100 |
| parents | 3a43f9e4cecf (current diff) 414913550c09 (diff) |
| children | 045825a3b2ba |
| files | docs/WAC2016/WAC2016.pdf docs/WAC2016/WAC2016.tex |
| diffstat | 4 files changed, 650 insertions(+), 68 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/analyse.html Wed Oct 14 20:59:21 2015 +0100 @@ -0,0 +1,567 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + + <!-- Always force latest IE rendering engine (even in intranet) & Chrome Frame + Remove this if you use the .htaccess --> + <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> + + <title>Analysis</title> + <meta name="description" content="Show results from subjective evaluation"> + <meta name="author" content="Brecht De Man"> + + <script type="text/javascript" src="https://www.google.com/jsapi"></script> + <script type="text/javascript"> + // To aid 'one-page set-up' all scripts and CSS must be included directly in this file! + + google.load("visualization", "1", {packages:["corechart"]}); + + /************* + * SETUP * + *************/ + // folder where to find the XML files + xmlFileFolder = "saves"; + // array of XML files + var xmlFiles = ['McG-A-2014-03.xml','McG-B-2014-03.xml','McG-C-2014-03.xml']; + + //TODO: make retrieval of file names automatic / drag files on here + + /**************** + * VARIABLES * + ****************/ + + // Counters + // How many files, audioHolders, audioElementes and statements annotated (don't count current one) + var numberOfFiles = -1; + var numberOfaudioHolders = -1; + var numberOfaudioElementes = -1; + var numberOfStatements = -1; + var numberOfSkippedComments = 0; + + // Object arrays + var fileNameArray = []; + var subjectArray = []; + var audioHolderArray = []; + var audioElementArray = []; + + // End of (file, audioholder, audioelement) flags + var newFile = true; + var newAudioHolder = true; + var newAudioElement = true; + + var fileCounter = 0; // file index + var audioHolderCounter=0; // audioholder index (current XML file) + var audioElementCounter=0; // audioelement index (current audioholder) + var statementNumber=0; // total number of statements + + var root; // root of XML file + var commentInFull = ''; // full comment + + var playAudio = true; // whether corresponding audio should be played back + + // // Measuring time + // var lastTimeMeasured = -1; // + // var durationLastAnnotation = -1; // duration of last annotation + // var timeArray = []; + // var MIN_TIME = 1.0; // minimum time counted as significant + // var measurementPaused = false; // whether time measurement is paused + // var timeInBuffer = 0; // + + var topLevel; + window.onload = function() { + // Initialise page + topLevel = document.getElementById('topLevelBody'); + var setup = document.createElement('div'); + setup.id = 'setupTagDiv'; + loadAllFiles(); + + makePlots(); + // measure time at this point: + lastTimeMeasured = new Date().getTime(); // in milliseconds + }; + + // Assert function + function assert(condition, message) { + if (!condition) { + message = message || "Assertion failed"; + if (typeof Error !== "undefined") { + throw new Error(message); + } + throw message; // Fallback + } + } + + function median(values) { + values.sort( function(a,b) {return a - b;} ); + var half = Math.floor(values.length/2); + if(values.length % 2) + return values[half]; + else + return (values[half-1] + values[half]) / 2.0; + } + + /*********************** + * TIME MEASUREMENT * + ************************/ + + // measure time since last time this function was called + function timeSinceLastCall() { + // current time + var currentTime = new Date().getTime(); + // calculate time difference + var timeDifference = currentTime - lastTimeMeasured + timeInBuffer; + // clear buffer (for pausing) + timeInBuffer = 0; + // remember last measured time + lastTimeMeasured = currentTime; + return timeDifference; + } + + // pause time measurement + function pauseTimeMeasurement() { + // UN-PAUSE + if (measurementPaused) { // already paused + // button shows 'pause' again + document.getElementById('pauseButton').innerHTML = 'Pause'; + // toggle state + measurementPaused = false; + // resume time measurement + lastTimeMeasured = new Date().getTime(); // reset time, discard time while paused + } else { // PAUSE + // button shows 'resume' + document.getElementById('pauseButton').innerHTML = 'Resume'; + // toggle state + measurementPaused = true; + // pause time measurement + timeInBuffer = timeSinceLastCall(); + } + } + + // show elapsed time on interface + function showTimeElapsedInSeconds() { + // if paused: un-pause + if (measurementPaused) { + pauseTimeMeasurement(); + } + + // time of last annotation + var lastAnnotationTime = timeSinceLastCall()/1000; + document.getElementById('timeDisplay').innerHTML = lastAnnotationTime.toFixed(2); + // average time over last ... annotations + var avgAnnotationTime; + var numberOfElementsToAverage = + document.getElementById('numberOfTimeAverages').value; + if (isPositiveInteger(numberOfElementsToAverage)) { + avgAnnotationTime = + calculateAverageTime(lastAnnotationTime, + Number(numberOfElementsToAverage)); + } else { + // change text field content to 'ALL' + document.getElementById('numberOfTimeAverages').value = 'ALL'; + avgAnnotationTime = calculateAverageTime(lastAnnotationTime, -1); + } + document.getElementById('timeAverageDisplay').innerHTML = avgAnnotationTime.toFixed(2); + } + + // auxiliary function: is string a positive integer? + // http://stackoverflow.com/questions/10834796/... + // validate-that-a-string-is-a-positive-integer + function isPositiveInteger(str) { + var n = ~~Number(str); + return String(n) === str && n >= 0; + } + + // calculate average time + function calculateAverageTime(newTimeMeasurementInSeconds,numberOfPoints) { + // append last measurement time to time array, if significant + if (newTimeMeasurementInSeconds > MIN_TIME) { + timeArray.push(newTimeMeasurementInSeconds); + } + // average over last N elements of this array + if (numberOfPoints < 0 || numberOfPoints>=timeArray.length) { // calculate average over all + var sum = 0; + for (var i = 0; i < timeArray.length; i++) { + sum += timeArray[i]; + } + averageOfTimes = sum/timeArray.length; + } else { // calculate average over specified number of times measured last + var sum = 0; + for (var i = timeArray.length-numberOfPoints; i < timeArray.length; i++) { + sum += timeArray[i]; + } + averageOfTimes = sum/numberOfPoints; + } + return averageOfTimes; + } + + + /******************************** + * PLAYBACK OF AUDIO * + ********************************/ + + //PLAYaudioElement + // Keep track of whether audio should be played + function playFlagChanged(){ + playAudio = playFlag.checked; // global variable + + if (!playAudio){ // if audio needs to stop + audio.pause(); // stop audio - if anything is playing + currently_playing = ''; // back to empty string so playaudioElement knows nothing's playing + } + } + + // audioHolder that's currently playing + var currently_playing_audioHolder = ''; // at first: empty string + var currently_playing_audioElement = ''; + var audio; + + // Play audioElement of audioHolder if available, from start or from same position + function playaudioElement(audioHolderName, audioElementerName){ + if (playAudio) { // if enabled + // get corresponding file from folder + var file_location = 'audio/'+audioHolderName + '/' + audioElementerName + '.mp3'; // fixed path and file name format + + // if not available, show error/warning message + //TODO ... + + // if nothing playing yet, start playing + if (currently_playing_audioHolder == ''){ // signal that nothing is playing + //playSound(audioBuffer); + audio = new Audio(file_location); + audio.loop = true; // loop when end is reached + audio.play(); + currently_playing_audioHolder = audioHolderName; + currently_playing_audioElement = audioElementerName; + } else if (currently_playing_audioHolder != audioHolderName) { + // if different audioHolder playing, stop that and start playing + audio.pause(); // stop audio + audio = new Audio(file_location); // load new file + audio.loop = true; // loop when end is reached + audio.play(); // play audio from the start + currently_playing_audioHolder = audioHolderName; + currently_playing_audioElement = audioElementerName; + } else if (currently_playing_audioElement != audioElementerName) { + // if same audioHolder playing, start playing from where it left off + skipTime = audio.currentTime; // time to skip to + audio.pause(); // stop audio + audio = new Audio(file_location); + audio.addEventListener('loadedmetadata', function() { + this.currentTime = skipTime; + console.log('Loaded '+audioHolderName+'-'+audioElementerName+', playing from '+skipTime); + }, false); // skip to same time when audio is loaded! + audio.loop = true; // loop when end is reached + audio.play(); // play from that time + audio.currentTime = skipTime; + currently_playing_audioHolder = audioHolderName; + currently_playing_audioElement = audioElementerName; + } + // if same audioElement playing: keep on playing (i.e. do nothing) + } + } + + /******************** + * READING FILES * + ********************/ + + // Read necessary data from XML file + function readXML(xmlFileName){ + if (window.XMLHttpRequest) + {// code for IE7+, Firefox, Chrome, Opera, Safari + xmlhttp=new XMLHttpRequest(); + } + else + {// code for IE6, IE5 + xmlhttp=new ActiveXObject("Microsoft.XMLHTTP"); + } + xmlhttp.open("GET",xmlFileName,false); + xmlhttp.send(); + return xmlhttp.responseXML; + } + + // go over all files and compute relevant statistics + function loadAllFiles() { + // retrieve information from XMLs + + for (fileIndex = 0; fileIndex < xmlFiles.length; fileIndex++) { + xmlFileName = xmlFileFolder+"/"+xmlFiles[fileIndex]; + xml = readXML(xmlFileName); + if (xml != null) { // if file exists + // append file name to array of file names + fileNameArray.push(xmlFiles[fileIndex]); + + // get root of XML file + root = xml.getElementsByTagName('browserevaluationresult')[0]; + + // get subject ID, add to array if not already there + pretest = root.getElementsByTagName('pretest')[0]; + subjectID = pretest.getElementsByTagName('comment')[0]; + if (subjectID.getAttribute('id')!='sessionId') { // warning in console when not available + console.log(xmlFiles[fileIndex]+': no SessionID available'); + } + if (subjectArray.indexOf(subjectID.textContent) == -1) { // if not already in array + subjectArray.push(subjectID.textContent); // append to array + } + + // go over all audioHolders, add to array if not already there + audioHolderNodes = root.getElementsByTagName('audioholder'); + // go over audioHolderNodes and append audioHolder name when not present yet + for (audioHolderIndex = 0; audioHolderIndex < audioHolderNodes.length; audioHolderIndex++) { + audioHolderName = audioHolderNodes[audioHolderIndex].getAttribute('id'); + if (audioHolderArray.indexOf(audioHolderName) == -1) { // if not already in array + audioHolderArray.push(audioHolderName); // append to array + } + // within each audioHolder, go over all audioElement IDs, add to array if not already there + audioElementNodes = audioHolderNodes[audioHolderIndex].getElementsByTagName('audioelement'); + for (audioElementIndex = 0; audioElementIndex < audioElementNodes.length; audioElementIndex++) { + audioElementName = audioElementNodes[audioElementIndex].getAttribute('id'); + if (audioElementArray.indexOf(audioElementName) == -1) { // if not already in array + audioElementArray.push(audioElementName); // append to array + } + } + } + // count occurrences of each audioHolder + // ... + } + else { + console.log('XML file '+xmlFileName+' not found.'); + } + } + + // sort alphabetically + fileNameArray.sort(); + subjectArray.sort(); + audioHolderArray.sort(); + audioElementArray.sort(); + + // display all information in HTML + // show XML file folder + document.getElementById('xmlFileFolder_span').innerHTML = "\""+xmlFileFolder+"/\""; + // show number of files + document.getElementById('numberOfFiles_span').innerHTML = fileNameArray.length; + // show list of subject names + document.getElementById('subjectArray_span').innerHTML = subjectArray.toString(); + // show list of audioHolders + document.getElementById('audioHolderArray_span').innerHTML = audioHolderArray.toString(); + // show list of audioElementes + document.getElementById('audioElementArray_span').innerHTML = audioElementArray.toString(); + } + + function makePlots() { + // create value array + var ratings = []; // 3D matrix of ratings (audioHolder, audioElement, subject) + for (audioHolderIndex = 0; audioHolderIndex < audioHolderNodes.length; audioHolderIndex++) { + ratings.push([]); + for (audioElementIndex = 0; audioElementIndex < audioElementNodes.length; audioElementIndex++) { + ratings[audioHolderIndex].push([]); + } + } + + // go over all XML files + for (fileIndex = 0; fileIndex < xmlFiles.length; fileIndex++) { + xmlFileName = xmlFileFolder+"/"+xmlFiles[fileIndex]; + xml = readXML(xmlFileName); + if (xml != null) { // if file exists + // get root of XML file + root = xml.getElementsByTagName('browserevaluationresult')[0]; + // go over all audioHolders + audioHolderNodes = root.getElementsByTagName('audioholder'); + for (audioHolderIndex = 0; audioHolderIndex < audioHolderNodes.length; audioHolderIndex++) { + audioHolderName = audioHolderNodes[audioHolderIndex].getAttribute('id'); + audioElementNodes = audioHolderNodes[audioHolderIndex].getElementsByTagName('audioelement'); + // go over all audioelements + for (audioElementIndex = 0; audioElementIndex < audioElementNodes.length; audioElementIndex++) { + audioElementName = audioElementNodes[audioElementIndex].getAttribute('id'); + // get value + var value = audioElementNodes[audioElementIndex].getElementsByTagName("value")[0].textContent; + if (value) { // if not empty, null, undefined... + ratingValue = parseFloat(value); + // add to matrix + ratings[audioHolderIndex][audioElementIndex].push(ratingValue) + } + } + } + + // go over all audioHolders + + // go over all audioElements within audioHolder, see if present in idMatrix, add if not + // add corresponding rating to 'ratings', at position corresponding with position in idMatrix + } + } + + for (audioHolderIndex = 0; audioHolderIndex < audioHolderArray.length; audioHolderIndex++) { + audioHolderName = audioHolderArray[audioHolderIndex]; // for this song + tickArray = [] + medianOfAudioElement = [] + + raw_data = [['SubjectID', 'Rating']]; + audioElIdx = 0; + for (audioElementIndex = 0; audioElementIndex<ratings[audioHolderIndex].length; audioElementIndex++){ + if (ratings[audioHolderIndex][audioElementIndex].length>0) { + audioElIdx++; // increase if not empty + // make tick label + tickArray.push({v:audioElIdx, f: audioElementArray[audioElementIndex]}); + // add median + medianOfAudioElement.push(median(ratings[audioHolderIndex][audioElementIndex])); + } + for (subject = 0; subject<ratings[audioHolderIndex][audioElementIndex].length; subject++){ + // add subject-value pair for each subject + raw_data.push([audioElIdx, ratings[audioHolderIndex][audioElementIndex][subject]]); + } + } + + // create plot (one per song) + var data = google.visualization.arrayToDataTable(raw_data); + + var options = { + title: audioHolderName, + hAxis: {title: 'Subject', minValue: 0, maxValue: audioElIdx+1, + ticks: tickArray}, + vAxis: {title: 'Rating', minValue: 0, maxValue: 1}, + seriesType: 'scatter', + legend: 'none' + }; + var div = document.createElement('div'); + document.body.appendChild(div); + div.id = 'div_'+audioHolderName; + div.style.width = '1100px'; + div.style.height = '350px'; + var chart = new google.visualization.ComboChart(document.getElementById('div_'+audioHolderName)); + chart.draw(data, options); + + // box plots + // function drawVisualization() { + // // Create and populate the data table. + // var data = google.visualization.arrayToDataTable([ + // ['ID', 'IQR', '', '', '', 'Median', 'Average'], + // ['Serie1', 20, 28, 38, 45, 20, 25], + // ['Serie2', 31, 38, 55, 66, 30, 35], + // ['Serie3', 50, 55, 77, 80, 10, 15], + // ['Serie4', 77, 77, 66, 50, 20, 25], + // ['Serie5', 68, 66, 22, 15, 30, 35] + // // Treat first row as data as well. + // ]); + // // Create and draw the visualization. + // var ac = new google.visualization.ComboChart(document.getElementById('visualization')); + // ac.draw(data, { + // title : 'Box Plot with Median and Average', + // width: 600, + // height: 400, + // vAxis: {title: "Value"}, + // hAxis: {title: "Serie ID"}, + // series: { 0: {type: "candlesticks"}, 1: {type: "line", pointSize: 10, lineWidth: + // 0 }, 2: {type: "line", pointSize: 10, lineWidth: 0, color: 'black' } } + // }); + // } + } + } + + </script> + + + + <style> + div { + padding: 2px; + margin-top: 2px; + margin-bottom: 2px; + } + div.head{ + margin-left: 10px; + border: black; + border-width: 2px; + border-style: solid; + } + div.attrib{ + margin-left:25px; + border: black; + border-width: 2px; + border-style: dashed; + margin-bottom: 10px; + } + div#headerMatter{ + background-color: #FFFFCC; + } + div#currentStatement{ + font-size:3.0em; + font-weight: bold; + + } + div#debugDisplay { + color: #CCCCCC; + font-size:0.3em; + } + span#scoreDisplay { + font-weight: bold; + } + div#wrapper { + width: 780px; + border: 1px solid black; + overflow: hidden; /* add this to contain floated children */ + } + div#instrumentSection { + width: 250px; + border: 1px solid red; + display: inline-block; + } + div#featureSection { + width: 250px; + border: 1px solid green; + display: inline-block; + } + div#valenceSection { + width: 250px; + border: 1px solid blue; + display: inline-block; + } + button#previousComment{ + width: 120px; + height: 150px; + font-size:1.5em; + } + button#nextComment{ + width: 666px; + height: 150px; + font-size:1.5em; + } + ul + { + list-style-type: none; /* no bullet points */ + margin-left: -20px; /* less indent */ + margin-top: 0px; + margin-bottom: 5px; + } + </style> + + </head> + + <body> + <h1>Subjective evaluation results</h1> + + <div id="debugDisplay"> + XML file folder: <span id="xmlFileFolder_span"></span> + </div> + + <div id="headerMatter"> + <div> + <strong>Result XML files:</strong> <span id="numberOfFiles_span"></span> + </div> + <div> + <strong>Audioholders in dataset:</strong> <span id="audioHolderArray_span"></span> + </div> + <div> + <strong>Subjects in dataset:</strong> <span id="subjectArray_span"></span> + </div> + <div> + <strong>Audioelements in dataset:</strong> <span id="audioElementArray_span"></span> + </div> + <br> + </div> + <br> + + <!-- Show time elapsed + The last annotation took <strong><span id="timeDisplay">(N/A)</span></strong> seconds. + <br>--> + + </body> +</html>
--- a/docs/WAC2016/WAC2016.tex Wed Oct 14 20:15:31 2015 +0100 +++ b/docs/WAC2016/WAC2016.tex Wed Oct 14 20:59:21 2015 +0100 @@ -3,6 +3,8 @@ \usepackage{enumitem} % tighten itemize etc by appending '[noitemsep,nolistsep]' \usepackage{cleveref} +\graphicspath{{img/}} % put the images in this folder + \begin{document} % Copyright @@ -132,16 +134,19 @@ % Listening tests/perceptual audio evaluation: what are they, why are they important % As opposed to limited scope of WAC15 paper: also musical features, realism of sound effects / sound synthesis, performance of source separation and other algorithms... Perceptual evaluation of audio, in the form of listening tests, is a powerful way to assess anything from audio codec quality over realism of sound synthesis to the performance of source separation, automated music production and other auditory evaluations. - In less technical areas, the framework of a listening test can be used to measure emotional response to music or test cognitive abilities. % maybe some references? If there's space. + In less technical areas, the framework of a listening test can be used to measure emotional response to music or test cognitive abilities. + % maybe some references? If there's space. % check out http://link.springer.com/article/10.1007/s10055-015-0270-8 - only paper that cited WAC15 paper % Why difficult? Challenges? What constitutes a good interface? % Technical, interfaces, user friendliness, reliability - There are multiple programs for performing perceptual listening tests, as can be seen in Table \ref{tab:toolboxes}. Some are designed to have only one interface type or only work using proprietary software. The Web Audio Evaluation Toolbox is different as it does not require proprietary software and provides many interface and test types in one, common environment. Note that the design of an effective listening test further poses many challenges unrelated to interface design, which are beyond the scope of this paper \cite{bech}. + Several applications for performing perceptual listening tests currently exist, a subset shown in Table \ref{tab:toolboxes}. The Web Audio Evaluation Toolbox stands out as it does not require proprietary software or a specific platform. It also provides a wide range of interface and test types in one user friendly environment. Furthermore, it does not require any progamming experience as any test based on the default test types can be configured in the browser as well. Note that the design of an effective listening test further poses many challenges unrelated to interface design, which are beyond the scope of this paper \cite{bech}. % Why in the browser? - Web Audio API has important features for performing perceptual tests including sample level manipulation of audio streams \cite{schoeffler2015mushra}, synchronous playback and flexible playback. Being in the browser also allows leveraging the flexible object oriented JavaScript format and native support for web documents, such as the extensible markup language (XML) which is used for configuration and test results. Using the web also simplifies test deployment to requiring a basic web server with advanced functionality such as test collection and automatic processing using PHP. As recruiting participants can be very time-consuming, and as for some tests a large number of participants is needed, browser-based tests \cite{schoeffler2015mushra}. However, to our knowledge, no tool currently exists that allows the creation of a remotely accessible listening test. BeaqleJS \cite{beaqlejs} also operates in the browser, however BeaqleJS does not make use of the Web Audio API.%requires programming knowledge?... + Web Audio API has important features for performing perceptual tests including sample level manipulation of audio streams \cite{schoeffler2015mushra} and the ability for synchronous and flexible playback. Being in the browser also allows leveraging the flexible object oriented JavaScript format and native support for web documents, such as the extensible markup language (XML) which is used for configuration and test result files. Using the web also reduces deployment requirements to a basic web server with advanced functionality such as test collection and automatic processing using PHP. As recruiting participants can be very time-consuming, and as for some tests a large number of participants is needed, browser-based tests \cite{schoeffler2015mushra} can resolve these problems by enabling participants in multiple locations to perform the test. However, to our knowledge, no tool currently exists that allows the creation of a remotely accessible listening test. + + Both BeaqleJS \cite{beaqlejs} and mushraJS\footnote{https://github.com/akaroice/mushraJS} also operate in the browser, however BeaqleJS does not make use of the Web Audio API and therefore lacks arbitrary manipulation of audio stream samples, and neither offer an adequately wide choice of test designs for them to be useful to many researchers. %requires programming knowledge?... % only browser-based? \begin{table*}[ht] @@ -153,10 +158,10 @@ \hline APE & \cite{ape} & MATLAB & multi-stimulus, 1 axis per attribute & & \\ BeaqleJS & \cite{beaqlejs} & JavaScript & ABX, MUSHRA & (not natively supported) & \\ - HULTI-GEN & \cite{hultigen} & MAX & See Table \ref{tab:toolbox_interfaces} & & \checkmark \\ - mushraJS & \footnote{https://github.com/akaroice/mushraJS} & JavaScript & MUSHRA & \checkmark & \\ + HULTI-GEN & \cite{hultigen} & MAX & See Table \ref{tab:toolbox_interfaces}& & \checkmark \\ + mushraJS & & JavaScript & MUSHRA & \checkmark & \\ MUSHRAM & \cite{mushram} & MATLAB & MUSHRA & & \\ - Scale & \cite{scale} & MATLAB & See Table \ref{tab:toolbox_interfaces} & & \\ + Scale & \cite{scale} & MATLAB & See Table \ref{tab:toolbox_interfaces} & & \\ WhisPER & \cite{whisper} & MATLAB & See Table \ref{tab:toolbox_interfaces} & & \checkmark \\ \textbf{WAET} & \cite{waet} & JavaScript & \textbf{all of the above, see Table \ref{tab:toolbox_interfaces}} & \checkmark & \checkmark \\ \hline @@ -182,14 +187,14 @@ Comparison Category Rating (CCR) Scale & \checkmark & & \checkmark & \checkmark \\ 9 Point Hedonic Category Rating Scale & \checkmark & & \checkmark & \checkmark \\ ITU-R 5 Point Continuous Impairment Scale & \checkmark & & & \checkmark \\ - Pairwise Comparison (Better/Worse) & \checkmark & & & \checkmark \\ - APE style & & & & \checkmark \\ - Multi attribute ratings & \checkmark & & & \checkmark \\ - AB Test & \checkmark & & & \checkmark \\ + Pairwise Comparison / AB test & \checkmark & & & \checkmark \\ + Multi-attribute ratings & \checkmark & & & \checkmark \\ ABX Test & \checkmark & & & \checkmark \\ - ``Adaptive psychophysical methods'' & & & \checkmark & \\ + Adaptive psychophysical methods & & & \checkmark & \\ Repertory Grid Technique (RGT) & & & \checkmark & \\ - (Semantic differential) & & \checkmark & (\checkmark) & \\ % same as a few of the above + (Semantic differential) & & & (\checkmark) & \\ % same as a few of the above + n-Alternative Forced choice & & \checkmark & & \\ + \hline \end{tabular} \end{center} @@ -232,116 +237,123 @@ \section{Architecture} % title? 'back end'? % NICK %A slightly technical overview of the system. Talk about XML, JavaScript, Web Audio API, HTML5. - WAET utilises the Web Audio API for audio playback and uses a sparse subset of the Web Audio API functionality, however the performance of WAET comes directly from the Web Audio API. Listening tests can convey large amounts of information other than obtaining the perceptual relationship between the audio fragments. WAET specifically can obtain which parts of the audio fragments were listened to and when, at what point in the audio stream did the participant switch to a different fragment and what new rating did they give a fragment. Therefore it is possible to not only evaluate the perceptual research question but also evaluate if the participant performed the test well and therefore if their results are representative or should be discarded as an outlier. + + While WAET uses a sparse subset of the Web Audio API functionality, its performance comes directly from using the Web Audio API for audio playback. Listening tests can convey large amounts of information other than obtaining the perceptual relationship between the audio fragments. Specifically, with WAET one can obtain which parts of the audio fragments were listened to and when, at what point in the audio stream the participant switched to a different fragment, and how a fragment's rating was adjusted over time within a session, to name a few. Not only does this allow to evaluate a wealth of perceptual aspects, but it helps detect poor participants whose results are potentially not representative. - One of the key initial design parameters for WAET is to make the tool as open as possible to non-programmers and to this end the tool has been designed in such a way that all of the user modifiable options are included in a single XML document. This document is loaded up automatically by the web page and the JavaScript code parses and loads any extra resources required to create the test. + One of the key initial design parameters for WAET is to make the tool as open as possible to non-programmers and to this end the tool has been designed in such a way that all of the user modifiable options are included in a single XML document. This XML document is called the specification document and can be designed either by manually writing the XML (or modifying an existing document or template) or using our included test creator. These are standalone HTML pages which do not require any server or internet connection and help a build your test specification document. The first (test\_create.html) is for simpler tests and operates step-by-step using in-page popups to guide the user. It supports media through drag and drop and clutter free interface. The advanced version is for more advanced tests where raw XML manipulation is not wanted but the same freedom is required (whilst keeping a safety net). Both models support automatic XML verification to ensure the XML file is valid and will highlight areas which are either incorrect and would cause an error, or options which should be removed as they are blank. + + The basic test create utilises some web audio of its own. It utilises the API to perform quick playback checks, but also allow for loudness normalisation techniques inspired from \cite{ape}. These are calculated offline by accessing the raw audio samples exposed from the buffer before being applied to the audio element as a gain attribute. This is used in the test to perform loudness normalisation without needing to edit any audio files. Equally the gain can be modified in either editor using an HTML5 slider or number box. %Describe and/or visualise audioholder-audioelement-... structure. - The specification document also contains the URL of the audio fragments for each test page. These fragments are downloaded asynchronously and decoded offline by the Web Audio offline decoder. The resulting buffers are assigned to a custom Audio Objects node which tracks the fragment buffer, the playback bufferSourceNode, the XML information including its unique test ID, the interface object(s) associated with the fragment and any metric or data collection objects. The Audio Object is controlled by an over-arching custom Audio Context node (not to be confused with the Web Audio Context), this parent JS Node allows for session wide control of the Audio Objects including starting and stopping playback of specific nodes. + The specification document also contains the URL of the audio fragments for each test page. These fragments are downloaded asynchronously in the test and decoded offline by the Web Audio offline decoder. The resulting buffers are assigned to a custom Audio Objects node which tracks the fragment buffer, the playback bufferSourceNode, the XML information including its unique test ID, the interface object(s) associated with the fragment and any metric or data collection objects. The Audio Object is controlled by an over-arching custom Audio Context node (not to be confused with the Web Audio Context), this parent JS Node allows for session wide control of the Audio Objects including starting and stopping playback of specific nodes. - The only issue with this model is the bufferNode in the Web Audio API, which is implemented as a 'use once' object which, once the buffer has been played, the buffer must be discarded as it cannot be instructed to play the buffer again. Therefore on each start request the buffer object must be created and then linked with the stored bufferSourceNode. This is an odd behaviour for such a simple object which has no alternative except to use the HTML5 audio element, however they do not have the ability to synchronously start on a given time and therefore not suited. + The only issue with this model is the bufferNode in the Web Audio API, which is implemented as a `use once' object which, once the buffer has been played, the buffer must be discarded as it cannot be instructed to play the buffer again. Therefore on each start request the buffer object must be created and then linked with the stored bufferSourceNode. This is an odd behaviour for such a simple object which has no alternative except to use the HTML5 audio element, however they do not have the ability to synchronously start on a given time and therefore not suited. + + In the test the each buffer node is connected to a gain node which will operate at the level determined by the specification document. Therefore it is technically possible to perform a 'Method of Adjustment' test where an interface could directly manipulate these gain nodes. Equally there is an optional 'Master Volume' slider which can be shown on the test GUI. This slider modifies a gain node before the destination node. This slider can also be monitored and therefore its data tracked providing extra validation. Of course this slider is not indicative of the final volume exiting the speakers and therefore its use should only be considered in a lab condition environment to ensure proper behaviour. Finally the gain nodes allow for cross-fading between samples when operating in synchronous playback. Cross-fading can either be fade-out fade-in or a true cross-fade. %Which type of files? WAV, anything else? Perhaps not exhaustive list, but say something along the lines of 'whatever browser supports'. Compatability? - The media files supported depend on the browser level support for the initial decoding of information and is the same as the browser support for the HTML5 audio element. Therefore the most widely supported media file is the wave (.WAV) format which can be accpeted by every browser supporting the Web Audio API. The next best supported audio only formats are MP3 and AAC (in MP4) which are supported by all major browsers, Firefox relies on OS decoders and therefore its support is predicated by the OS support. + The media files supported depend on the browser level support for the initial decoding of information and is the same as the browser support for the HTML5 audio element. Therefore the most widely supported media file is the wave (.WAV) format which can be accpeted by every browser supporting the Web Audio API. The next best supported audio only formats are MP3 and AAC (in MP4) which are supported by all major browsers, Firefox relies on OS decoders and therefore its support is predicated by the OS support. The toolbox will work in any browser which supports the Web Audio API, which at point of writing are the major desktop browsers except Microsoft's Internet Explorer, however its newer Edge browser should be supported\footnote{\url{https://msdn.microsoft.com/en-us/library/dn985708.aspx}}. All the collected session data is returned in an XML document structured similarly to the configuration document, where test pages contain the audio elements with their trace collection, results, comments and any other interface-specific data points. \section{Remote tests} % with previous? + \label{sec:remote} If the experimenter is willing to trade some degree of control for a higher number of participants, the test can be hosted on a web server so that participants can take part remotely. This way, a link can be shared widely in the hope of attracting a large amount of subjects, while listening conditions and subject reliability may be less ideal. However, a sound system calibration page and a wide range of metrics logged during the test mitigate these problems. Note also that in some experiments, it may be preferred that the subject has a `real life', familiar listening set-up, for instance when perceived quality differences on everyday sound systems are investigated. Furthermore, a fully browser-based test, where the collection of the results is automatic, is more efficient and technically reliable even when the test still takes place under lab conditions. The following features allow easy and effective remote testing: - \begin{itemize}[noitemsep,nolistsep] - \item PHP script to collect result XML files - \item Randomly pick specified number of audioholders - \item Calibration + \begin{description}[noitemsep,nolistsep] + \item[PHP script to collect result XML files] and store on central server. + \item[Randomly pick a specified number of pages] to ensure an equal and randomised spread of the different pages (`audioHolders') across participants. + \item[Calibration of the sound system (and participant)] by a perceptual pre-test to gather information about the frequency response and speaker configuration - this can be supplemented with a survey. % In theory calibration could be applied anywhere?? - \item Functionality to participate multiple times - \begin{itemize}[noitemsep,nolistsep] - \item Possible to log in with unique ID (no password) - \item Pick `new user' (generates new, unique ID) or `already participated' (need already available ID) - \item Store XML on server with IDs plus which audioholders have already been listened to - \item Don't show `post-test' survey after first time - \item Pick `new' audioholders if available - \item Copy survey information first time to new XMLs - \end{itemize} - \item Intermediate saves - \item Collect Public IP address information for geographic location (by country). - \item Collect Browser and Display information - \end{itemize} + % \item Functionality to participate multiple times + % \begin{itemize}[noitemsep,nolistsep] + % \item Possible to log in with unique ID (no password) + % \item Pick `new user' (generates new, unique ID) or `already participated' (need already available ID) + % \item Store XML on server with IDs plus which audioholders have already been listened to + % \item Don't show `post-test' survey after first time + % \item Pick `new' audioholders if available + % \item Copy survey information first time to new XMLs + % \end{itemize} + \item[Intermediate saves] for tests which were interrupted or unfinished. + \item[Collect IP address information] for geographic location, through PHP function which grabs address and appends to XML file. + \item[Collect Browser and Display information] to the extent it is available and reliable. + \end{description} \section{Interfaces} % title? 'Front end'? % Dave +\label{sec:interfaces} The purpose of this listening test framework is to allow any user the maximum flexibility to design a listening test for their exact application with minimum effort. To this end, a large range of standard listening test interfaces have been implemented. A review of existing listening test frameworks was undertaken and presented in~\Cref{tab:toolboxes}. HULTI-GEN~\cite{hultigen} is a single toolbox that presents the user with a large number of different test interfaces and allows for customisation of each test interface. To provide users with a flexible system, a large range of `standard' listening test interfaces have been implemented, including: % pretty much the same wording as two sentences earlier \begin{itemize}[noitemsep,nolistsep] \item MUSHRA (ITU-R BS. 1534)~\cite{recommendation20031534} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Multiple stimuli are presented and rated on a continuous scale, which includes a reference, hidden reference and hidden anchors. \end{itemize} \item Rank Scale~\cite{pascoe1983evaluation} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Stimuli ranked on single horizontal scale, where they are ordered in preference order. \end{itemize} \item Likert scale~\cite{likert1932technique} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimuli has a five point scale with values: Strongly Agree, Agree, Neutral, Disagree and Strongly Disagree. \end{itemize} \item ABC/HR (ITU-R BS. 1116)~\cite{recommendation19971116} (Mean Opinion Score: MOS) - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimulus has a continuous scale (5-1), labeled as Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying. \end{itemize} \item -50 to 50 Bipolar with Ref - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimulus has a continuous scale -50 to 50 with default values as 0 in middle and a comparison. There is also a provided reference \end{itemize} \item Absolute Category Rating (ACR) Scale~\cite{rec1996p} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimuli has a five point scale with values: Bad, Poor, Fair, Good, Excellent \end{itemize} \item Degredation Category Rating (DCR) Scale~\cite{rec1996p} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimuli has a five point scale with values: (5) Inaudible, (4) Audible but not annoying, (3) slightly annoying, (2) annoying, (1) very annoying. \end{itemize} \item Comparison Category Rating (CCR) Scale~\cite{rec1996p} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimuli has a seven point scale with values: Much Better, Better, Slightly Better, About the same, slightly worse, worse, much worse. There is also a provided reference. \end{itemize} \item 9 Point Hedonic Category Rating Scale~\cite{peryam1952advanced} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimuli has a seven point scale with values: Like Extremely, Like Very Much, Like Moderate, Like Slightly, Neither Like nor Dislike, dislike Extremely, dislike Very Much, dislike Moderate, dislike Slightly. There is also a provided reference. \end{itemize} \item ITU-R 5 Point Continuous Impairment Scale~\cite{rec1997bs} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Each stimuli has a five point scale with values: (5) Imperceptible, (4) Perceptible but not annoying, (3) slightly annoying, (2) annoying, (1) very annoying. There is also a provided reference. \end{itemize} \item Pairwise Comparison (Better/Worse)~\cite{david1963method} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item A reference is provided and ever stimulus is rated as being either better or worse than the reference. \end{itemize} \item APE style \cite{ape} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Multiple stimuli on a single horizontal slider for inter-sample rating. \end{itemize} \item Multi attribute ratings - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Multiple stimuli as points on a 2D plane for inter-sample rating (eg. Valence Arousal) \end{itemize} \item AB Test~\cite{lipshitz1981great} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Two stimuli are presented at a time and the participant has to select a preferred stimulus. \end{itemize} \item ABX Test~\cite{clark1982high} - \begin{itemize} + \begin{itemize}[noitemsep,nolistsep] \item Two stimuli are presented along with a reference and the participant has to select a preferred stimulus, often the closest to the reference. \end{itemize} \end{itemize} While implementing all of these interfaces, it is possible to include any number of references, anchors, hidden references and hidden anchors into all of these listening test formats. - Because of the design choice to separate the core code and interface modules, it is possible for a 3rd party interface to be built with minimal effort. The repository includes documentation on which functions must be called and the specific functions they expect your interface to perform. To this end, there is an 'Interface' object which includes functions for creating the on-page comment boxes (including those with radio or checkbox responses), start and stop buttons with function handles pre-attached and the playhead / transport bars. + Because of the design choice to separate the core code and interface modules, it is possible for a 3rd party interface to be built with minimal effort. The repository includes documentation on which functions must be called and the specific functions they expect your interface to perform. To this end, there is an `Interface' object which includes functions for creating the on-page comment boxes (including those with radio or checkbox responses), start and stop buttons with function handles pre-attached and the playhead / transport bars. %%%% \begin{itemize}[noitemsep,nolistsep] %%%% \item (APE style) \cite{ape} @@ -396,35 +408,38 @@ \end{comment} \section{Analysis and diagnostics} +\label{sec:analysis} % don't mention Python scripts - It would be great to have easy-to-use analysis tools to visualise the collected data and even do science with it. Even better would be to have all this in the browser. Complete perfection would be achieved if and when only limited setup, installation time, and expertise are required for the average non-CS researcher to use this. Tools such as \cite{scale} include analysis features inside their packages as well. - One advantage to web based tests is the ability to process data as it becomes available using server-side programming. Since entire test sessions are uploaded the results can be immediately parsed and the current test results updated, meaning the researcher simply needs to browse to the web page to collect the current test results in a friendly interface rather than downloading the XML files. - - The following functionality is available: - + There are several benefits to providing basic analysis tools in the browser: they allow diagnosing problems, with the interface or with the test subject; they may be sufficient for many researchers' purposes; and test subjects may enjoy seeing an overview of their own results and/or results thus far at the end of their tests. + % \begin{figure*}[bhf] + % \centering + % \includegraphics[width=.7\textwidth]{timeline.pdf} + % \caption{This timeline of a single subject's listening test shows playback of fragments (red segments) and marker movements on the rating axis in function of time. } + % \label{fig:timeline} + % \end{figure*} + For this reason, we include a proof-of-concept web page with: \begin{itemize}[noitemsep,nolistsep] - \item Web page showing all audioholder IDs, file names, subject IDs, audio element IDs, ... in the collected XMLs so far (\texttt{saves/*.xml}) - \item Check/uncheck each of the above for analysis (e.g. zoom in on a certain song, or exclude a subset of subjects) - \item Click a mix to hear it (follow path in XML setup file, which is also embedded in the XML result file) - \item Box plot, confidence plot, scatter plot of values (for a given audioholder) - \item Timeline for a specific subject / song %(see Python scripts), perhaps re-playing the experiment in X times realtime. (If actual realtime, you could replay the audio...) ---> A LOT of work, not sure I can guarantee this one - \item Distribution plots of any radio button and number questions %(drop-down menu with `pretest', `posttest', ...; then drop-down menu with question `IDs' like `gender', `age', ...; make pie chart/histogram of these values over selected range of XMLs) - \item All `comments' on a specific audioelement and export to CSV / XML + \item All audioholder IDs, file names, subject IDs, audio element IDs, ... in the collected XMLs so far (\texttt{saves/*.xml}) + \item Selection of subjects and/or test samples to zoom in on a subset of the data %Check/uncheck each of the above for analysis (e.g. zoom in on a certain song, or exclude a subset of subjects) + \item Embedded audio to hear corresponding test samples % (follow path in XML setup file, which is also embedded in the XML result file) + \item Box plot, confidence plot, and scatter plot of rating values + \item Timeline for a specific subject or song %(see Python scripts), perhaps re-playing the experiment in X times realtime. (If actual realtime, you could replay the audio...) ---> A LOT of work, not sure I can guarantee this one + \item Distribution plots of any radio button and number questions. Also pie charts and histograms when over a range of participants %(drop-down menu with `pretest', `posttest', ...; then drop-down menu with question `IDs' like `gender', `age', ...; make pie chart/histogram of these values over selected range of XMLs) + \item All `comments' on a specific audioelement \item A `download' button for a nice CSV of various things (values, survey responses, comments) %people might want to use for analysis, e.g. when XML scares them %\item Validation of setup XMLs (easily spot `errors', like duplicate IDs or URLs, missing/dangling tags, ...) --> Took this out as a feature as the test_create will already do this as will the test console. \end{itemize} %A subset of the above would already be nice for this paper. + [Some pictures here please.] +\section{Concluding remarks and future work} +\label{sec:conclusion} - [Some pictures here please.] - -\section{Concluding remarks and future work} - - The code and documentation can be pulled or downloaded from \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}. + The code and documentation can be pulled or downloaded from our online repository available at \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}. [Talking a little bit about what else might happen. Unless we really want to wrap this up. ] - \cite{schoeffler2015mushra} gives a 'checklist' for subjective evaluation of audio systems. The Web Audio Evaluation Toolbox meets most of its given requirements including remote testing, crossfading between audio streams, collecting browser information, utilising UI elements and working with various audio formats including uncompressed PCM or WAV format. + \cite{schoeffler2015mushra} gives a `checklist' for subjective evaluation of audio systems. The Web Audio Evaluation Toolbox meets most of its given requirements including remote testing, crossfading between audio streams, collecting browser information, utilising UI elements and working with various audio formats including uncompressed PCM or WAV format. % remote % language support (not explicitly stated) % crossfades