annotate docs/SMC15/smc2015template.tex @ 979:62b638aa1849

Paper: Edit Intro with tool Justification
author Dave Moffat <djmoffat@users.noreply.github.com>
date Fri, 24 Apr 2015 00:11:20 +0100
parents c65d4bd1c660
children ceaaebf9e017
rev   line source
BrechtDeMan@697 1 % -----------------------------------------------
BrechtDeMan@697 2 % Template for SMC 2012
BrechtDeMan@697 3 % adapted from the template for SMC 2011, which was adapted from that of SMC 2010
BrechtDeMan@697 4 % -----------------------------------------------
BrechtDeMan@697 5
BrechtDeMan@697 6 \documentclass{article}
BrechtDeMan@697 7 \usepackage{smc2015}
BrechtDeMan@697 8 \usepackage{times}
BrechtDeMan@697 9 \usepackage{ifpdf}
BrechtDeMan@697 10 \usepackage[english]{babel}
BrechtDeMan@697 11 \usepackage{cite}
BrechtDeMan@697 12
BrechtDeMan@686 13 \hyphenation{Java-script}
BrechtDeMan@686 14
BrechtDeMan@697 15 %%%%%%%%%%%%%%%%%%%%%%%% Some useful packages %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
BrechtDeMan@697 16 %%%%%%%%%%%%%%%%%%%%%%%% See related documentation %%%%%%%%%%%%%%%%%%%%%%%%%%
BrechtDeMan@697 17 %\usepackage{amsmath} % popular packages from Am. Math. Soc. Please use the
BrechtDeMan@697 18 %\usepackage{amssymb} % related math environments (split, subequation, cases,
BrechtDeMan@697 19 %\usepackage{amsfonts}% multline, etc.)
BrechtDeMan@697 20 %\usepackage{bm} % Bold Math package, defines the command \bf{}
BrechtDeMan@697 21 %\usepackage{paralist}% extended list environments
BrechtDeMan@697 22 %%subfig.sty is the modern replacement for subfigure.sty. However, subfig.sty
BrechtDeMan@697 23 %%requires and automatically loads caption.sty which overrides class handling
BrechtDeMan@697 24 %%of captions. To prevent this problem, preload caption.sty with caption=false
BrechtDeMan@697 25 %\usepackage[caption=false]{caption}
BrechtDeMan@697 26 %\usepackage[font=footnotesize]{subfig}
BrechtDeMan@697 27
BrechtDeMan@697 28
BrechtDeMan@697 29 %user defined variables
BrechtDeMan@683 30 \def\papertitle{WEB AUDIO EVALUATION TOOL: A BROWSER-BASED LISTENING TEST ENVIRONMENT} %?
BrechtDeMan@702 31 \def\firstauthor{Nicholas Jillings}
BrechtDeMan@702 32 \def\secondauthor{Brecht De Man}
BrechtDeMan@697 33 \def\thirdauthor{David Moffat}
BrechtDeMan@697 34 \def\fourthauthor{Joshua D. Reiss}
BrechtDeMan@697 35
BrechtDeMan@697 36 % adds the automatic
BrechtDeMan@697 37 % Saves a lot of ouptut space in PDF... after conversion with the distiller
BrechtDeMan@697 38 % Delete if you cannot get PS fonts working on your system.
BrechtDeMan@697 39
BrechtDeMan@697 40 % pdf-tex settings: detect automatically if run by latex or pdflatex
BrechtDeMan@697 41 \newif\ifpdf
BrechtDeMan@697 42 \ifx\pdfoutput\relax
BrechtDeMan@697 43 \else
BrechtDeMan@697 44 \ifcase\pdfoutput
BrechtDeMan@697 45 \pdffalse
BrechtDeMan@697 46 \else
BrechtDeMan@697 47 \pdftrue
BrechtDeMan@697 48 \fi
BrechtDeMan@697 49
BrechtDeMan@697 50 \ifpdf % compiling with pdflatex
BrechtDeMan@697 51 \usepackage[pdftex,
BrechtDeMan@697 52 pdftitle={\papertitle},
BrechtDeMan@697 53 pdfauthor={\firstauthor, \secondauthor, \thirdauthor},
BrechtDeMan@697 54 bookmarksnumbered, % use section numbers with bookmarks
BrechtDeMan@697 55 pdfstartview=XYZ % start with zoom=100% instead of full screen;
BrechtDeMan@697 56 % especially useful if working with a big screen :-)
BrechtDeMan@697 57 ]{hyperref}
BrechtDeMan@697 58 %\pdfcompresslevel=9
BrechtDeMan@697 59
BrechtDeMan@697 60 \usepackage[pdftex]{graphicx}
BrechtDeMan@697 61 % declare the path(s) where your graphic files are and their extensions so
BrechtDeMan@697 62 %you won't have to specify these with every instance of \includegraphics
BrechtDeMan@697 63 \graphicspath{{./figures/}}
BrechtDeMan@697 64 \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
BrechtDeMan@697 65
BrechtDeMan@697 66 \usepackage[figure,table]{hypcap}
BrechtDeMan@697 67
BrechtDeMan@697 68 \else % compiling with latex
BrechtDeMan@697 69 \usepackage[dvips,
BrechtDeMan@697 70 bookmarksnumbered, % use section numbers with bookmarks
BrechtDeMan@697 71 pdfstartview=XYZ % start with zoom=100% instead of full screen
BrechtDeMan@697 72 ]{hyperref} % hyperrefs are active in the pdf file after conversion
BrechtDeMan@697 73
BrechtDeMan@697 74 \usepackage[dvips]{epsfig,graphicx}
BrechtDeMan@697 75 % declare the path(s) where your graphic files are and their extensions so
BrechtDeMan@697 76 %you won't have to specify these with every instance of \includegraphics
BrechtDeMan@697 77 \graphicspath{{./figures/}}
BrechtDeMan@697 78 \DeclareGraphicsExtensions{.eps}
BrechtDeMan@697 79
BrechtDeMan@697 80 \usepackage[figure,table]{hypcap}
BrechtDeMan@697 81 \fi
BrechtDeMan@697 82
BrechtDeMan@697 83 %setup the hyperref package - make the links black without a surrounding frame
BrechtDeMan@697 84 \hypersetup{
BrechtDeMan@697 85 colorlinks,%
BrechtDeMan@697 86 citecolor=black,%
BrechtDeMan@697 87 filecolor=black,%
BrechtDeMan@697 88 linkcolor=black,%
BrechtDeMan@697 89 urlcolor=black
BrechtDeMan@697 90 }
BrechtDeMan@697 91
BrechtDeMan@697 92
BrechtDeMan@697 93 % Title.
BrechtDeMan@697 94 % ------
BrechtDeMan@697 95 \title{\papertitle}
BrechtDeMan@697 96
BrechtDeMan@697 97 % Authors
BrechtDeMan@697 98 % Please note that submissions are NOT anonymous, therefore
BrechtDeMan@697 99 % authors' names have to be VISIBLE in your manuscript.
BrechtDeMan@697 100 %
BrechtDeMan@697 101 % Single address
BrechtDeMan@697 102 % To use with only one author or several with the same address
BrechtDeMan@697 103 % ---------------
BrechtDeMan@697 104 %\oneauthor
BrechtDeMan@697 105 % {\firstauthor} {Affiliation1 \\ %
BrechtDeMan@697 106 % {\tt \href{mailto:author1@smcnetwork.org}{author1@smcnetwork.org}}}
BrechtDeMan@697 107
BrechtDeMan@697 108 %Two addresses
BrechtDeMan@697 109 %--------------
BrechtDeMan@697 110 % \twoauthors
BrechtDeMan@697 111 % {\firstauthor} {Affiliation1 \\ %
BrechtDeMan@697 112 % {\tt \href{mailto:author1@smcnetwork.org}{author1@smcnetwork.org}}}
BrechtDeMan@697 113 % {\secondauthor} {Affiliation2 \\ %
BrechtDeMan@697 114 % {\tt \href{mailto:author2@smcnetwork.org}{author2@smcnetwork.org}}}
BrechtDeMan@697 115
BrechtDeMan@702 116
BrechtDeMan@702 117
BrechtDeMan@702 118 % FIX!!!
BrechtDeMan@697 119 \fourauthors
BrechtDeMan@697 120 {\firstauthor} {%Affiliation1 \\
BrechtDeMan@702 121 {\tt \href{mailto:b.deman@qmul.ac.uk}{n.g.r.jillings@se14.qmul.ac.uk, }}}
BrechtDeMan@697 122 {\secondauthor} {%Affiliation2\\ %
BrechtDeMan@702 123 {\tt \href{mailto:n.g.r.jillings@se14.qmul.ac.uk}{\{b.deman,}}}
BrechtDeMan@697 124 {\thirdauthor} {%Affiliation3\\ %
BrechtDeMan@702 125 {\tt \href{mailto:d.j.moffat@qmul.ac.uk}{d.j.moffat, }}}
BrechtDeMan@697 126 {\fourthauthor} {%Affiliation4\\ %
BrechtDeMan@702 127 {\tt \href{mailto:joshua.reiss@qmul.ac.uk}{joshua.reiss\}@qmul.ac.uk}}}
BrechtDeMan@697 128
BrechtDeMan@697 129 % ***************************************** the document starts here ***************
BrechtDeMan@697 130 \begin{document}
BrechtDeMan@697 131 %
BrechtDeMan@697 132 \capstartfalse
BrechtDeMan@697 133 \maketitle
BrechtDeMan@697 134 \capstarttrue
BrechtDeMan@697 135 %
BrechtDeMan@697 136 \begin{abstract}
BrechtDeMan@683 137 New functionality in HTML5, notably its Web Audio API, allow for increasingly powerful applications in the browser. % is this true?
BrechtDeMan@683 138 Perceptual evaluation tests for audio, where the subject assesses certain qualities of different audio fragments through a graphical user interface and/or text boxes, require playback of audio and rapid switching between different files. % what else?
BrechtDeMan@683 139 The advantage of a web application is easy deployment on any platform, without requiring any other application or library, easy storing of results on a server.
BrechtDeMan@683 140 [...]
BrechtDeMan@683 141 %Place your abstract at the top left column on the first page.
BrechtDeMan@683 142 %Please write about 150-200 words that specifically highlight the purpose of your work,
BrechtDeMan@683 143 %its context, and provide a brief synopsis of your results.
BrechtDeMan@683 144 %Avoid equations in this part.\\
BrechtDeMan@683 145
BrechtDeMan@697 146 \end{abstract}
BrechtDeMan@714 147
BrechtDeMan@714 148 % TOTAL PAPER: Minimum 4 pages, 6 preferred, max. 8 (6 for demos/posters)\\
BrechtDeMan@697 149
BrechtDeMan@697 150 \section{Introduction}\label{sec:introduction}
BrechtDeMan@697 151
nicholas@690 152 %NICK: examples of what kind of audio applications HTML5 has made possible, with references to publications (or website)\\
nicholas@690 153
BrechtDeMan@975 154 Perceptual evaluation of audio plays an important role in a wide range of research including audio effect design \cite{}, sound synthesis \cite{de2013real}, source separation \cite{}, codec design \cite{}, general perception of audio \cite{schoeffler2013impact}, and many others \cite{friberg2011comparison}. \\ % EXPAND
BrechtDeMan@975 155
BrechtDeMan@714 156 The Web Audio API is a high-level JavaScript API designed for real-time processing audio inside the browser through various processing nodes \cite{webaudioapi}. Various web sites have used the Web Audio API for either creative purposes, such as drum machines and score creation tools \cite{webaudiodemo}, %http://webaudio.github.io/demo-list/
nicholas@690 157 others from the list show real-time captured audio processing such as room reverberation tools and a phase vocoder from the system microphone. The BBC Radiophonic Workshop shows effects used on famous TV shows such as Doctor Who, being simulated inside the browser \cite{bbcradiophonics}. %http://webaudio.prototyping.bbc.co.uk/
BrechtDeMan@975 158 Another example is the BBC R\&D automatic compressor which applies a dynamic range compressor on a radio station which dynamically adjusts the compressor settings to match the listener's environment \cite{mason2015compression}.\\ % The paper for this has not been released yet by AES...
BrechtDeMan@714 159
BrechtDeMan@975 160 This work is based in part on the APE audio perceptual evaluation interface for MATLAB \cite{deman2014b}. An important drawback of this toolbox is the need to have MATLAB to create a test and even to run (barring the use of an executable generated by MATLAB), and limited compatibility with both earlier and newer versions of MATLAB, which makes it hard to maintain. On the other hand, a web application generally has the advantage of running in most browsers on most applications, and we present a tool that requires no specialised software or even programming knowledge to set up.
BrechtDeMan@973 161
BrechtDeMan@978 162 [TO ADD: other interfaces for perceptual evaluation of audio, browser-based or not!] % IMPORTANT
BrechtDeMan@978 163
BrechtDeMan@978 164 We present a browser-based perceptual evaluation tool from which any kind of multiple stimulus audio evaluation tool where subjects need to rank, rate, select, or comment on different audio samples can be built.
BrechtDeMan@978 165 We also include an example of the multiple stimulus user interface included with the APE tool \cite{deman2014b}, which presents the subject with a number of axes on which a number of markers, corresponding with audio samples, can be moved to reflect any subjective quality, as well as corresponding comment boxes.
BrechtDeMan@978 166 However, other graphical user interfaces can be put on top of the engine that we provide with minimal or no modifications. Examples of this are the MUSHRA test \cite{mushra}, single or multiple stimulus evaluation with a two-dimensional interface (such as valence and arousal dimensions), or simple annotation (using free-form text, check boxes, radio buttons or drop-down menus) of one or more audio samples at a time.
BrechtDeMan@978 167 In case of for instance method of adjustment, where the audio is processed by the user \cite{bech}, or of AB test, where the interface does not show all audio samples to be evaluated at once, the back end of the tool needs to be modified as well.
BrechtDeMan@978 168
djmoffat@979 169 There are a number of advantages to building a web audio based listening test environment. The ability to easily deploy a flexible and scalable testing environment that requires no proprietary software to run makes the web audio evaluation tool a very flexible testing tool. The ability to host a single test server and create multiple clients not only allows multiple participants to be involved in a trial simultaneously, but also permits participants to be located anywhere in the world. There are limited user experience issues, as all users should have some experience with using existing web technologies.
BrechtDeMan@973 170
djmoffat@979 171
djmoffat@979 172 %DAVE: justify:
djmoffat@979 173 %\begin{itemize}
djmoffat@979 174 %\item Easy deployment - no propitiatory software required
djmoffat@979 175 %\item Entirely scalable
djmoffat@979 176 %\item Trial multiple participants simultaneously
djmoffat@979 177 %\item Intuitive environment - everyone uses the internet
djmoffat@979 178 %\item Can be hosted and run from single location with participants worldwide
djmoffat@979 179 %\end{itemize}
djmoffat@974 180
BrechtDeMan@978 181 [TO ADD: in the following sections, we describe the interface in more detail, discuss the implementation, and go over design considerations and flexibility ... ]
BrechtDeMan@702 182
BrechtDeMan@702 183
BrechtDeMan@975 184 \section{Interface}\label{sec:interface}
BrechtDeMan@702 185
BrechtDeMan@978 186 At this point, we have implemented the interface of the MATLAB-based APE Perceptual Evaluation for Audio toolbox \cite{deman2014b}, which shows one marker for each simultaneously evaluated audio fragment on one or more horizontal axes, that can be moved to rate or rank the respective fragments in terms of any subjective quality, as well as a comment box for every marker, and any extra text boxes for extra comments. See \ref{fig:interface} for an example of the interface, with eleven fragments and one axis. %? change if a new interface is shown
BrechtDeMan@978 187
BrechtDeMan@978 188 However, the back end of this test environment allows for many more established and novel interfaces for listening tests, particularly ones where the subject only assesses audio without manipulating it (i.e. method of adjustment, which would require additional features to be implemented).
BrechtDeMan@655 189
BrechtDeMan@973 190 \begin{figure*}[htbp]
BrechtDeMan@973 191 \begin{center}
BrechtDeMan@973 192 \includegraphics[width=0.9\textwidth]{interface.png}
djmoffat@974 193 \caption{Example of interface, with 1 axis and 11 fragments}
BrechtDeMan@973 194 \label{fig:interface}
BrechtDeMan@973 195 \end{center}
BrechtDeMan@973 196 \end{figure*}
BrechtDeMan@975 197 [DAVE to make new picture]
BrechtDeMan@655 198
BrechtDeMan@975 199 [BRECHT to expand]
BrechtDeMan@683 200
n@704 201
BrechtDeMan@975 202
BrechtDeMan@975 203 \section{Architecture}\label{sec:architecture} % or implementation?
BrechtDeMan@975 204
nicholas@971 205 The tool runs entirely inside the browser through the new HTML5 Web Audio API and is supported by most major web browsers (with the exception of Internet Explorer). The API allows for constructing audio processing elements and connecting them together to produce a high quality, real time signal process to manipulate audio streams. The API supports multi-channel processing and has an accurate playback timer for precise scheduled playback control. The API is controlled through the browser JavaScript engine and is therefore highly configurable. Processing is all performed in a separate, low latency thread to the main JavaScript thread, meaning there is no blocking due to real time processing.
n@704 206
n@704 207 The web tool itself is split into several files to operate:
n@704 208 \begin{itemize}
nicholas@971 209 \item \texttt{index.html}: The main index file to load the scripts, this is the file the browser must request to load.
nicholas@971 210 \item \texttt{core.js}: Contains functions and objects to define the audio objects, audio playback engine and loading media files
BrechtDeMan@683 211 \item \texttt{ape.js}: Parses setup files to create the interface as instructed, following the same style chain as the MATLAB APE Tool \cite{deman2014b}.
n@704 212 \end{itemize}
n@704 213
nicholas@690 214 The HTML file loads the \texttt{core.js} file along with a few other ancillary files (such as the jQuery JavaScript extensions)% should we cite jQuery.... https://jquery.com/
nicholas@971 215 , at which point the browser JavaScript begins to execute the on-page instructions, which gives the URL of the test setup XML document (outlined in Section \ref{sec:setupresultsformats}). \texttt{core.js} parses this document and executes the function in \texttt{ape.js} to build the web page with the given audio files. The reason for separating these two files is to allow for further interface designs (such as MUSHRA \cite{mushra} or A-B tests \cite{bech}) to be used, which would still require the same underlying core functions outlined in \texttt{core.js}, see also Section \ref{sec:interface}.
n@704 216
nicholas@971 217 The \texttt{ape.js} file has several main functions but the most important are \textit{loadInterface(xmlDoc)}, \textit{loadTest(id)}, \textit{pageXMLSave(testId)} and \textit{interfaceXMLSave()}. \textit{loadInterface(xmlDoc)} is called to decode the supplied project document in respect for the interface specified and define any global structures (such as the slider interface). It also identifies the number of pages in the test and randomises the order, if specified to do so. This is the only madatory function in any of the interface files as this is called by \texttt{core.js} when the document is ready. The design style is such that \texttt{core.js} cannot 'see' any interface specific functions and therefore cannot assume any are available. Therefore the \textit{loadInterface(xmlDoc)} is very important to setup the entire test environment. It can be assumed that the interface files can `see' the \texttt{core.js} file and can therefore not only interact with it, but also modify it.
BrechtDeMan@978 218
nicholas@971 219 Each test page is loaded using \textit{loadTest(id)} which performs two major tasks: to populate the interface with the slider elements and comment boxes; and secondly to instruct the \textit{audioEngine} to load the audio fragments and construct the backend audio graph. The markers on the slider at the top of the page are positioned randomly, to minimise the bias that may be introduced when the initial positions are near the beginning, end or middle of the slider. While another approach is to place the markers outside of the slider bar at first and have the subject drag them in, the authors believe this doesn't encourage careful consideration and comparison of the different fragments as the implicit goal of the test becomes to audition and drag each fragment in just once, rather than to compare all fragments rigorously.
BrechtDeMan@978 220
BrechtDeMan@978 221 \textit{loadTest(id)} in \texttt{ape.js} also instructs the audio engine in \texttt{core.js} to create the \textit{audioObject} These are custom audio nodes, one representing each audio element specified in each page.
nicholas@971 222 They consist of a \textit{bufferSourceNode} (a node which holds a buffer of audio samples for playback) and a \textit{gainNode}. There are various functions applied depending on metric collection which record the interaction with the audio element. These nodes are then connected to the \textit{audioEngine} (itself a custom web audio node) containing a \textit{gainNode} (where the various \textit{audioObject} connect to) for summation before passing the output to the \textit{destinationNode}, a permanent fixed node of the Web Audio API created as the master output where the browser then passes the audio information to the system sound device. % Does this now make sense?
nicholas@971 223 % audio object/audioObject/Audio Object: -- should always be audioObject if talking about the JavaScript object, otherwise should say audio element or audio fragment.
BrechtDeMan@978 224
nicholas@971 225 When an \textit{audioObject} is created, it is given the URL of the audio sample to load. This is downloaded into the browser asynchronously using the \textit{XMLHttpRequest} object. This downloads any file into the JavaScript environment for further processing which is particularly useful for the Web Audio API because it supports downloading of files in their binary form for decoding. Once downloaded the file is decoded using the Web Audio API offline decoder. This uses the browser available decoding schemes to decode the audio files into raw float32 arrays, which are in turn passed to the relevant \textit{audioObject} for playback.
nicholas@971 226
nicholas@971 227 Once each page of the test is completed, identified by pressing the Submit button, the \textit{pageXMLSave(testId)} is called to store all of the collected data until all pages of the test are completed. After the final test and any post-test questions are completed, the \textit{interfaceXMLSave()} function is called. This function generates the final XML file for submission as outlined in Section \ref{sec:setupresultsformats}.
nicholas@971 228
nicholas@971 229 Browsers support various audio file formats and are not consistent in any format. Currently the Web Audio API is best supported in Chrome, Firefox, Opera and Safari. All of these support the use of the uncompressed WAV format. Although not a compact, web friendly format, most transport systems are of a high enough bandwidth this should not be a problem. Ogg Vorbis is another well supported format across the 4 supported major desktop browsers, as well as MP3 (although Firefox may not support all MP3 types) \cite{mozdevSupportedMedia}. %https://developer.mozilla.org/en-US/docs/Web/HTML/Supported_media_formats
nicholas@971 230 One issue of the Web Audio API is that the sample rate is assigned by the system sound device, rather than requested and does not have the ability to request a different one. % Does this make sense? The problem is across all audio files.
nicholas@971 231 Therefore, the default operation when an audio file is loaded with a different sample rate to that of the system is to convert the sample rate. To provide a check for this, the desired sample rate can be supplied with the setup XML and checked against. If the sample rates do not match, a browser alert window is shown asking for the sample rate to be correctly adjusted.
BrechtDeMan@978 232 As this happens before any loading or decoding of audio files, the system will only fetch files as soon as the system's sample rate meets any requirements, avoiding requests for large files until they are actually needed.
BrechtDeMan@978 233
BrechtDeMan@978 234 %During playback, the playback nodes loop indefinitely until playback is stopped. The gain nodes in the \textit{audioObject}s enable dynamic muting of nodes. When a bar in the sliding ranking is clicked, the audio engine mutes all \textit{audioObject}s and un-mutes the clicked one. Therefore, if the audio samples are perfectly aligned up and of the same sample length, they will remain perfectly aligned with each other.
BrechtDeMan@978 235 % Don't think this is relevant anymore
BrechtDeMan@978 236
BrechtDeMan@978 237 \section{Setup and results formats}\label{sec:setupresultsformats}
BrechtDeMan@978 238
BrechtDeMan@978 239 [DAVE: somewhere add input file]
BrechtDeMan@978 240
BrechtDeMan@978 241 Setup and the results both use the common XML document format to outline the various parameters. The setup file determines which interface to use, the location of audio files, how many pages and other general setup rules to define the testing envrionment. Having one document to modify allows for quick manipulation in a `human readable' form to create new tests, or adjust current ones, without needing to edit multiple web files. % I mean the .js and .html files, though not sure if any better.
BrechtDeMan@978 242 The setup document has several defined nodes and structure which are documented with the source code. For example there is a section for general setup options where the pre-test and post-test questions and statements are defined:
BrechtDeMan@978 243
BrechtDeMan@978 244 \texttt{<question id="location" mandatory="true"> Please enter your listening location \\ </question>}
BrechtDeMan@978 245
BrechtDeMan@978 246 From the above example it can be seen that a question box should be generated, with the id 'location' and it is mandatory to answer. The question is in the PreTest node meaning it will appear before any testing will begin. When the result for the entire test is shown, then this will appear in the PreTest node of the response with the id 'location' allowing it to be found easily. This outlines the importance of having clear and meaningful ID values. Pre- and post-test dialog boxes allow for comments or questions to be presented before or after the test, to convey listening test instructions, gather information about the subject, listening environment, and overall experience of the test.
BrechtDeMan@978 247
BrechtDeMan@978 248 [BRECHT to say we try to cater to as many audiences as possible, simple yet tweakable]
BrechtDeMan@978 249
BrechtDeMan@978 250 Further options in the setup file are:
BrechtDeMan@978 251
BrechtDeMan@978 252 \begin{itemize}
BrechtDeMan@978 253 \item \textbf{Snap to corresponding position}: When this is enabled, and a fragment is playing, the playhead skips to the same position in the next fragment that is clicked. If it is not enabled, every fragment is played from the start.
BrechtDeMan@978 254 \item \textbf{Loop fragments}: Repeat current fragment when end is reached, until the `Stop audio' or `Submit' button is clicked.
BrechtDeMan@978 255 \item \textbf{Comments}: Displays a separate comment box for each fragment in the page.
BrechtDeMan@978 256 \item \textbf{General comment}: One comment box, additional to the individual comment boxes, to comment on the test or a feature that some or all of the fragments share.
BrechtDeMan@978 257 \item \textbf{Resampling}: When this is enabled, tracks are resampled to match the subject's system's sample rate (a default feature of the Web Audio API). When it is not, an error is shown when the system does not match the requested sample rate.
BrechtDeMan@978 258 \item \textbf{Randomise page order}: Randomises the order in which different `pages' are presented. % are we calling this 'pages'?
BrechtDeMan@978 259 \item \textbf{Randomise fragment order}: Randomises the order and numbering of the markers and comment boxes corresponding with the fragments. This permutation is stored as well, to be able to interpret references to the numbers in the comments (such as `this is much [brighter] then 4').
BrechtDeMan@978 260 \item \textbf{Require playback}: Require that each fragment has been played at least once, if not in full.
BrechtDeMan@978 261 \item \textbf{Require full playback}: If `Require playback' is active, require that each fragment has been played in full.
BrechtDeMan@978 262 \item \textbf{Require moving}: Require that each marker is moved (dragged) at least once.
BrechtDeMan@978 263 \item \textbf{Require comments}: This option allows requiring the subject to require a comment for each track.
nicholas@971 264 \item \textbf{Repeat test}: Number of times test should be repeated (none by default), to allow familiarisation with the content and experiment, and to investigate consistency of user and variability due to familiarity. In the setup, each 'page' can be given a repeat count. These are all gathered before shuffling the order so repeated tests are not back-to-back if possible.
BrechtDeMan@978 265 \item \textbf{Returning to previous pages}: Indicates whether it is possible to go back to a previous `page' in the test.
BrechtDeMan@978 266 \item \textbf{Lowest rating below [value]}: To enforce a certain use of the rating scale, it can be required to rate at least one sample below a specified value.
BrechtDeMan@978 267 \item \textbf{Highest rating above [value]}: To enforce a certain use of the rating scale, it can be required to rate at least one sample above a specified value.
BrechtDeMan@978 268 \item \textbf{Reference}: Allows for a separate sample (outside of the axis) to be the `reference', which the subject can play back during the test to help with the task at hand \cite{mushra}.
BrechtDeMan@978 269 \item \textbf{Hidden reference}: Whether or not an explicit `reference' is provided, the `hidden reference' should be rated above a certain value \cite{mushra} - this can be enforced.
BrechtDeMan@978 270 \item \textbf{Hidden anchor}: The `hidden anchor' should be rated lower than a certain value \cite{mushra} - this can be enforced.
BrechtDeMan@978 271
BrechtDeMan@978 272 \end{itemize}
BrechtDeMan@978 273
BrechtDeMan@978 274 When one of these options is not included in the setup file, they assume a default value.
BrechtDeMan@978 275
BrechtDeMan@978 276 % loop, snap to corresponding position, comments, 'general' comment, require same sampling rate, different types of randomisation
BrechtDeMan@978 277
BrechtDeMan@978 278 The results file is dynamically generated by the interface upon clicking the `Submit' button. This also executes checks, depending on the setup file, to ensure that all tracks have been played back, rated and commented on. The XML output returned contains a node per audioObject and contains both the corresponding marker's position and any comments written in the associated comment box. The rating returned is normalised to be a value between 0 and 1, normalising the pixel representation of different browser windows.
BrechtDeMan@978 279
BrechtDeMan@978 280 The results will also contain information collected by any defined pre/post questions. These are referenced against the setup XML by using the same ID so readable responses can be obtained. Taking from the earlier example of setting up a pre-test question, an example reponse would be shown as the following.
BrechtDeMan@978 281
BrechtDeMan@978 282 \texttt{<comment id="location"> Queen Mary \\University of London </comment>}
BrechtDeMan@978 283
BrechtDeMan@978 284 Each page of testing is returned with the results of the entire page included in the structure. One `audioElement' node is created per audio fragment per page, along with its ID. This includes several child nodes including the value holding the rating between 0 and 1, and any metrics collected. These include how long the element was listened for, the initial position, boolean flags if the element was listened to, if the element was moved and if the element comment box had any comment. Furthermore, each user action (manipulation of any interface element, such as playback or moving a marker) is logged along with a the corresponding time code and stored or sent along with the results.
BrechtDeMan@978 285
BrechtDeMan@978 286 Future development will also evolve to include any session data, such as the browser the tool was used in. Currently the results files are downloaded on the user side of the browser as a .xml file to be manually returned. However the end goal is to allow the XML files to be submitted over the web to a receiving server to store them, allowing for automated collection.
BrechtDeMan@978 287
BrechtDeMan@978 288 % right?
BrechtDeMan@978 289
BrechtDeMan@978 290 %Here is an example of the setup XML and the results XML: % perhaps best to refer to each XML after each section (setup <> results)
BrechtDeMan@978 291 % Should we include an Example of the input and output XML structure?? --> Sure.
BrechtDeMan@978 292
BrechtDeMan@978 293 An example of the returned \textit{audioElement} node in the results XML file is as follows.
BrechtDeMan@978 294
BrechtDeMan@978 295 \texttt{<audioelement id="8"> \\
BrechtDeMan@978 296 <comment> \\
BrechtDeMan@978 297 <question>Comment on track 0</question> \\
BrechtDeMan@978 298 <response> The drums were punchy </response> \\
BrechtDeMan@978 299 </comment> \\
BrechtDeMan@978 300 <value> 0.25169491525423726 </value> \\
BrechtDeMan@978 301 <metric> \\
BrechtDeMan@978 302 <metricresult id="elementTimer"> \\ 2.3278004535147385< /metricresult> \\
BrechtDeMan@978 303 <metricresult id="elementTrackerFull"> \\
BrechtDeMan@978 304 <timepos id="0"> \\
BrechtDeMan@978 305 <time>1.7937414965986385</time> \\
BrechtDeMan@978 306 <position>0.41694915254237286</position> \\
BrechtDeMan@978 307 </timepos> \\
BrechtDeMan@978 308 <timepos id="1"> \\
BrechtDeMan@978 309 <time>2.6993197278911563</time> \\
BrechtDeMan@978 310 <position>0.45847457627118643</position> \\
BrechtDeMan@978 311 </timepos> \\</metricresult> \\
BrechtDeMan@978 312 <metricresult id="elementInitialPosition"> 0.47796610169491527 </metricresult> \\
BrechtDeMan@978 313 <metricresult id="elementFlagListenedTo"> true< /metricresult> \\
BrechtDeMan@978 314 <metricresult id="elementFlagMoved"> true </metricresult> \\
BrechtDeMan@978 315 </metric> \\
BrechtDeMan@978 316 </audioelement>}
BrechtDeMan@978 317
BrechtDeMan@978 318 As can be seen, the parent tag \texttt{audioelement} holds the id of the element passed in from the setup document. The first child element is \texttt{comment} and holds both the question shown and the response from the comment box inside.
BrechtDeMan@978 319 The child element \texttt{value} holds the normalised ranking value. Next comes the metric node structure, there is one \texttt{metricresult} node per metric event collected. The id of the node identifies the type of data it contains. For example, the first holds the id \textit{elementTimer} and the data contained represents how long, in seconds, the audio element was listened to. The next holds the id \textit{elementTrackerFull} and contains a pair of elements per entry. This represents the entire movement of the elements' slider giving the time the event took place in seconds from when the current test page started, and the new position. In our example there are three \texttt{timepos} children with their id representing their order. There is one of these \texttt{audioelement} tags per audio element outlined on each test page.
BrechtDeMan@978 320
BrechtDeMan@978 321 \section{Conclusions and future work}\label{sec:conclusions}
BrechtDeMan@978 322
BrechtDeMan@978 323 In this paper we have presented an approach to creating a browser-based listening test environment that can be used for a variety of types of perceptual evaluation of audio.
BrechtDeMan@978 324 Specifically, we discussed the use of the toolbox in the context of assessment of preference for different production practices, with identical source material.
BrechtDeMan@978 325 The purpose of this paper is to outline the design of this tool, to describe our implementation using basic HTML5 functionality, and to discuss design challenges and limitations of our approach. % or something
BrechtDeMan@978 326
BrechtDeMan@978 327 % future work
BrechtDeMan@978 328 Further work may include the development of other common test designs, such as MUSHRA \cite{mushra}, AB, ABX and method of adjustment tests.
BrechtDeMan@978 329 In addition, [...].
BrechtDeMan@978 330 [BRECHT to correct and expand]
BrechtDeMan@978 331
BrechtDeMan@978 332 from community
BrechtDeMan@978 333
BrechtDeMan@978 334 %\begin{itemize}
BrechtDeMan@978 335 %\item Options for MUSHRA style experiment with vertical slide per track
BrechtDeMan@978 336 %\item APE style experiment where all tracks are on a single horizontal axis % isn't that what we're doing now?
BrechtDeMan@978 337 %\item AB test
BrechtDeMan@978 338 %\item ABX test
BrechtDeMan@978 339 %\item Method of adjustment tests
BrechtDeMan@978 340 %\end{itemize}
BrechtDeMan@978 341
BrechtDeMan@978 342
BrechtDeMan@978 343 The source code of this tool can be found on \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}.
BrechtDeMan@978 344
BrechtDeMan@978 345
BrechtDeMan@978 346 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
BrechtDeMan@978 347 %bibliography here
BrechtDeMan@978 348 \bibliography{smc2015template}
BrechtDeMan@978 349
BrechtDeMan@978 350
BrechtDeMan@978 351
BrechtDeMan@978 352 \end{document}
BrechtDeMan@978 353
BrechtDeMan@978 354
BrechtDeMan@978 355
BrechtDeMan@978 356 % RUBBISH
BrechtDeMan@978 357
BrechtDeMan@978 358 %\subsection{Equations}
BrechtDeMan@978 359 %Equations of importance,
BrechtDeMan@978 360 %or to which you refer later,
BrechtDeMan@978 361 %should be placed on separated lines and numbered.
BrechtDeMan@978 362 %The number should be on the right side, in parentheses.
BrechtDeMan@978 363 %\begin{equation}
BrechtDeMan@978 364 %E=mc^{2+\delta}.
BrechtDeMan@978 365 %\label{eq:Emc2}
BrechtDeMan@978 366 %\end{equation}
BrechtDeMan@978 367 %Refer to equations like so:
BrechtDeMan@978 368 %As (\ref{eq:Emc2}) shows,
BrechtDeMan@978 369 %I do not completely trust Special Relativity.
BrechtDeMan@978 370 %
BrechtDeMan@978 371 %\subsection{Figures, Tables and Captions}
BrechtDeMan@978 372 %\begin{table}[t]
BrechtDeMan@978 373 % \begin{center}
BrechtDeMan@978 374 % \begin{tabular}{|l|l|}
BrechtDeMan@978 375 % \hline
BrechtDeMan@978 376 % String value & Numeric value \\
BrechtDeMan@978 377 % \hline
BrechtDeMan@978 378 % Hej SMC & 2015 \\
BrechtDeMan@978 379 % \hline
BrechtDeMan@978 380 % \end{tabular}
BrechtDeMan@978 381 %\end{center}
BrechtDeMan@978 382 % \caption{Table captions should be placed below the table, exactly like this,
BrechtDeMan@978 383 % but using words different from these.}
BrechtDeMan@978 384 % \label{tab:example}
BrechtDeMan@978 385 %\end{table}
BrechtDeMan@978 386
BrechtDeMan@978 387 %\begin{figure}[t]
BrechtDeMan@978 388 %\figbox{
BrechtDeMan@978 389 %\subfloat[][]{\includegraphics[width=60mm]{figure}\label{fig:subfigex_a}}\\
BrechtDeMan@978 390 %\subfloat[][]{\includegraphics[width=80mm]{figure}\label{fig:subfigex_b}}
BrechtDeMan@978 391 %}
BrechtDeMan@978 392 %\caption{Here's an example using the subfig package.\label{fig:subfigex} }
BrechtDeMan@978 393 %\end{figure}
BrechtDeMan@978 394
BrechtDeMan@978 395
BrechtDeMan@978 396
BrechtDeMan@978 397
BrechtDeMan@978 398
BrechtDeMan@978 399 %\begin{acknowledgments}
BrechtDeMan@978 400 %You may acknowledge people, projects,
BrechtDeMan@978 401 %funding agencies, etc.
BrechtDeMan@978 402 %which can be included after the second-level heading
BrechtDeMan@978 403 %``Acknowledgments'' (with no numbering).
BrechtDeMan@978 404 %\end{acknowledgments}
BrechtDeMan@978 405