annotate docs/WAC2016/WAC2016.tex @ 820:7b0ce3a9ddc1

Merge from branch "WAC2016"
author Nicholas Jillings <n.g.r.jillings@se14.qmul.ac.uk>
date Mon, 23 Nov 2015 09:13:12 +0000
parents
children 888292c88c33
rev   line source
n@820 1 \documentclass{sig-alternate}
n@820 2 \usepackage{hyperref} % make links (like references, links to Sections, ...) clickable
n@820 3 \usepackage{enumitem} % tighten itemize etc by appending '[noitemsep,nolistsep]'
n@820 4 \usepackage{cleveref}
n@820 5
n@820 6 \graphicspath{{img/}} % put the images in this folder
n@820 7
n@820 8 \begin{document}
n@820 9
n@820 10 % Copyright
n@820 11 \setcopyright{waclicense}
n@820 12
n@820 13 \newcommand*\rot{\rotatebox{90}}
n@820 14
n@820 15
n@820 16 %% DOI
n@820 17 %\doi{10.475/123_4}
n@820 18 %
n@820 19 %% ISBN
n@820 20 %\isbn{123-4567-24-567/08/06}
n@820 21 %
n@820 22 %%Conference
n@820 23 %\conferenceinfo{PLDI '13}{June 16--19, 2013, Seattle, WA, USA}
n@820 24 %
n@820 25 %\acmPrice{\$15.00}
n@820 26
n@820 27 %
n@820 28 % --- Author Metadata here ---
n@820 29 \conferenceinfo{Web Audio Conference WAC-2016,}{April 4--6, 2016, Atlanta, USA}
n@820 30 \CopyrightYear{2016} % Allows default copyright year (20XX) to be over-ridden - IF NEED BE.
n@820 31 %\crdata{0-12345-67-8/90/01} % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
n@820 32 % --- End of Author Metadata ---
n@820 33
n@820 34 \title{Web Audio Evaluation Tool: A framework for subjective assessment of audio}
n@820 35 %\subtitle{[Extended Abstract]
n@820 36 %\titlenote{A full version of this paper is available as
n@820 37 %\textit{Author's Guide to Preparing ACM SIG Proceedings Using
n@820 38 %\LaTeX$2_\epsilon$\ and BibTeX} at
n@820 39 %\texttt{www.acm.org/eaddress.htm}}}
n@820 40 %
n@820 41 % You need the command \numberofauthors to handle the 'placement
n@820 42 % and alignment' of the authors beneath the title.
n@820 43 %
n@820 44 % For aesthetic reasons, we recommend 'three authors at a time'
n@820 45 % i.e. three 'name/affiliation blocks' be placed beneath the title.
n@820 46 %
n@820 47 % NOTE: You are NOT restricted in how many 'rows' of
n@820 48 % "name/affiliations" may appear. We just ask that you restrict
n@820 49 % the number of 'columns' to three.
n@820 50 %
n@820 51 % Because of the available 'opening page real-estate'
n@820 52 % we ask you to refrain from putting more than six authors
n@820 53 % (two rows with three columns) beneath the article title.
n@820 54 % More than six makes the first-page appear very cluttered indeed.
n@820 55 %
n@820 56 % Use the \alignauthor commands to handle the names
n@820 57 % and affiliations for an 'aesthetic maximum' of six authors.
n@820 58 % Add names, affiliations, addresses for
n@820 59 % the seventh etc. author(s) as the argument for the
n@820 60 % \additionalauthors command.
n@820 61 % These 'additional authors' will be output/set for you
n@820 62 % without further effort on your part as the last section in
n@820 63 % the body of your article BEFORE References or any Appendices.
n@820 64
n@820 65 % FIVE authors instead of four, to leave space between first two authors.
n@820 66 \numberofauthors{5} % in this sample file, there are a *total*
n@820 67 % of EIGHT authors. SIX appear on the 'first-page' (for formatting
n@820 68 % reasons) and the remaining two appear in the \additionalauthors section.
n@820 69 %
n@820 70 \author{
n@820 71 % You can go ahead and credit any number of authors here,
n@820 72 % e.g. one 'row of three' or two rows (consisting of one row of three
n@820 73 % and a second row of one, two or three).
n@820 74 %
n@820 75 % The command \alignauthor (no curly braces needed) should
n@820 76 % precede each author name, affiliation/snail-mail address and
n@820 77 % e-mail address. Additionally, tag each line of
n@820 78 % affiliation/address with \affaddr, and tag the
n@820 79 % e-mail address with \email.
n@820 80 %
n@820 81 % 1st. author
n@820 82 \alignauthor Nicholas Jillings\\
n@820 83 \email{n.g.r.jillings@se14.qmul.ac.uk}
n@820 84 % dummy author for nicer spacing
n@820 85 \alignauthor
n@820 86 % 2nd. author
n@820 87 \alignauthor Brecht De Man\\
n@820 88 \email{b.deman@qmul.ac.uk}
n@820 89 \and % use '\and' if you need 'another row' of author names
n@820 90 % 3rd. author
n@820 91 \alignauthor David Moffat\\
n@820 92 \email{d.j.moffat@qmul.ac.uk}
n@820 93 % 4th. author
n@820 94 \alignauthor Joshua D. Reiss\\
n@820 95 \email{joshua.reiss@qmul.ac.uk}
n@820 96 \and % new line for address
n@820 97 \affaddr{Centre for Digital Music, School of Electronic Engineering and Computer Science}\\
n@820 98 \affaddr{Queen Mary University of London}\\
n@820 99 \affaddr{Mile End Road,}
n@820 100 \affaddr{London E1 4NS}\\
n@820 101 \affaddr{United Kingdom}\\
n@820 102 }
n@820 103 %Centre for Digital Music, School of Electronic Engineering and Computer Science, Queen Mary University of London
n@820 104 %% 5th. author
n@820 105 %\alignauthor Sean Fogarty\\
n@820 106 % \affaddr{NASA Ames Research Center}\\
n@820 107 % \affaddr{Moffett Field}\\
n@820 108 % \email{fogartys@amesres.org}
n@820 109 %% 6th. author
n@820 110 %\alignauthor Charles Palmer\\
n@820 111 % \affaddr{Palmer Research Laboratories}\\
n@820 112 % \affaddr{8600 Datapoint Drive}\\
n@820 113 % \email{cpalmer@prl.com}
n@820 114 %}
n@820 115 % There's nothing stopping you putting the seventh, eighth, etc.
n@820 116 % author on the opening page (as the 'third row') but we ask,
n@820 117 % for aesthetic reasons that you place these 'additional authors'
n@820 118 % in the \additional authors block, viz.
n@820 119 %\additionalauthors{Additional authors: John Smith (The Th{\o}rv{\"a}ld Group,
n@820 120 %email: {\texttt{jsmith@affiliation.org}}) and Julius P.~Kumquat
n@820 121 %(The Kumquat Consortium, email: {\texttt{jpkumquat@consortium.net}}).}
n@820 122 \date{1 October 2015}
n@820 123 % Just remember to make sure that the TOTAL number of authors
n@820 124 % is the number that will appear on the first page PLUS the
n@820 125 % number that will appear in the \additionalauthors section.
n@820 126
n@820 127 \maketitle
n@820 128 \begin{abstract}
n@820 129
n@820 130 Perceptual listening tests are commonplace in audio research and a vital form of evaluation. Many tools exist to run such tests, however many operate one test type and are therefore limited whilst most require proprietary software. Using Web Audio the Web Audio Evaluation Tool (WAET) addresses these concerns by having one toolbox which can be configured to run many different tests, perform it through a web browser and without needing proprietary software or computer programming knowledge. In this paper the role of the Web Audio API in giving WAET key functionalities are shown. The paper also highlights less common features, available to web based tools, such as easy remote testing environment and in-browser analytics.
n@820 131
n@820 132 \end{abstract}
n@820 133
n@820 134
n@820 135 \section{Introduction}
n@820 136
n@820 137 % Listening tests/perceptual audio evaluation: what are they, why are they important
n@820 138 % As opposed to limited scope of WAC15 paper: also musical features, realism of sound effects / sound synthesis, performance of source separation and other algorithms...
n@820 139 Perceptual evaluation of audio, in the form of listening tests, is a powerful way to assess anything from audio codec quality to realism of sound synthesis to the performance of source separation, automated music production and other auditory evaluations.
n@820 140 In less technical areas, the framework of a listening test can be used to measure emotional response to music or test cognitive abilities.
n@820 141 % maybe some references? If there's space.
n@820 142
n@820 143 % check out http://link.springer.com/article/10.1007/s10055-015-0270-8 - only paper that cited WAC15 paper
n@820 144
n@820 145 % Why difficult? Challenges? What constitutes a good interface?
n@820 146 % Technical, interfaces, user friendliness, reliability
n@820 147 Several applications for performing perceptual listening tests currently exist. A review of existing listening test frameworks was undertaken and presented in~\Cref{tab:toolboxes}. Note that many rely on proprietary, 3rd party software such as MATLAB and MAX, making them less attractive for many. With the exception of the existing JavaScript-based toolboxes, remote deployment (web-based test hosting and result collection) is not possible.
n@820 148
n@820 149 HULTI-GEN~\cite{hultigen} is a single example of a toolbox that presents the user with a large number of different test interfaces and allows for customisation of each test interface, without requiring knowledge of any programming language. The Web Audio Evaluation Toolbox (WAET), presented here, stands out as it does not require proprietary software or a specific platform. It also provides a wide range of interface and test types in one user friendly environment. Furthermore any test based on the default test types can be configured in the browser as well. Note that the design of an effective listening test further poses many challenges unrelated to interface design, which are beyond the scope of this paper \cite{bech}.
n@820 150
n@820 151 % Why in the browser?
n@820 152 The Web Audio API provides important features including sample level manipulation of audio streams \cite{schoeffler2015mushra} and synchronous and flexible playback. Being in the browser allows leveraging the flexible object oriented JavaScript language and native support for web documents, such as the extensible markup language (XML) which is used for configuration and test result files. Using the web also reduces deployment requirements to a basic web server with extra functionality, such as test collection and automatic processing, using PHP. As recruiting participants can be very time-consuming, and as for some tests a large number of participants is needed, browser-based tests can enable participants in multiple locations to perform the test \cite{schoeffler2015mushra}.
n@820 153
n@820 154 Both BeaqleJS \cite{beaqlejs} and mushraJS\footnote{https://github.com/akaroice/mushraJS} also operate in the browser. However, BeaqleJS does not make use of the Web Audio API and therefore lacks arbitrary manipulation of audio stream samples, and neither offer an adequately wide choice of test designs for them to be useful to many researchers. %requires programming knowledge?...
n@820 155
n@820 156 % only browser-based?
n@820 157 \begin{table*}[ht]
n@820 158 \caption{Table with existing listening test platforms and their features}
n@820 159 \small
n@820 160 \begin{center}
n@820 161 \begin{tabular}{|*{9}{l|}}
n@820 162 \hline
n@820 163 \textbf{Toolbox} & \rot{\textbf{APE}} & \rot{\textbf{BeaqleJS}} &\rot{\textbf{HULTI-GEN}} & \rot{\textbf{mushraJS}} & \rot{\textbf{MUSHRAM}} & \rot{\textbf{Scale}} & \rot{\textbf{WhisPER}} & \rot{\textbf{WAET}} \\ \hline
n@820 164 \textbf{Reference} & \cite{ape} & \cite{beaqlejs} & \cite{hultigen} & & \cite{mushram} & \cite{scale} & \cite{whisper} & \cite{waet} \\ \hline
n@820 165 \textbf{Language} & MATLAB & JS & MAX & JS & MATLAB & MATLAB & MATLAB & JS \\ \hline
n@820 166 \textbf{Remote} & & (\checkmark) & & \checkmark & & & & \checkmark \\ \hline \hline
n@820 167 MUSHRA (ITU-R BS. 1534) & & \checkmark & \checkmark & \checkmark & \checkmark & & & \checkmark \\ \hline
n@820 168 APE & \checkmark & & & & & & & \checkmark \\ \hline
n@820 169 Rank Scale & & & \checkmark & & & & & \checkmark \\ \hline
n@820 170 Likert Scale & & & \checkmark & & & & \checkmark & \checkmark \\ \hline
n@820 171 ABC/HR (ITU-R BS. 1116) & & & \checkmark & & & & & \checkmark \\ \hline
n@820 172 -50 to 50 Bipolar with ref. & & & \checkmark & & & & & \checkmark \\ \hline
n@820 173 Absolute Category Rating Scale & & & \checkmark & & & & & \checkmark \\ \hline
n@820 174 Degradation Category Rating Scale & & & \checkmark & & & & & \checkmark \\ \hline
n@820 175 Comparison Category Rating Scale & & & \checkmark & & & & \checkmark & \checkmark \\ \hline
n@820 176 9 Point Hedonic Category Rating Scale & & & \checkmark & & & & \checkmark & \checkmark \\ \hline
n@820 177 ITU-R 5 Continuous Impairment Scale & & & \checkmark & & & & & \checkmark \\ \hline
n@820 178 Pairwise / AB Test & & & \checkmark & & & & & \checkmark \\ \hline
n@820 179 Multi-attribute ratings & & & \checkmark & & & & & \checkmark \\ \hline
n@820 180 ABX Test & & \checkmark & \checkmark & & & & & \checkmark \\ \hline
n@820 181 Adaptive psychophysical methods & & & & & & & \checkmark & \\ \hline
n@820 182 Repertory Grid Technique & & & & & & & \checkmark & \\ \hline
n@820 183 Semantic Differential & & & & & & \checkmark & \checkmark &\checkmark \\ \hline
n@820 184 n-Alternative Forced Choice & & & & & & \checkmark & & \\ \hline
n@820 185 \end{tabular}
n@820 186 \end{center}
n@820 187 \label{tab:toolboxes}
n@820 188 \end{table*}
n@820 189 %
n@820 190 %Selling points: remote tests, visualisaton, create your own test in the browser, many interfaces, few/no dependencies, flexibility
n@820 191
n@820 192 %[Talking about what we do in the various sections of this paper. Referring to \cite{waet}. ]
n@820 193 To meet the need for a cross-platform, versatile and easy-to-use listening test tool, we previously developed the Web Audio Evaluation Tool \cite{waet} which at the time of its inception was capable of running a listening test in the browser from an XML configuration file, and storing an XML file as well, with one particular interface. This has now expanded into a tool with which a wide range of listening test types can easily be constructed and set up remotely, without any need for manually altering code or configuration files, and allows visualisation of the collected results in the browser. In this paper, we discuss these different aspects and explore which future improvements would be possible.
n@820 194
n@820 195 \begin{figure}[tb]
n@820 196 \centering
n@820 197 \includegraphics[width=.5\textwidth]{interface.png}
n@820 198 \caption{A simple example of a multi-stimulus, single attribute, single rating scale test with a reference and comment fields.}
n@820 199 \label{fig:interface}
n@820 200 \end{figure}
n@820 201
n@820 202 \begin{comment}
n@820 203 % MEETING 8 OCTOBER
n@820 204 \subsection{Meeting 8 October}
n@820 205 \begin{itemize}
n@820 206 \item Do we manipulate audio?\\
n@820 207 \begin{itemize}
n@820 208 \item Add loudness equalisation? (test\_create.html) Tag with gains.
n@820 209 \item Add volume slider?
n@820 210 \item Cross-fade (in interface node): default 0, number of seconds
n@820 211 \item Also: we use the playback buffer to present metrics of which portion is listened to
n@820 212 \end{itemize}
n@820 213 \item Logging system information: whichever are possible (justify others)
n@820 214 \item Input streams as audioelements
n@820 215 \item Capture microphone to estimate loudness (especially Macbook)
n@820 216 \item Test page (in-built oscillators): left-right calibration, ramp up test tone until you hear it; optional compensating EQ (future work implementing own filters) --> Highlight issues!
n@820 217 \item Record IP address (PHP function, grab and append to XML file)
n@820 218 \item Expand anchor/reference options
n@820 219 \item AB / ABX
n@820 220 \end{itemize}
n@820 221
n@820 222 \subsubsection{Issues}
n@820 223 \begin{itemize}
n@820 224 \item Filters not consistent (Nick to test across browsers)
n@820 225 \item Playback audiobuffers need to be destroyed and rebuilt each time
n@820 226 \item Can't get channel data, hardware input/output...
n@820 227 \end{itemize}
n@820 228 \end{comment}
n@820 229
n@820 230 \section{Architecture} % title? 'back end'? % NICK
n@820 231 \label{sec:architecture}
n@820 232 %A slightly technical overview of the system. Talk about XML, JavaScript, Web Audio API, HTML5.
n@820 233
n@820 234 Although WAET uses a sparse subset of the Web Audio API functionality, its performance comes directly from it. Listening tests can convey large amounts of information other than obtaining the perceptual relationship between the audio fragments. With WAET it is possible to track which parts of the audio fragments were listened to and when, at what point in the audio stream the participant switched to a different fragment, and how a fragment's rating was adjusted over time within a session, to name a few. Not only does this allow evaluation of a wealth of perceptual aspects, but it also helps detect poor participants whose results are potentially not representative.
n@820 235
n@820 236 One of the key initial design parameters for WAET was to make the tool as open as possible to non-programmers and to this end all of the user modifiable options are included in a single XML document. This document is the specification document and can be designed either by manually writing the XML (or modifying an existing document or template) or using the included test creator. These standalone HTML pages do not require any server or internet connection and help a build the specification document. The first (test\_create.html) is for simple tests and operates step-by-step to guide the user through a drag and drop, clutter free interface. The advanced version is for more complex tests. Both models support automatic verification to ensure the XML file is valid and will highlight areas which are either incorrect and would cause an error, or options which should be removed as they are blank.
n@820 237
n@820 238 The basic test creator, Figure \ref{fig:test_create}, utilises the Web Audio API to perform quick playback checks and also allows for loudness normalisation techniques inspired from \cite{ape}. These are calculated offline by accessing the raw audio samples exposed from the buffer before being applied to the audio element as a gain attribute. Therefore the tool performs loudness normalisation without editing any audio files. Equally the gain attribute can be modified in either editor using an HTML5 slider or number box respectively.
n@820 239 \begin{comment}
n@820 240 \begin{figure}[h!]
n@820 241 \centering
n@820 242 \includegraphics[width=.45\textwidth]{test_create_2.png}
n@820 243 \caption{Screen-shot of test creator tool using drag and drop to create specification document}
n@820 244 \label{fig:test_create}
n@820 245 \end{figure}
n@820 246 \end{comment}
n@820 247
n@820 248 %Describe and/or visualise audioholder-audioelement-... structure.
n@820 249 The specification document contains the URL of the audio fragments for each test page. These fragments are downloaded asynchronously in the test and decoded offline by the Web Audio offline decoder. The resulting buffers are assigned to a custom Audio Objects node which tracks the fragment buffer, the playback \textit{bufferSourceNode}, other specification attributes including its unique test ID, the interface object(s) associated with the fragment and any metric or data collection objects. The Audio Object is controlled by an over-arching custom Audio Context node (not to be confused with the Web Audio Context). This parent JS Node allows for session wide control of the Audio Objects including starting and stopping playback of specific nodes.
n@820 250
n@820 251 The only issue with this model is the \textit{bufferNode} in the Web Audio API, implemented in the standard as a `use once' object. Once this has been played, the node must be discarded as it cannot be instructed to play the same \textit{bufferSourceNode} again. Therefore on each play request the buffer object must be created and then linked with the stored \textit{bufferSourceNode}. This is an odd behaviour for such a simple object which has no alternative except to use the HTML5 audio element. However, they do not have the ability to synchronously start on a given time and therefore not suited.
n@820 252
n@820 253 In the test, each buffer node is connected to a gain node which will operate at the level determined by the specification document. Therefore it is possible to perform a `Method of Adjustment' test where an interface could directly manipulate these gain nodes. These gain nodes are used for cross-fading between samples when operating in synchronous playback. Cross-fading can either be fade-out fade-in or a true cross-fade. There is also an optional `Master Volume' slider which can be shown on the test GUI. This slider modifies a gain node before the destination node. This slider can also be monitored and therefore its data tracked providing extra validation. This is not indicative of the final volume exiting the speakers and therefore its use should only be considered in a lab environment to ensure proper usage.
n@820 254
n@820 255 %Which type of files? WAV, anything else? Perhaps not exhaustive list, but say something along the lines of 'whatever browser supports'. Compatability?
n@820 256 The media files supported depend on the browser level support for the initial decoding of information and is the same as the browser support for the HTML5 audio element. The most widely supported media file is the wave (.WAV) format which is accepted by every browser supporting the Web Audio API. The toolbox will work in any browser which supports the Web Audio API.
n@820 257
n@820 258 All the collected session data is returned in an XML document structured similarly to the configuration document, where test pages contain the audio elements with their trace collection, results, comments and any other interface-specific data points.
n@820 259
n@820 260 \section{Remote tests} % with previous?
n@820 261 \label{sec:remote}
n@820 262
n@820 263 If the experimenter is willing to trade some degree of control for a higher number of participants, the test can be hosted on a public web server so that participants can take part remotely. This way, a link can be shared widely in the hope of attracting a large amount of subjects, while listening conditions and subject reliability may be less ideal. However, a sound system calibration page and a wide range of metrics logged during the test mitigate these problems. In some experiments, it may be preferred that the subject has a `real life', familiar listening set-up, for instance when perceived quality differences on everyday sound systems are investigated.
n@820 264 Furthermore, a fully browser-based test, where the collection of the results is automatic, is more efficient and technically reliable even when the test still takes place under lab conditions.
n@820 265
n@820 266 The following features allow easy and effective remote testing:
n@820 267 \begin{description}[noitemsep,nolistsep]
n@820 268 \item[PHP script to collect result XML files] and store on central server.
n@820 269 \item[Randomly pick a specified number of pages] to ensure an equal and randomised spread of the different pages (`audioHolders') across participants.
n@820 270 \item[Calibration of the sound system (and participant)] by a perceptual pre-test to gather information about the frequency response and speaker configuration - this can be supplemented with a survey.
n@820 271 % In theory calibration could be applied anywhere??
n@820 272 % \item Functionality to participate multiple times
n@820 273 % \begin{itemize}[noitemsep,nolistsep]
n@820 274 % \item Possible to log in with unique ID (no password)
n@820 275 % \item Pick `new user' (generates new, unique ID) or `already participated' (need already available ID)
n@820 276 % \item Store XML on server with IDs plus which audioholders have already been listened to
n@820 277 % \item Don't show `post-test' survey after first time
n@820 278 % \item Pick `new' audioholders if available
n@820 279 % \item Copy survey information first time to new XMLs
n@820 280 % \end{itemize}
n@820 281 \item[Intermediate saves] for tests which were interrupted or unfinished.
n@820 282 \item[Collect IP address information] for geographic location, through PHP function which grabs address and appends to XML file.
n@820 283 \item[Collect Browser and Display information] to the extent it is available and reliable.
n@820 284 \end{description}
n@820 285
n@820 286
n@820 287 \section{Interfaces} % title? 'Front end'? % Dave
n@820 288 \label{sec:interfaces}
n@820 289
n@820 290 The purpose of this listening test framework is to allow any user the maximum flexibility to design a listening test for their exact application with minimum effort. To this end, a large range of standard listening test interfaces have been implemented.
n@820 291
n@820 292 To provide users with a flexible system, a large range of `standard' listening test interfaces have been implemented, including: % pretty much the same wording as two sentences earlier
n@820 293 \begin{itemize}[noitemsep,nolistsep]
n@820 294 \item MUSHRA (ITU-R BS. 1534)~\cite{recommendation20031534}
n@820 295 \begin{comment}
n@820 296 \begin{itemize}[noitemsep,nolistsep]
n@820 297 \item Multiple stimuli are presented and rated on a continuous scale, which includes a reference, hidden reference and hidden anchors.
n@820 298 \end{itemize}
n@820 299 \end{comment}
n@820 300 \item Rank Scale~\cite{pascoe1983evaluation}: stimuli ranked on single horizontal scale, where they are ordered in preference order.
n@820 301 \item Likert scale~\cite{likert1932technique}: each stimuli has a five point scale with values: Strongly Agree, Agree, Neutral, Disagree and Strongly Disagree.
n@820 302 \item ABC/HR (ITU-R BS. 1116)~\cite{recommendation19971116} (Mean Opinion Score: MOS): each stimulus has a continuous scale (5-1), labeled as Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying.
n@820 303 \item -50 to 50 Bipolar with Ref: each stimulus has a continuous scale -50 to 50 with default values as 0 in middle and a reference.
n@820 304 \item Absolute Category Rating (ACR) Scale~\cite{rec1996p}: Likert but labels are Bad, Poor, Fair, Good, Excellent
n@820 305 \item Degredation Category Rating (DCR) Scale~\cite{rec1996p}: ABC \& Likert but labels are (5) Inaudible, (4) Audible but not annoying, (3) slightly annoying, (2) annoying, (1) very annoying.
n@820 306 \item Comparison Category Rating (CCR) Scale~\cite{rec1996p}: ACR \& DCR but 7 point scale: Much Better, Better, Slightly Better, About the same, slightly worse, worse, much worse. There is also a provided reference.
n@820 307 \item 9 Point Hedonic Category Rating Scale~\cite{peryam1952advanced}: each stimuli has a seven point scale with values: Like Extremely, Like Very Much, Like Moderate, Like Slightly, Neither Like nor Dislike, dislike Extremely, dislike Very Much, dislike Moderate, dislike Slightly. There is also a provided reference.
n@820 308 \item ITU-R 5 Point Continuous Impairment Scale~\cite{rec1997bs}: Same as ABC/HR but with a reference.
n@820 309 \item Pairwise Comparison (Better/Worse)~\cite{david1963method}: every stimulus is rated as being either better or worse than the reference.
n@820 310 \item APE style \cite{ape}: Multiple stimuli as points on a 2D plane for inter-sample rating (eg. Valence Arousal)
n@820 311 \item AB Test~\cite{lipshitz1981great}: Two stimuli presented at a time, participant selects a preferred stimulus.
n@820 312 \item ABX Test~\cite{clark1982high}: Two stimuli are presented along with a reference and the participant has to select a preferred stimulus, often the closest to the reference.
n@820 313 \end{itemize}
n@820 314
n@820 315 It is possible to include any number of references, anchors, hidden references and hidden anchors into all of these listening test formats.
n@820 316
n@820 317 Because of the design to separate the core code and interface modules, it is possible for a 3rd party interface to be built with minimal effort. The repository includes documentation on which functions must be called and the specific functions they expect your interface to perform. The core includes an `Interface' object which includes object prototypes for the on-page comment boxes (including those with radio or checkbox responses), start and stop buttons and the playhead / transport bars.
n@820 318
n@820 319 %%%% \begin{itemize}[noitemsep,nolistsep]
n@820 320 %%%% \item (APE style) \cite{ape}
n@820 321 %%%% \item Multi attribute ratings
n@820 322 %%%% \item MUSHRA (ITU-R BS. 1534)~\cite{recommendation20031534}
n@820 323 %%%% \item Interval Scale~\cite{zacharov1999round}
n@820 324 %%%% \item Rank Scale~\cite{pascoe1983evaluation}
n@820 325 %%%%
n@820 326 %%%% \item 2D Plane rating - e.g. Valence vs. Arousal~\cite{carroll1969individual}
n@820 327 %%%% \item Likert scale~\cite{likert1932technique}
n@820 328 %%%%
n@820 329 %%%% \item {\bf All the following are the interfaces available in HULTI-GEN~\cite{hultigen} }
n@820 330 %%%% \item ABC/HR (ITU-R BS. 1116)~\cite{recommendation19971116}
n@820 331 %%%% \begin{itemize}
n@820 332 %%%% \item Continuous Scale (5-1) Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying. (default Inaudible?)
n@820 333 %%%% \end{itemize}
n@820 334 %%%% \item -50 to 50 Bipolar with Ref
n@820 335 %%%% \begin{itemize}
n@820 336 %%%% \item Scale -50 to 50 on Mushra with default values as 0 in middle and a comparison ``Reference'' to compare to 0 value
n@820 337 %%%% \end{itemize}
n@820 338 %%%% \item Absolute Category Rating (ACR) Scale~\cite{rec1996p}
n@820 339 %%%% \begin{itemize}
n@820 340 %%%% \item 5 point Scale - Bad, Poor, Fair, Good, Excellent (Default fair?)
n@820 341 %%%% \end{itemize}
n@820 342 %%%% \item Degredation Category Rating (DCR) Scale~\cite{rec1996p}
n@820 343 %%%% \begin{itemize}
n@820 344 %%%% \item 5 point Scale - Inaudible, Audible but not annoying, slightly annoying, annoying, very annoying. (default Inaudible?) - {\it Basically just quantised ABC/HR?}
n@820 345 %%%% \end{itemize}
n@820 346 %%%% \item Comparison Category Rating (CCR) Scale~\cite{rec1996p}
n@820 347 %%%% \begin{itemize}
n@820 348 %%%% \item 7 point scale: Much Better, Better, Slightly Better, About the same, slightly worse, worse, much worse - Default about the same with reference to compare to
n@820 349 %%%% \end{itemize}
n@820 350 %%%% \item 9 Point Hedonic Category Rating Scale~\cite{peryam1952advanced}
n@820 351 %%%% \begin{itemize}
n@820 352 %%%% \item 9 point scale: Like Extremely, Like Very Much, Like Moderate, Like Slightly, Neither Like nor Dislike, dislike Extremely, dislike Very Much, dislike Moderate, dislike Slightly - Default Neither Like nor Dislike with reference to compare to
n@820 353 %%%% \end{itemize}
n@820 354 %%%% \item ITU-R 5 Point Continuous Impairment Scale~\cite{rec1997bs}
n@820 355 %%%% \begin{itemize}
n@820 356 %%%% \item 5 point Scale (5-1) Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying. (default Inaudible?)- {\it Basically just quantised ABC/HR, or Different named DCR}
n@820 357 %%%% \end{itemize}
n@820 358 %%%% \item Pairwise Comparison (Better/Worse)~\cite{david1963method}
n@820 359 %%%% \begin{itemize}
n@820 360 %%%% \item 2 point Scale - Better or Worse - (not sure how to default this - they default everything to better, which is an interesting choice)
n@820 361 %%%% \end{itemize}
n@820 362 %%%% \end{itemize}
n@820 363
n@820 364 % Build your own test
n@820 365
n@820 366 \begin{comment}
n@820 367 { \bf A screenshot would be nice.
n@820 368
n@820 369 Established tests (see below) included as `presets' in the build-your-own-test page. }
n@820 370 \end{comment}
n@820 371
n@820 372 \section{Analysis and diagnostics}
n@820 373 \label{sec:analysis}
n@820 374 % don't mention Python scripts
n@820 375 There are several benefits to providing basic analysis tools in the browser: they allow diagnosing problems, with the interface or with the test subject; they may be sufficient for many researchers' purposes; and test subjects may enjoy seeing an overview of their own results and/or results thus far at the end of their tests.
n@820 376 \begin{figure}[bhf]
n@820 377 \centering
n@820 378 \includegraphics[width=.5\textwidth]{boxplot.png}
n@820 379 %\caption{This timeline of a single subject's listening test shows playback of fragments (red segments) and marker movements on the rating axis in function of time. }
n@820 380 \caption{Box and whisker plot showing the aggregated numerical ratings of six stimuli by a group of subjects.}
n@820 381 \label{fig:timeline}
n@820 382 \end{figure}
n@820 383 For this reason, we include a proof-of-concept web page with:
n@820 384 \begin{itemize}[noitemsep,nolistsep]
n@820 385 \item All audioholder IDs, file names, subject IDs, audio element IDs, ... in the collected XMLs so far (\texttt{saves/*.xml})
n@820 386 \item Selection of subjects and/or test samples to zoom in on a subset of the data %Check/uncheck each of the above for analysis (e.g. zoom in on a certain song, or exclude a subset of subjects)
n@820 387 \item Embedded audio to hear corresponding test samples % (follow path in XML setup file, which is also embedded in the XML result file)
n@820 388 \item Scatter plot, confidence plot and box plot of rating values (see Figure )
n@820 389 \item Timeline for a specific subject %(see Figure \ref{fig:timeline})%, perhaps re-playing the experiment in X times realtime. (If actual realtime, you could replay the audio...)
n@820 390 \item Distribution plots of any radio button and number questions in pre- and post-test survey %(drop-down menu with `pretest', `posttest', ...; then drop-down menu with question `IDs' like `gender', `age', ...; make pie chart/histogram of these values over selected range of XMLs)
n@820 391 \item All `comments' on a specific audioelement
n@820 392 \item A `download' function for a CSV of ratings, survey responses and comments% various things (values, survey responses, comments) people might want to use for analysis, e.g. when XML scares them
n@820 393 %\item Validation of setup XMLs (easily spot `errors', like duplicate IDs or URLs, missing/dangling tags, ...)
n@820 394 \end{itemize}
n@820 395
n@820 396
n@820 397 %A subset of the above would already be nice for this paper.
n@820 398 \section{Concluding remarks and future work}
n@820 399 \label{sec:conclusion}
n@820 400
n@820 401 We have developed a browser-based tool for the design and deployment of listening tests, essentially requiring no programming experience and third party software. Following the predictions or guidelines in \cite{schoeffler2015mushra}, it supports remote testing, cross-fading between audio streams, collecting information about the system, among others.
n@820 402
n@820 403 Whereas many other types of interfaces do exist, we felt that supporting e.g. a range of `method of adjustment' tests would be beyond the scope of a tool that aims to be versatile enough while not claiming to support any custom experiment one might want to set up. Rather, it supports any non-adaptive listening test up to multi-stimulus, multi-attribute evaluation including references, anchors, text boxes, radio buttons and/or checkboxes, with arbitrary placement of the various UI elements.
n@820 404
n@820 405 The code and documentation can be pulled or downloaded from our online repository available at \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}.
n@820 406 % remote
n@820 407 % language support (not explicitly stated)
n@820 408 % crossfades
n@820 409 % choosing speakers/sound device from within browser? --- NOT POSSIBLE, can only determine channel output counts and its up to the hardware to determine
n@820 410 % collect information about software and sound system
n@820 411 % buttons, scales, ... UI elements
n@820 412 % must be able to load uncompressed PCM
n@820 413
n@820 414 %
n@820 415 % The following two commands are all you need in the
n@820 416 % initial runs of your .tex file to
n@820 417 % produce the bibliography for the citations in your paper.
n@820 418 \bibliographystyle{ieeetr}
n@820 419 \small
n@820 420 \bibliography{WAC2016} % sigproc.bib is the name of the Bibliography in this case
n@820 421 % You must have a proper ".bib" file
n@820 422 % and remember to run:
n@820 423 % latex bibtex latex latex
n@820 424 % to resolve all references
n@820 425 %
n@820 426 % ACM needs 'a single self-contained file'!
n@820 427 %
n@820 428 \end{document}