Mercurial > hg > webaudioevaluationtool

--- a/docs/SMC15/smc2015template.bbl	Fri Apr 24 02:02:58 2015 +0100
+++ b/docs/SMC15/smc2015template.bbl	Fri Apr 24 11:54:45 2015 +0100
@@ -21,16 +21,48 @@
 \providecommand{\BIBdecl}{\relax}
 \BIBdecl

-\bibitem{de2013real}
-A.~de~G{\"o}tzen, E.~Sikstr{\"o}m, F.~Grani, and S.~Serafin, ``Real, foley or
-  synthetic? an evaluation of everyday walking sounds,'' in \emph{Proceedings
-  of SMC 2013 : 10th Sound and Music Computing Conference}, 2013.
-
 \bibitem{schoeffler2013impact}
 M.~Schoeffler and J.~Herre, ``About the impact of audio quality on overall
   listening experience,'' in \emph{Proceedings of Sound and Music Computing
   Conference}, 2013, pp. 48--53.

+\bibitem{repp}
+R.~Repp, ``Recording quality ratings by music professionals,'' in
+  \emph{Proceedings of the 2006 International Computer Music Conference}, 2006,
+  pp. pp. 468--474.
+
+\bibitem{de2013real}
+A.~de~G{\"o}tzen, E.~Sikstr{\"o}m, F.~Grani, and S.~Serafin, ``Real, foley or
+  synthetic? {A}n evaluation of everyday walking sounds,'' in \emph{Proceedings
+  of SMC 2013 : 10th Sound and Music Computing Conference}, 2013.
+
+\bibitem{durr2015implementation}
+G.~Durr, L.~Peixoto, M.~Souza, R.~Tanoue, and J.~D. Reiss, ``Implementation and
+  evaluation of dynamic level of audio detail,'' in \emph{Audio Engineering
+  Society Conference: 56th International Conference: Audio for Games}, February
+  2015.
+
+\bibitem{deman2014a}
+B.~De~Man and J.~D. Reiss, ``Adaptive control of amplitude distortion
+  effects,'' in \emph{53rd Conference of the Audio Engineering Society},
+  January 2014.
+
+\bibitem{uhlereiss}
+J.~D. Reiss and C.~Uhle, ``Determined source separation for microphone
+  recordings using iir filters,'' in \emph{129th Convention of the Audio
+  Engineering Society}.\hskip 1em plus 0.5em minus 0.4em\relax Audio
+  Engineering Society, November 2010.
+
+\bibitem{song2013b}
+Y.~Song, S.~Dixon, and A.~R. Halpern, ``Do online social tags predict perceived
+  or induced emotional responses to music?'' in \emph{Proceedings of the 14th
+  International Society for Music Information Retrieval (ISMIR2013)}, 2013.
+
+\bibitem{song2013a}
+Y.~Song, S.~Dixon, M.~T. Pearce, and G.~Fazekas, ``Using tags to select stimuli
+  in the study of music and emotion,'' \emph{Proceedings of the 3rd
+  International Conference on Music \& Emotion (ICME)}, 2013.
+
 \bibitem{friberg2011comparison}
 A.~Friberg and A.~Hedblad, ``A comparison of perceptual ratings and computed
   audio features,'' in \emph{Proceedings of the SMC2011, 8th Sound and Music
@@ -53,13 +85,32 @@
 A.~Mason, N.~Jillings, Z.~Ma, J.~D. Reiss, and F.~Melchior, ``Adaptive audio
   reproduction using personalized compression,'' in \emph{Audio Engineering
   Society Conference: 57th International Conference: The Future of Audio
-  Entertainment Technology -- Cinema, Television and the Internet}, Mar 2015.
+  Entertainment Technology -- Cinema, Television and the Internet}, March 2015.

 \bibitem{deman2014b}
 B.~De~Man and J.~D. Reiss, ``{APE}: {A}udio {P}erceptual {E}valuation toolbox
   for {MATLAB},'' in \emph{136th Convention of the Audio Engineering Society},
   April 2014.

+\bibitem{mushram}
+E.~Vincent, M.~G. Jafari, and M.~D. Plumbley, ``Preliminary guidelines for
+  subjective evalutation of audio source separation algorithms,'' in \emph{UK
+  ICA Research Network Workshop}, 2006.
+
+\bibitem{whisper}
+S.~Ciba, A.~Wlodarski, and H.-J. Maempel, ``Whis{PER} -- a new tool for
+  performing listening tests,'' in \emph{126th Convention of the Audio
+  Engineering Society}, May 7-10 2009.
+
+\bibitem{scale}
+A.~V. Giner, ``Scale - a software tool for listening experiments,'' in
+  \emph{AIA/DAGA Conference on Acoustics, Merano (Italy)}, 2013.
+
+\bibitem{beaqlejs}
+S.~Kraft and U.~Z{\"o}lzer, ``{BeaqleJS}: {HTML5} and {JavaScript} based
+  framework for the subjective evaluation of audio quality,'' in \emph{Linux
+  Audio Conference, Karlsruhe, DE}, 2014.
+
 \bibitem{mushra}
 \emph{Method for the subjective assessment of intermediate quality level of
   coding systems}.\hskip 1em plus 0.5em minus 0.4em\relax Recommendation {ITU-R
@@ -70,8 +121,12 @@
   Application}.\hskip 1em plus 0.5em minus 0.4em\relax John Wiley \& Sons,
   2007.

+\bibitem{jquery}
+{The jQuery Foundation}, ``Homepage of the {jQuery} and {jQueryUI} {JavaScript}
+  libraries.'' \url{http://jquery.com/} [Accessed 24th April, 2015].
+
 \bibitem{mozdevSupportedMedia}
-Mozilla, ``Media formats supported by the html audio and video elements,''
+Mozilla, ``Media formats supported by the {HTML} audio and video elements,''
   2015.

 \end{thebibliography}
--- a/docs/SMC15/smc2015template.bib	Fri Apr 24 02:02:58 2015 +0100
+++ b/docs/SMC15/smc2015template.bib	Fri Apr 24 11:54:45 2015 +0100
@@ -50,13 +50,13 @@

 @misc{mozdevSupportedMedia,
 	Author = {Mozilla},
-	Title = {Media formats supported by the HTML audio and video elements},
+	Title = {Media formats supported by the {HTML} audio and video elements},
 	Year = {2015},
 	Node = {\url{https://developer.mozilla.org/en-US/docs/Web/HTML/Supported_media_formats} [Accessed 22nd April, 2015]}
 	}

 @inproceedings{de2013real,
-  title={Real, Foley or synthetic? An evaluation of everyday walking sounds},
+  title={Real, foley or synthetic? {A}n evaluation of everyday walking sounds},
   author={de G{\"o}tzen, Amalia and Sikstr{\"o}m, Erik and Grani, Francesco and Serafin, Stefania},
   year={2013},
   booktitle={Proceedings of SMC 2013 : 10th Sound and Music Computing Conference},
@@ -73,7 +73,7 @@
 @conference{mason2015compression,
 	Author = {Mason, Andrew and Jillings, Nick and Ma, Zheng and Reiss, Joshua D. and Melchior, Frank},
 	Booktitle = {Audio Engineering Society Conference: 57th International Conference: The Future of Audio Entertainment Technology -- Cinema, Television and the Internet},
-	Month = {Mar},
+	Month = {March},
 	Title = {Adaptive Audio Reproduction Using Personalized Compression},
 	Year = {2015}}

@@ -83,4 +83,88 @@
   booktitle={Proceedings of Sound and Music Computing Conference},
   pages={48--53},
   year={2013}
-}
\ No newline at end of file
+}
+
+
+@inproceedings{beaqlejs,
+	Author = {Kraft, Sebastian and Z{\"o}lzer, Udo},
+	Booktitle = {Linux Audio Conference, Karlsruhe, DE},
+	Title = {{BeaqleJS}: {HTML5} and {JavaScript} based framework for the subjective evaluation of audio quality},
+	Year = {2014}}
+
+
+@inproceedings{uhlereiss,
+	Author = {Reiss, Joshua D. and Uhle, Christian},
+	Booktitle = {129th Convention of the Audio Engineering Society},
+	Month = {November},
+	Organization = {Audio Engineering Society},
+	Title = {Determined Source Separation for Microphone Recordings Using IIR Filters},
+	Year = {2010}}
+
+
+@conference{deman2014a,
+	Author = {De Man, Brecht and Joshua D. Reiss},
+	Booktitle = {53rd Conference of the Audio Engineering Society},
+	Month = {January},
+	Title = {Adaptive Control of Amplitude Distortion Effects},
+	Year = {2014}}
+
+
+@article{jquery,
+	Author = {{The jQuery Foundation}},
+	Note = {\url{http://jquery.com/} [Accessed 24th April, 2015]},
+	Title = {Homepage of the {jQuery} and {jQueryUI} {JavaScript} libraries.}}
+
+
+@conference{song2013b,
+	Author = {Song, Yading and Dixon, Simon and Halpern, Andrea R.},
+	Booktitle = {Proceedings of the 14th International Society for Music Information Retrieval (ISMIR2013)},
+	Title = {Do online social tags predict perceived or induced emotional responses to music?},
+	Year = {2013}}
+
+@article{song2013a,
+	Author = {Song, Yading and Dixon, Simon and Pearce, Marcus T. and Fazekas, Gy{\"o}rgy},
+	Journal = {Proceedings of the 3rd International Conference on Music \& Emotion (ICME)},
+	Title = {Using tags to select stimuli in the study of music and emotion},
+	Year = {2013}}
+
+@conference{whisper,
+	Author = {Simon Ciba and Andr{\'e} Wlodarski and Hans-Joachim Maempel},
+	Booktitle = {126th Convention of the Audio Engineering Society},
+	Month = {May 7-10},
+	Title = {Whis{PER} -- A new tool for performing listening tests},
+	Year = {2009}}
+
+@conference{scale,
+	Author = {Arnau Vazquez Giner},
+	Booktitle = {AIA/DAGA Conference on Acoustics, Merano (Italy)},
+	Title = {Scale - A Software Tool for Listening Experiments},
+	Year = {2013}}
+
+@inproceedings{mushram,
+	Author = {Emmanuel Vincent and Maria G. Jafari and Mark D. Plumbley},
+	Booktitle = {UK ICA Research Network Workshop},
+	Title = {Preliminary guidelines for subjective evalutation of audio source separation algorithms},
+	Year = {2006}}
+
+@inproceedings{schoeffler2013impact,
+	Author = {Schoeffler, Michael and Herre, J{\"u}rgen},
+	Booktitle = {Proceedings of Sound and Music Computing Conference},
+	Pages = {48--53},
+	Title = {About the Impact of Audio Quality on Overall Listening Experience},
+	Year = {2013}}
+
+@conference{repp,
+	Author = {Richard Repp},
+	Booktitle = {Proceedings of the 2006 International Computer Music Conference},
+	Pages = {pp. 468-474},
+	Title = {Recording Quality Ratings by Music Professionals},
+	Year = {2006}}
+
+@conference{durr2015implementation,
+	Author = {Durr, Gabriel and Peixoto, Lys and Souza, Marcelo and Tanoue, Raisa and Reiss, Joshua D.},
+	Booktitle = {Audio Engineering Society Conference: 56th International Conference: Audio for Games},
+	Month = {February},
+	Title = {Implementation and Evaluation of Dynamic Level of Audio Detail},
+	Year = {2015}}
+
--- a/docs/SMC15/smc2015template.tex	Fri Apr 24 02:02:58 2015 +0100
+++ b/docs/SMC15/smc2015template.tex	Fri Apr 24 11:54:45 2015 +0100
@@ -151,15 +151,23 @@

 %NICK: examples of what kind of audio applications HTML5 has made possible, with references to publications (or website)\\

-Perceptual evaluation of audio plays an important role in a wide range of research including audio effect design \cite{}, sound synthesis \cite{de2013real}, source separation \cite{}, codec design \cite{}, general perception of audio \cite{schoeffler2013impact}, and many others \cite{friberg2011comparison}. \\ % EXPAND
+Perceptual evaluation of audio plays an important role in a wide range of research including audio quality \cite{schoeffler2013impact,repp}, sound synthesis \cite{de2013real,durr2015implementation}, audio effect design \cite{deman2014a}, source separation \cite{uhlereiss}, music and emotion \cite{song2013b,song2013a}, and many others \cite{friberg2011comparison}.  % codec design?

 The Web Audio API is a high-level JavaScript API designed for real-time processing audio inside the browser through various processing nodes \cite{webaudioapi}. Various web sites have used the Web Audio API for either creative purposes, such as drum machines and score creation tools \cite{webaudiodemo}, %http://webaudio.github.io/demo-list/
 others from the list show real-time captured audio processing such as room reverberation tools and a phase vocoder from the system microphone. The BBC Radiophonic Workshop shows effects used on famous TV shows such as Doctor Who, being simulated inside the browser \cite{bbcradiophonics}. %http://webaudio.prototyping.bbc.co.uk/
-Another example is the BBC R\&D automatic compressor which applies a dynamic range compressor on a radio station which dynamically adjusts the compressor settings to match the listener's environment \cite{mason2015compression}.\\ % The paper for this has not been released yet by AES...
+Another example is the BBC R\&D automatic compressor which applies a dynamic range compressor on a radio station which dynamically adjusts the compressor settings to match the listener's environment \cite{mason2015compression}.

 This work is based in part on the APE audio perceptual evaluation interface for MATLAB \cite{deman2014b}. An important drawback of this toolbox is the need to have MATLAB to create a test and even to run (barring the use of an executable generated by MATLAB), and limited compatibility with both earlier and newer versions of MATLAB, which makes it hard to maintain. On the other hand, a web application generally has the advantage of running in most browsers on most applications, and we present a tool that requires no specialised software or even programming knowledge to set up.

-[TO ADD: other interfaces for perceptual evaluation of audio, browser-based or not!] % IMPORTANT
+% IMPORTANT
+%[TO ADD: other interfaces for perceptual evaluation of audio, browser-based or not!] \\
+%BROWSER-BASED: \cite{song2013b,song2013a,beaqlejs} \\
+%MATLAB: \cite{whisper,mushram,scale}
+% to add: OPAQUE, Rumsey's repertory grid technique
+
+Many perceptual audio interfaces are already available, such as the MATLAB-based tools MUSHRAM \cite{mushram}, WhisPER \cite{whisper}, and Scale \cite{scale}, and the browser-based tool used in \cite{song2013a,song2013b} and BeaqleJS \cite{beaqlejs}. The latter makes use of the HTML5 audio capabilities and comes with a number of predefined, established test interfaces such as ABX and MUSHRA \cite{mushra}.
+In this paper, we provide a listening test back end that allows for easy setup of a wide variety of listening tests, highly flexible yet very simple and not requiring any programming skills.
+% [How is this one different from all these?] improve

 We present a browser-based perceptual evaluation tool from which any kind of multiple stimulus audio evaluation tool where subjects need to rank, rate, select, or comment on different audio samples can be built.
 We also include an example of the multiple stimulus user interface included with the APE tool \cite{deman2014b}, which presents the subject with a number of axes on which a number of markers, corresponding with audio samples, can be moved to reflect any subjective quality, as well as corresponding comment boxes.
@@ -168,15 +176,15 @@

 There are a number of advantages to building a web audio based listening test environment. The ability to easily deploy a flexible and scalable testing environment that requires no proprietary software to run makes the web audio evaluation tool a very flexible testing tool. The ability to host a single test server and create multiple clients not only allows multiple participants to be involved in a trial simultaneously, but also permits participants to be located anywhere in the world. There are limited user experience issues, as all users should have some experience with using existing web technologies.

-
-[TO ADD: in the following sections, we describe the interface in more detail, discuss the implementation, and go over design considerations and flexibility ... ]
-
+In the following sections, we describe the included interface in more detail, discuss the implementation, and cover considerations that were made in the design process of this tool.

 \section{Interface}\label{sec:interface}

-At this point, we have implemented the interface of the MATLAB-based APE Perceptual Evaluation for Audio toolbox \cite{deman2014b}, which shows one marker for each simultaneously evaluated audio fragment on one or more horizontal axes, that can be moved to rate or rank the respective fragments in terms of any subjective quality, as well as a comment box for every marker, and any extra text boxes for extra comments. See \ref{fig:interface} for an example of the interface, with eleven fragments and one axis. %? change if a new interface is shown
+At this point, we have implemented the interface of the MATLAB-based APE Perceptual Evaluation for Audio toolbox \cite{deman2014b}, which shows one marker for each simultaneously evaluated audio fragment on one or more horizontal axes, that can be moved to rate or rank the respective fragments in terms of any subjective quality, as well as a comment box for every marker, and any extra text boxes for extra comments.
+The reason for such an interface, where all stimuli are presented on a single rating axis (or multiple axes if multiple subjective qualities need to be evaluated), is that it urges the subject to consider the rating and/or ranking of the stimuli relative to one another, as opposed to comparing each individual stimulus to a given reference, as is the case with e.g. a MUSHRA test \cite{mushra}. In this case, multiple stimuli are presented at once, but as each stimulus has its corresponding slider, the
+See \ref{fig:interface} for an example of the interface, with eleven fragments and one axis. %? change if a new interface is shown

-However, the back end of this test environment allows for many more established and novel interfaces for listening tests, particularly ones where the subject only assesses audio without manipulating it (i.e. method of adjustment, which would require additional features to be implemented).
+However, the back end of this test environment allows for many more established and novel interfaces for listening tests, particularly ones where the subject only assesses audio without manipulating it (i.e. method of adjustment), which would require additional features to be implemented.

 \begin{figure*}[ht]
 \begin{center}
@@ -187,16 +195,6 @@
 \end{figure*}


-************
-
-[DAVE to make new picture] - Apologies but I am having a few troubles being able to get a good looking example of the interface - The text boxes keep rolling out of bounds - and the resolution on my laptop being far from ideal. Can someone else please make a quick mockup test and take a screenshot.
-
-************
-
-[BRECHT to expand]
-
-
-
 \section{Architecture}\label{sec:architecture} % or implementation?

 The tool runs entirely inside the browser through the new HTML5 Web Audio API and is supported by most major web browsers (with the exception of Internet Explorer). The API allows for constructing audio processing elements and connecting them together to produce a high quality, real time signal process to manipulate audio streams. The API supports multi-channel processing and has an accurate playback timer for precise scheduled playback control. The API is controlled through the browser JavaScript engine and is therefore highly configurable. Processing is all performed in a separate, low latency thread to the main JavaScript thread, meaning there is no blocking due to real time processing.
@@ -208,8 +206,7 @@
 \item \texttt{ape.js}: Parses setup files to create the interface as instructed, following the same style chain as the MATLAB APE Tool \cite{deman2014b}.
 \end{itemize}

-The HTML file loads the \texttt{core.js} file along with a few other ancillary files (such as the jQuery JavaScript extensions)% should we cite jQuery.... https://jquery.com/
-, at which point the browser JavaScript begins to execute the on-page instructions, which gives the URL of the test setup XML document (outlined in Section \ref{sec:setupresultsformats}). \texttt{core.js} parses this document and executes the function in \texttt{ape.js} to build the web page with the given audio files. The reason for separating these two files is to allow for further interface designs (such as MUSHRA \cite{mushra} or A-B tests \cite{bech}) to be used, which would still require the same underlying core functions outlined in \texttt{core.js}, see also Section \ref{sec:interface}.
+The HTML file loads the \texttt{core.js} file along with a few other ancillary files (such as the jQuery JavaScript extensions \cite{jquery}), at which point the browser JavaScript begins to execute the on-page instructions, which gives the URL of the test setup XML document (outlined in Section \ref{sec:setupresultsformats}). \texttt{core.js} parses this document and executes the function in \texttt{ape.js} to build the web page with the given audio files. The reason for separating these two files is to allow for further interface designs (such as MUSHRA \cite{mushra} or AB tests \cite{bech}) to be used, which would still require the same underlying core functions outlined in \texttt{core.js}, see also Section \ref{sec:interface}.

 The \texttt{ape.js} file has several main functions but the most important are \textit{loadInterface(xmlDoc)}, \textit{loadTest(id)}, \textit{pageXMLSave(testId)} and \textit{interfaceXMLSave()}. \textit{loadInterface(xmlDoc)} is called to decode the supplied project document in respect for the interface specified and define any global structures (such as the slider interface). It also identifies the number of pages in the test and randomises the order, if specified to do so. This is the only madatory function in any of the interface files as this is called by \texttt{core.js} when the document is ready. The design style is such that \texttt{core.js} cannot 'see' any interface specific functions and therefore cannot assume any are available. Therefore the \textit{loadInterface(xmlDoc)} is very important to setup the entire test environment. It can be assumed that the interface files can `see' the \texttt{core.js} file and can therefore not only interact with it, but also modify it.

@@ -231,11 +228,11 @@
 %During playback, the playback nodes loop indefinitely until playback is stopped. The gain nodes in the \textit{audioObject}s enable dynamic muting of nodes. When a bar in the sliding ranking is clicked, the audio engine mutes all \textit{audioObject}s and un-mutes the clicked one. Therefore, if the audio samples are perfectly aligned up and of the same sample length, they will remain perfectly aligned with each other.
 % Don't think this is relevant anymore

-\section{Setup and results formats}\label{sec:setupresultsformats}
+\section{Input and result files}\label{sec:setupresultsformats}

-[DAVE: somewhere add input file]
+The setup and result files both use the common XML document format to outline the various parameters. The setup file determines which interface to use, the location of audio files, how many pages and other general setup rules to define the testing environment. Having one document to modify allows for quick manipulation in a `human readable' form to create new tests, or adjust current ones, without needing to edit multiple web files. An example of this XML document is presented in Figure~\ref{fig:xmlIn}% I mean the .js and .html files, though not sure if any better.

-Setup and the results both use the common XML document format to outline the various parameters. The setup file determines which interface to use, the location of audio files, how many pages and other general setup rules to define the testing environment. Having one document to modify allows for quick manipulation in a `human readable' form to create new tests, or adjust current ones, without needing to edit multiple web files. An example of this XML document is presented in Figure~\ref{fig:xmlIn}% I mean the .js and .html files, though not sure if any better.
+\subsection{Setup and configurability}

 \begin{figure}[ht]
 \begin{center}
@@ -247,9 +244,8 @@

 The setup document has several defined nodes and structure which are documented with the source code. For example there is a section for general setup options where the pre-test and post-test questions and statements are set up and from the example in Figure~\ref{fig:xmlIn}, it can be seen that a question box should be generated, with the id 'location' and it is mandatory to answer. The question is in the PreTest node meaning it will appear before any testing will begin. When the result for the  entire test is shown, then this will appear in the PreTest node of the response with the id 'location' allowing it to be found easily. This outlines the importance of having clear and meaningful ID values. Pre- and post-test dialog boxes allow for comments or questions to be presented before or after the test, to convey listening test instructions, gather information about the subject, listening environment, and overall experience of the test.

-[BRECHT to say we try to cater to as many audiences as possible, simple yet tweakable]

-Further options in the setup file are:
+We try to cater to a diverse audience with this toolbox, while ensuring the toolbox is simple, elegant and straightforward. To that end, we include the following options that can be easily switched on and off, by setting the value in the input XML file.

 \begin{itemize} %Should have used a description list for this.
 \item \textbf{Snap to corresponding position}: When this is enabled, and a fragment is playing, the playhead skips to the same position in the next fragment that is clicked. If it is not enabled, every fragment is played from the start.
@@ -270,13 +266,14 @@
 \item \textbf{Reference}: Allows for a separate sample (outside of the axis) to be the `reference', which the subject can play back during the test to help with the task at hand \cite{mushra}.
 \item \textbf{Hidden reference}: Whether or not an explicit `reference' is provided, the `hidden reference' should be rated above a certain value \cite{mushra} - this can be enforced.
 \item \textbf{Hidden anchor}: The `hidden anchor' should be rated lower than a certain value \cite{mushra} - this can be enforced.
-
 \end{itemize}

-When one of these options is not included in the setup file, they assume a default value.
+When one of these options is not included in the setup file, they assume a default value. As a result, the input file can be kept very compact if default values suffice for the test.

 % loop, snap to corresponding position, comments, 'general' comment, require same sampling rate, different types of randomisation

+\subsection{Results}
+
 The results file is dynamically generated by the interface upon clicking the `Submit' button. This also executes checks, depending on the setup file, to ensure that all tracks have been played back, rated and commented on. The XML output returned contains a node per audioObject and contains both the corresponding marker's position and any comments written in the associated comment box. The rating returned is normalised to be a value between 0 and 1, normalising the pixel representation of different browser windows. An example output file is presented in Figure~\ref{fig:xmlOut}

 \begin{figure}[ht]
@@ -287,21 +284,14 @@
 \end{center}
 \end{figure}

-The results will also contain information collected by any defined pre/post questions. These are referenced against the setup XML by using the same ID so readable responses can be obtained. Taking from the earlier example of setting up a pre-test question, an example reponse would be shown as the following.
+The results also contain information collected by any defined pre/post questions. These are referenced against the setup XML by using the same ID so readable responses can be obtained. Taking from the earlier example of setting up a pre-test question, an example reponse would be shown as the following.

 \texttt{<comment id="location"> Queen Mary \\University of London </comment>}

-Each page of testing is returned with the results of the entire page included in the structure. One `audioElement' node is created per audio fragment per page, along with its ID. This includes several child nodes including the value holding the rating between 0 and 1, and any metrics collected. These include how long the element was listened for, the initial position, boolean flags if the element was listened to, if the element was moved and if the element comment box had any comment. Furthermore, each user action (manipulation of any interface element, such as playback or moving a marker) is logged along with a the corresponding time code and stored or sent along with the results.
+Each page of testing is returned with the results of the entire page included in the structure. One `audioElement' node is created per audio fragment per page, along with its ID. This includes several child nodes including the value holding the rating between 0 and 1, the comment, and any other collected metrics. The latter include how long the element was listened for, the initial position, boolean flags if the element was listened to, if the element was moved and if the element comment box had any comment. Furthermore, each user action (manipulation of any interface element, such as playback or moving a marker) is logged along with a the corresponding time code and stored or sent along with the results.
+Furthermore, we also store session data such as the browser the tool was used in.

-Future development will also evolve to include any session data, such as the browser the tool was used in. Currently the results files are downloaded on the user side of the browser as a .xml file to be manually returned. However the end goal is to allow the XML files to be submitted over the web to a receiving server to store them, allowing for automated collection.
-
-*********************
-
-Do we want to discuss this in the future of present tense - as it will be a working feature by the time we are published (July at the earliest?) So this can be present tense - as in 'this work has already been done' or should this go into further work?
-
-*********************
-
- % right?
+We provide the option to store the results locally, and/or to have them sent to a server.

 %Here is an example of the setup XML and the results XML: % perhaps best to refer to each XML after each section (setup <> results)
 % Should we include an Example of the input and output XML structure?? --> Sure.
@@ -331,9 +321,10 @@
 %</metric> \\
 %</audioelement>}

-As can be seen from Figure~\ref{fig:xmlOut}, the parent tag \texttt{audioelement} holds the id of the element passed in from the setup document. The first child element is \texttt{comment} and holds both the question shown and the response from the comment box inside.
+As can be seen from Figure~\ref{fig:xmlOut}, the parent tag \texttt{audioelement} holds the id of the element passed in from the setup document. The first child element is \texttt{comment} and holds both the question shown and the response from the comment box inside.
 The child element \texttt{value} holds the normalised ranking value. Next comes the metric node structure, there is one \texttt{metricresult} node per metric event collected. The id of the node identifies the type of data it contains. For example, the first holds the id \textit{elementTimer} and the data contained represents how long, in seconds, the audio element was listened to. The next holds the id \textit{elementTrackerFull} and contains a pair of elements per entry. This represents the entire movement of the elements' slider giving the time the event took place in seconds from when the current test page started, and the new position. In our example there are three \texttt{timepos} children with their id representing their order. There is one of these \texttt{audioelement} tags per audio element outlined on each test page.

+
 \section{Conclusions and future work}\label{sec:conclusions}

 In this paper we have presented an approach to creating a browser-based listening test environment that can be used for a variety of types of perceptual evaluation of audio.
@@ -341,20 +332,8 @@
 The purpose of this paper is to outline the design of this tool, to describe our implementation using basic HTML5 functionality, and to discuss design challenges and limitations of our approach. % or something

 % future work
-Further work may include the development of other common test designs, such as MUSHRA \cite{mushra}, AB, ABX and method of adjustment tests.
-In addition, [...].
-[BRECHT to correct and expand]
-
-from community
-
-%\begin{itemize}
-%\item Options for MUSHRA style experiment with vertical slide per track
-%\item APE style experiment where all tracks are on a single horizontal axis % isn't that what we're doing now?
-%\item AB test
-%\item ABX test
-%\item Method of adjustment tests
-%\end{itemize}
-
+Further work may include the development of other common test designs, such as MUSHRA \cite{mushra}, 2D valence and arousal rating, and others. We will add functionality to assist with setting up large-scale tests with remote subjects, so this becomes straightforward and intuitive.
+In addition, we will keep on improving and expanding the tool, and highly welcome feedback and contributions from the community.

 The source code of this tool can be found on \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}.

@@ -363,59 +342,4 @@
 %bibliography here
 \bibliography{smc2015template}

-
-
 \end{document}
-
-
-
-% RUBBISH
-
-%\subsection{Equations}
-%Equations of importance,
-%or to which you refer later,
-%should be placed on separated lines and numbered.
-%The number should be on the right side, in parentheses.
-%\begin{equation}
-%E=mc^{2+\delta}.
-%\label{eq:Emc2}
-%\end{equation}
-%Refer to equations like so:
-%As (\ref{eq:Emc2}) shows,
-%I do not completely trust Special Relativity.
-%
-%\subsection{Figures, Tables and Captions}
-%\begin{table}[t]
-% \begin{center}
-% \begin{tabular}{|l|l|}
-%  \hline
-%  String value & Numeric value \\
-%  \hline
-%  Hej SMC  & 2015 \\
-%  \hline
-% \end{tabular}
-%\end{center}
-% \caption{Table captions should be placed below the table, exactly like this,
-% but using words different from these.}
-% \label{tab:example}
-%\end{table}
-
-%\begin{figure}[t]
-%\figbox{
-%\subfloat[][]{\includegraphics[width=60mm]{figure}\label{fig:subfigex_a}}\\
-%\subfloat[][]{\includegraphics[width=80mm]{figure}\label{fig:subfigex_b}}
-%}
-%\caption{Here's an example using the subfig package.\label{fig:subfigex} }
-%\end{figure}
-
-
-
-
-
-%\begin{acknowledgments}
-%You may acknowledge people, projects,
-%funding agencies, etc.
-%which can be included after the second-level heading
-%``Acknowledgments'' (with no numbering).
-%\end{acknowledgments}
-