annotate docs/WAC2016/WAC2016.tex @ 1948:a14124d3578b

Paper: outline, minor edits

\documentclass{sig-alternate}

\begin{document}

% Copyright
\setcopyright{waclicense}


%% DOI
%\doi{10.475/123_4}
%
%% ISBN
%\isbn{123-4567-24-567/08/06}
%
%%Conference
%\conferenceinfo{PLDI '13}{June 16--19, 2013, Seattle, WA, USA}
%
%\acmPrice{\$15.00}

%
% --- Author Metadata here ---
\conferenceinfo{Web Audio Conference WAC-2016,}{April 4--6, 2016, Atlanta, USA}
\CopyrightYear{2016} % Allows default copyright year (20XX) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{Web Audio Evaluation Tool something something}
%\subtitle{[Extended Abstract]
%\titlenote{A full version of this paper is available as
%\textit{Author's Guide to Preparing ACM SIG Proceedings Using
%\LaTeX$2_\epsilon$\ and BibTeX} at
%\texttt{www.acm.org/eaddress.htm}}}
%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

% FIVE authors instead of four, to leave space between first two authors.
\numberofauthors{5} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor Nicholas Jillings\\
       \email{[e-mail]}
 % dummy author for nicer spacing
 \alignauthor 
% 2nd. author
\alignauthor Brecht De Man\\
       \email{[e-mail]}
\and  % use '\and' if you need 'another row' of author names
% 3rd. author
\alignauthor David Moffat\\
       \email{[e-mail]}
% 4th. author
\alignauthor Joshua D. Reiss\\
\email{[e-mail]}
\and % new line for address
       \affaddr{Centre for Digital Music}\\
       \affaddr{School of Electronic Engineering and Computer Science}\\
       \affaddr{Queen Mary University of London}\\
       \affaddr{Mile End Road,}
       \affaddr{London E1 4NS}\\
       \affaddr{United Kingdom}\\
}
%Centre for Digital Music, School of Electronic Engineering and Computer Science, Queen Mary University of London
%% 5th. author
%\alignauthor Sean Fogarty\\
%       \affaddr{NASA Ames Research Center}\\
%       \affaddr{Moffett Field}\\
%       \email{[e-mail]}
%% 6th. author
%\alignauthor Charles Palmer\\
%       \affaddr{Palmer Research Laboratories}\\
%       \affaddr{8600 Datapoint Drive}\\
%       \email{[e-mail]}
%}
% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
%\additionalauthors{Additional authors: John Smith (The Th{\o}rv{\"a}ld Group,
%email: {\texttt{[e-mail]}}) and Julius P.~Kumquat
%(The Kumquat Consortium, email: {\texttt{[e-mail]}}).}
\date{1 October 2015}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
Here comes the abstract. 
\end{abstract}


\section{Introduction}
	Introducing the paper. Referring to \cite{waet}. Talking about what we do in the various sections of this paper. Pointing out that the header of the paper kind of looks like the Bat-sign. 
	
	Situating the Web Audio Evaluation Tool between other currently available evaluation tools, ...
	
	% only browser-based? 
	\begin{table*}[htdp]
        \caption{Table with existing listening test platforms and their features}
        \begin{center}
        	\begin{tabular}{|*{6}{l|}}
        		\hline
        		\textbf{Name} 	& \textbf{Ref.} 	& \textbf{Language} 	& \textbf{Interfaces} 			& \textbf{Remote} 			& \textbf{Programming} 	\\
        		\hline
        		APE 			& \cite{ape}		& MATLAB				& multiple stimulus one axis 	& 							& \checkmark			\\
        		BeaqleJS		& \cite{beaqlejs}	& JavaScript			& 								& not natively supported 	& \\
        		HULTI-GEN 		& \cite{hultigen}	& MAX 					& 								& 							& \\
        		\textbf{WAET}	& \cite{waet}		& JavaScript			& \textbf{all of the above}		& \checkmark				& \\
        		\hline
        	\end{tabular}
        \end{center}
        \label{tab:toolboxes}
        \end{table*}%

        % about BeaqleJS
        ... However, BeaqleJS \cite{beaqlejs} does not make use of the Web Audio API, %requires programming knowledge... 

        % 
        Selling points: remote tests, visualisaton, create your own test in the browser, many interfaces

	
\section{Architecture}  % title? 'back end'?
	A slightly technical overview of the system. Talk about XML, JavaScript, Web Audio API, HTML5. 
	Describe and/or visualise audioholder-audioelement-... structure. 

	% see also SMC12
	
	Streaming audio? % probably not, unless it's easy

	Compatibility? 
	
	
\section{Remote tests} % with previous? 
	The following features allow easy and effective remote testing: 
	\begin{itemize}
		\item PHP script to collect result XML files
		\item Randomly pick specified number of audioholders
		\item Functionality to participate multiple times
			\begin{itemize}
				\item Possible to log in with unique ID (no password)
				\item Pick `new user' (need new, unique ID) or `already participated' (need already available ID)
				\item Store XML on server with IDs plus which audioholders have already been listened to
				\item Don't show `post-test' survey after first time
				\item Pick `new' audioholders if available
				\item Copy survey information first time to new XMLs
			\end{itemize}
		\item Intermediate saves
		\item Collect IP address information (privacy issues?) --> geo-related API? 
	\end{itemize}

	
\section{Interfaces} % title? 'Front end'? % Dave
	`Build your own test'

	We could add more interfaces, such as: 
	\begin{itemize}
		\item (APE style) \cite{ape}
		\item Multi attribute ratings
		\item MUSHRA (ITU-R BS. 1534)~\cite{recommendation20031534}
		\item Interval Scale~\cite{zacharov1999round}
		\item Rank Scale~\cite{pascoe1983evaluation}
		
		\item 2D Plane rating - e.g. Valence vs. Arousal~\cite{carroll1969individual}
		\item Likert scale~\cite{likert1932technique}
		
		\item {\bf All the following are the interfaces available in HULTI-GEN~\cite{hultigen} }
		\item ABC/HR (ITU-R BS. 1116)~\cite{recommendation19971116}
		\begin{itemize}
			\item Continuous Scale (5-1) Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying. (default Inaudible?)
		\end{itemize}
		\item -50 to 50 Bipolar with Ref
		\begin{itemize}
			\item Scale -50 to 50 on Mushra with default values as 0 in middle and a comparison ``Reference'' to compare to 0 value
		\end{itemize}
		\item Absolute Category Rating (ACR) Scale~\cite{rec1996p}
		\begin{itemize}
			\item 5 point Scale - Bad, Poor, Fair, Good, Excellent (Default fair?)
		\end{itemize}
		\item Degredation Category Rating (DCR) Scale~\cite{rec1996p}
		\begin{itemize}
			\item 5 point Scale - Inaudible, Audible but not annoying, slightly annoying, annoying, very annoying. (default Inaudible?) - {\it Basically just quantised ABC/HR?}
		\end{itemize}
		\item Comparison Category Rating (CCR) Scale~\cite{rec1996p}
		\begin{itemize}
			\item 7 point scale: Much Better, Better, Slightly Better, About the same, slightly worse, worse, much worse - Default about the same with reference to compare to
		\end{itemize}
		\item 9 Point Hedonic Category Rating Scale~\cite{peryam1952advanced}
		\begin{itemize}
			\item 9 point scale: Like Extremely, Like Very Much, Like Moderate, Like Slightly, Neither Like nor Dislike, dislike Extremely, dislike Very Much, dislike Moderate, dislike Slightly  - Default Neither Like nor Dislike with reference to compare to
		\end{itemize}
		\item ITU-R 5 Point Continuous Impairment Scale~\cite{rec1997bs}
		\begin{itemize}
			\item 5 point Scale (5-1) Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying. (default Inaudible?)- {\it Basically just quantised ABC/HR, or Different named DCR}
		\end{itemize}
		\item Pairwise Comparison (Better/Worse)~\cite{david1963method}
		\begin{itemize}
			\item 2 point Scale - Better or Worse - (not sure how to default this - they default everything to better, which is an interesting choice)
		\end{itemize}
	\end{itemize}
	
	There are also the following interfaces, which would require a slightly different `engine' underneath, e.g. loading a different page for every possible pair. 
	\begin{itemize}
		\item AB Test~\cite{lipshitz1981great}
		\item ABX Test~\cite{clark1982high}
		\item JND
	\end{itemize}
	
	A screenshot would be nice. 

\section{Analysis and diagnostics}
	It would be great to have easy-to-use analysis tools to visualise the collected data and even do science with it. Even better would be to have all this in the browser. Complete perfection would be achieved if and when only limited setup, installation time, and expertise are required for the average non-CS researcher to use this. 
	
	The following could be nice: 
	
	\begin{itemize}
		\item Web page showing all audioholder IDs, file names, subject IDs, audio element IDs, ... in the collected XMLs so far (\texttt{saves/*.xml})
		\item Check/uncheck each of the above for analysis (e.g. zoom in on a certain song, or exclude a subset of subjects)
		\item Click a mix to hear it (follow path in XML setup file, which is also embedded in the XML result file)
		\item Box plot, confidence plot, scatter plot of values (for a given audioholder)
		\item Timeline for a specific subject (see Python scripts), perhaps re-playing the experiment in X times realtime. (If actual realtime, you could replay the audio...)
		\item Distribution plots of any radio button and number questions (drop-down menu with `pretest', `posttest', ...; then drop-down menu with question `IDs' like `gender', `age', ...; make pie chart/histogram of these values over selected range of XMLs)
		\item All `comments' on a specific audioelement
		\item A `download' button for a nice CSV of various things (values, survey responses, comments) people might want to use for analysis, e.g. when XML scares them
		\item Validation of setup XMLs (easily spot `errors', like duplicate IDs or URLs, missing/dangling tags, ...)
	\end{itemize}

	A subset of the above would already be nice for this paper. 
	
	Some pictures here please. 

\section{Concluding remarks and future work}
	Perhaps an `engineering brief' such as this one doesn't really have a lot of conclusion, except `We made this'. 
	
	You can check it out at \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}. 
	
	Talking a little bit about what else might happen. Unless we really want to wrap this up. 

	Use \cite{schoeffler2015mushra} as a `checklist'.

%
% The following two commands are all you need in the
% initial runs of your .tex file to
% produce the bibliography for the citations in your paper.
\bibliographystyle{abbrv}
\bibliography{WAC2016}  % sigproc.bib is the name of the Bibliography in this case
% You must have a proper ".bib" file
%  and remember to run:
% latex bibtex latex latex
% to resolve all references
%
% ACM needs 'a single self-contained file'!
%
\end{document}