# HG changeset patch # User Brecht De Man # Date 1456742722 -3600 # Node ID 49d27886aa0360642bfb8e4d4dfa8113b4357435 # Parent 80e6cf42a17b69392a12e3c71fda59e589bca6fd WAC paper final review diff -r 80e6cf42a17b -r 49d27886aa03 docs/WAC2016/WAC2016.bib --- a/docs/WAC2016/WAC2016.bib Thu Feb 25 11:57:24 2016 +0000 +++ b/docs/WAC2016/WAC2016.bib Mon Feb 29 11:45:22 2016 +0100 @@ -1,7 +1,7 @@ %% This BibTeX bibliography file was created using BibDesk. %% http://bibdesk.sourceforge.net/ -%% Created for Brecht De Man at 2015-10-12 17:58:50 +0100 +%% Created for Brecht De Man at 2016-02-29 11:43:56 +0100 %% Saved with string encoding Unicode (UTF-8) @@ -16,28 +16,27 @@ Keywords = {perceptual evaluation}, Title = {Preliminary guidelines for subjective evalutation of audio source separation algorithms}, Year = {2006}, - Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QNS4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL211c2hyYW0ucGRm0hcLGBlXTlMuZGF0YU8RAaAAAAAAAaAAAgAADE1hY2ludG9zaCBIRAAAAAAAAAAAAAAAAAAAANBcT3dIKwAAAApfEQttdXNocmFtLnBkZgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqdQz0+BaAAAAAAAAAAAAAQABAAACSAAAAAAAAAAAAAAAAAAAAAGUGFwZXJzABAACAAA0FxBZwAAABEACAAAz0+BaAAAAAEAFAAKXxEACl67AApdAgAJRF4AApPVAAIASE1hY2ludG9zaCBIRDpVc2VyczoAQnJlY2h0OgBHb29nbGUgRHJpdmU6AERvY3VtZW50czoAUGFwZXJzOgBtdXNocmFtLnBkZgAOABgACwBtAHUAcwBoAHIAYQBtAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgA2VXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL211c2hyYW0ucGRmABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4AxgDLANMCdwJ5An4CiQKSAqACpAKrArQCuQLGAskC2wLeAuMAAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAAC5Q==}} + Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QNS4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL211c2hyYW0ucGRm0hcLGBlXTlMuZGF0YU8RAaAAAAAAAaAAAgAADE1hY2ludG9zaCBIRAAAAAAAAAAAAAAAAAAAANBcXYdIKwAAAApfEQttdXNocmFtLnBkZgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqdQz0+PeAAAAAAAAAAAAAQABAAACSAAAAAAAAAAAAAAAAAAAAAGUGFwZXJzABAACAAA0FxBZwAAABEACAAAz0+BaAAAAAEAFAAKXxEACl67AApdAgAJRF4AApPVAAIASE1hY2ludG9zaCBIRDpVc2VyczoAQnJlY2h0OgBHb29nbGUgRHJpdmU6AERvY3VtZW50czoAUGFwZXJzOgBtdXNocmFtLnBkZgAOABgACwBtAHUAcwBoAHIAYQBtAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgA2VXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL211c2hyYW0ucGRmABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4AxgDLANMCdwJ5An4CiQKSAqACpAKrArQCuQLGAskC2wLeAuMAAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAAC5Q==}} @conference{scale, Author = {Arnau Vazquez Giner}, Booktitle = {AIA/DAGA Conference on Acoustics, Merano (Italy)}, Date-Added = {2015-10-12 16:55:54 +0000}, - Date-Modified = {2015-10-12 16:55:54 +0000}, + Date-Modified = {2016-02-29 10:29:38 +0000}, Keywords = {perceptual evaluation}, - Title = {Scale - A Software Tool for Listening Experiments}, + Title = {Scale - {A} Software Tool for Listening Experiments}, Year = {2013}, - Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QMy4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL1NjYWxlLnBkZtIXCxgZV05TLmRhdGFPEQGYAAAAAAGYAAIAAAxNYWNpbnRvc2ggSEQAAAAAAAAAAAAAAAAAAADQXE93SCsAAAAKXxEJU2NhbGUucGRmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqmAM9QyZAAAAAAAAAAAAAEAAQAAAkgAAAAAAAAAAAAAAAAAAAABlBhcGVycwAQAAgAANBcQWcAAAARAAgAAM9QyZAAAAABABQACl8RAApeuwAKXQIACUReAAKT1QACAEZNYWNpbnRvc2ggSEQ6VXNlcnM6AEJyZWNodDoAR29vZ2xlIERyaXZlOgBEb2N1bWVudHM6AFBhcGVyczoAU2NhbGUucGRmAA4AFAAJAFMAYwBhAGwAZQAuAHAAZABmAA8AGgAMAE0AYQBjAGkAbgB0AG8AcwBoACAASABEABIANFVzZXJzL0JyZWNodC9Hb29nbGUgRHJpdmUvRG9jdW1lbnRzL1BhcGVycy9TY2FsZS5wZGYAEwABLwAAFQACAA3//wAAgAbSGxwdHlokY2xhc3NuYW1lWCRjbGFzc2VzXU5TTXV0YWJsZURhdGGjHR8gVk5TRGF0YVhOU09iamVjdNIbHCIjXE5TRGljdGlvbmFyeaIiIF8QD05TS2V5ZWRBcmNoaXZlctEmJ1Ryb290gAEACAARABoAIwAtADIANwBAAEYATQBVAGAAZwBqAGwAbgBxAHMAdQB3AIQAjgDEAMkA0QJtAm8CdAJ/AogClgKaAqECqgKvArwCvwLRAtQC2QAAAAAAAAIBAAAAAAAAACgAAAAAAAAAAAAAAAAAAALb}} + Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QMy4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL1NjYWxlLnBkZtIXCxgZV05TLmRhdGFPEQGYAAAAAAGYAAIAAAxNYWNpbnRvc2ggSEQAAAAAAAAAAAAAAAAAAADQXF2HSCsAAAAKXxEJU2NhbGUucGRmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqmAM9Q16AAAAAAAAAAAAAEAAQAAAkgAAAAAAAAAAAAAAAAAAAABlBhcGVycwAQAAgAANBcQWcAAAARAAgAAM9QyZAAAAABABQACl8RAApeuwAKXQIACUReAAKT1QACAEZNYWNpbnRvc2ggSEQ6VXNlcnM6AEJyZWNodDoAR29vZ2xlIERyaXZlOgBEb2N1bWVudHM6AFBhcGVyczoAU2NhbGUucGRmAA4AFAAJAFMAYwBhAGwAZQAuAHAAZABmAA8AGgAMAE0AYQBjAGkAbgB0AG8AcwBoACAASABEABIANFVzZXJzL0JyZWNodC9Hb29nbGUgRHJpdmUvRG9jdW1lbnRzL1BhcGVycy9TY2FsZS5wZGYAEwABLwAAFQACAA3//wAAgAbSGxwdHlokY2xhc3NuYW1lWCRjbGFzc2VzXU5TTXV0YWJsZURhdGGjHR8gVk5TRGF0YVhOU09iamVjdNIbHCIjXE5TRGljdGlvbmFyeaIiIF8QD05TS2V5ZWRBcmNoaXZlctEmJ1Ryb290gAEACAARABoAIwAtADIANwBAAEYATQBVAGAAZwBqAGwAbgBxAHMAdQB3AIQAjgDEAMkA0QJtAm8CdAJ/AogClgKaAqECqgKvArwCvwLRAtQC2QAAAAAAAAIBAAAAAAAAACgAAAAAAAAAAAAAAAAAAALb}} @conference{whisper, Author = {Simon Ciba and Andr{\'e} Wlodarski and Hans-Joachim Maempel}, Booktitle = {126th Convention of the AES}, Date-Added = {2015-10-12 16:55:54 +0000}, - Date-Modified = {2015-10-12 16:55:54 +0000}, + Date-Modified = {2016-02-29 10:43:56 +0000}, Keywords = {perceptual evaluation}, - Month = {May 7-10}, - Title = {WhisPER -- {A} new tool for performing listening tests}, + Title = {Whis{PER} - {A} new tool for performing listening tests}, Year = {2009}, - Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QNS4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3doaXNwZXIucGRm0hcLGBlXTlMuZGF0YU8RAaAAAAAAAaAAAgAADE1hY2ludG9zaCBIRAAAAAAAAAAAAAAAAAAAANBcT3dIKwAAAApfEQt3aGlzcGVyLnBkZgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACngFz1DL0QAAAAAAAAAAAAQABAAACSAAAAAAAAAAAAAAAAAAAAAGUGFwZXJzABAACAAA0FxBZwAAABEACAAAz1DL0QAAAAEAFAAKXxEACl67AApdAgAJRF4AApPVAAIASE1hY2ludG9zaCBIRDpVc2VyczoAQnJlY2h0OgBHb29nbGUgRHJpdmU6AERvY3VtZW50czoAUGFwZXJzOgB3aGlzcGVyLnBkZgAOABgACwB3AGgAaQBzAHAAZQByAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgA2VXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3doaXNwZXIucGRmABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4AxgDLANMCdwJ5An4CiQKSAqACpAKrArQCuQLGAskC2wLeAuMAAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAAC5Q==}} + Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QNS4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3doaXNwZXIucGRm0hcLGBlXTlMuZGF0YU8RAaAAAAAAAaAAAgAADE1hY2ludG9zaCBIRAAAAAAAAAAAAAAAAAAAANBcXYdIKwAAAApfEQt3aGlzcGVyLnBkZgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABG5Iuz1DZ4QAAAAAAAAAAAAQABAAACSAAAAAAAAAAAAAAAAAAAAAGUGFwZXJzABAACAAA0FxBZwAAABEACAAAz1DL0QAAAAEAFAAKXxEACl67AApdAgAJRF4AApPVAAIASE1hY2ludG9zaCBIRDpVc2VyczoAQnJlY2h0OgBHb29nbGUgRHJpdmU6AERvY3VtZW50czoAUGFwZXJzOgB3aGlzcGVyLnBkZgAOABgACwB3AGgAaQBzAHAAZQByAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgA2VXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3doaXNwZXIucGRmABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4AxgDLANMCdwJ5An4CiQKSAqACpAKrArQCuQLGAskC2wLeAuMAAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAAC5Q==}} @book{bech, Annote = {p 115: GLS @@ -77,22 +76,21 @@ Author = {Schoeffler, Michael and St{\"o}ter, Fabian-Robert and Edler, Bernd and Herre, J{\"u}rgen}, Booktitle = {1st Web Audio Conference}, Date-Added = {2015-09-29 18:35:27 +0000}, - Date-Modified = {2015-09-29 18:37:59 +0000}, - Title = {Towards the Next Generation of Web-based Experiments: {A} Case Study Assessing Basic Audio Quality Following the {ITU-R} Recommendation {BS}. 1534 ({MUSHRA})}, + Date-Modified = {2016-02-29 10:35:38 +0000}, + Title = {Towards the Next Generation of Web-based Experiments: {A} Case Study Assessing Basic Audio Quality Following the {ITU-R} {R}ecommendation {BS}. 1534 ({MUSHRA})}, Year = {2015}, - Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QOi4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3dhYzE1X211c2hyYS5wZGbSFwsYGVdOUy5kYXRhTxEBtgAAAAABtgACAAAMTWFjaW50b3NoIEhEAAAAAAAAAAAAAAAAAAAA0FxPd0grAAAACl8REHdhYzE1X211c2hyYS5wZGYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAESrbDSMIYSAAAAAAAAAAAABAAEAAAJIAAAAAAAAAAAAAAAAAAAAAZQYXBlcnMAEAAIAADQXEFnAAAAEQAIAADSMHgCAAAAAQAUAApfEQAKXrsACl0CAAlEXgACk9UAAgBNTWFjaW50b3NoIEhEOlVzZXJzOgBCcmVjaHQ6AEdvb2dsZSBEcml2ZToARG9jdW1lbnRzOgBQYXBlcnM6AHdhYzE1X211c2hyYS5wZGYAAA4AIgAQAHcAYQBjADEANQBfAG0AdQBzAGgAcgBhAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgA7VXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3dhYzE1X211c2hyYS5wZGYAABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4AywDQANgCkgKUApkCpAKtArsCvwLGAs8C1ALhAuQC9gL5Av4AAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAADAA==}} + Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QOi4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3dhYzE1X211c2hyYS5wZGbSFwsYGVdOUy5kYXRhTxEBtgAAAAABtgACAAAMTWFjaW50b3NoIEhEAAAAAAAAAAAAAAAAAAAA0Fxdh0grAAAACl8REHdhYzE1X211c2hyYS5wZGYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAESrbDSMJQiAAAAAAAAAAAABAAEAAAJIAAAAAAAAAAAAAAAAAAAAAZQYXBlcnMAEAAIAADQXEFnAAAAEQAIAADSMHgCAAAAAQAUAApfEQAKXrsACl0CAAlEXgACk9UAAgBNTWFjaW50b3NoIEhEOlVzZXJzOgBCcmVjaHQ6AEdvb2dsZSBEcml2ZToARG9jdW1lbnRzOgBQYXBlcnM6AHdhYzE1X211c2hyYS5wZGYAAA4AIgAQAHcAYQBjADEANQBfAG0AdQBzAGgAcgBhAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgA7VXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3dhYzE1X211c2hyYS5wZGYAABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4AywDQANgCkgKUApkCpAKtArsCvwLGAs8C1ALhAuQC9gL5Av4AAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAADAA==}} @conference{ape, Author = {De Man, Brecht and Joshua D. Reiss}, Booktitle = {136th Convention of the AES}, Date-Added = {2015-09-29 17:07:16 +0000}, - Date-Modified = {2015-09-29 17:07:20 +0000}, + Date-Modified = {2016-02-29 10:30:11 +0000}, Keywords = {perceptual evaluation}, - Month = {April}, Read = {1}, Title = {{APE}: {A}udio {P}erceptual {E}valuation toolbox for {MATLAB}}, Year = {2014}, - Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QOi4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Xcml0aW5ncy9fcHVibGljYXRpb25zL2FlczEzNi5wZGbSFwsYGVdOUy5kYXRhTxEBsgAAAAABsgACAAAMTWFjaW50b3NoIEhEAAAAAAAAAAAAAAAAAAAA0FxPd0grAAAACl8UCmFlczEzNi5wZGYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKaS7PXG0EAAAAAAAAAAAABAAEAAAJIAAAAAAAAAAAAAAAAAAAAA1fcHVibGljYXRpb25zAAAQAAgAANBcQWcAAAARAAgAAM9cbQQAAAABABQACl8UAApeugAKXQIACUReAAKT1QACAE1NYWNpbnRvc2ggSEQ6VXNlcnM6AEJyZWNodDoAR29vZ2xlIERyaXZlOgBXcml0aW5nczoAX3B1YmxpY2F0aW9uczoAYWVzMTM2LnBkZgAADgAWAAoAYQBlAHMAMQAzADYALgBwAGQAZgAPABoADABNAGEAYwBpAG4AdABvAHMAaAAgAEgARAASADtVc2Vycy9CcmVjaHQvR29vZ2xlIERyaXZlL1dyaXRpbmdzL19wdWJsaWNhdGlvbnMvYWVzMTM2LnBkZgAAEwABLwAAFQACAA3//wAAgAbSGxwdHlokY2xhc3NuYW1lWCRjbGFzc2VzXU5TTXV0YWJsZURhdGGjHR8gVk5TRGF0YVhOU09iamVjdNIbHCIjXE5TRGljdGlvbmFyeaIiIF8QD05TS2V5ZWRBcmNoaXZlctEmJ1Ryb290gAEACAARABoAIwAtADIANwBAAEYATQBVAGAAZwBqAGwAbgBxAHMAdQB3AIQAjgDLANAA2AKOApAClQKgAqkCtwK7AsICywLQAt0C4ALyAvUC+gAAAAAAAAIBAAAAAAAAACgAAAAAAAAAAAAAAAAAAAL8}} + Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QOi4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Xcml0aW5ncy9fcHVibGljYXRpb25zL2FlczEzNi5wZGbSFwsYGVdOUy5kYXRhTxEBsgAAAAABsgACAAAMTWFjaW50b3NoIEhEAAAAAAAAAAAAAAAAAAAA0Fxdh0grAAAACl8UCmFlczEzNi5wZGYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKaS7PXHsUAAAAAAAAAAAABAAEAAAJIAAAAAAAAAAAAAAAAAAAAA1fcHVibGljYXRpb25zAAAQAAgAANBcQWcAAAARAAgAAM9cbQQAAAABABQACl8UAApeugAKXQIACUReAAKT1QACAE1NYWNpbnRvc2ggSEQ6VXNlcnM6AEJyZWNodDoAR29vZ2xlIERyaXZlOgBXcml0aW5nczoAX3B1YmxpY2F0aW9uczoAYWVzMTM2LnBkZgAADgAWAAoAYQBlAHMAMQAzADYALgBwAGQAZgAPABoADABNAGEAYwBpAG4AdABvAHMAaAAgAEgARAASADtVc2Vycy9CcmVjaHQvR29vZ2xlIERyaXZlL1dyaXRpbmdzL19wdWJsaWNhdGlvbnMvYWVzMTM2LnBkZgAAEwABLwAAFQACAA3//wAAgAbSGxwdHlokY2xhc3NuYW1lWCRjbGFzc2VzXU5TTXV0YWJsZURhdGGjHR8gVk5TRGF0YVhOU09iamVjdNIbHCIjXE5TRGljdGlvbmFyeaIiIF8QD05TS2V5ZWRBcmNoaXZlctEmJ1Ryb290gAEACAARABoAIwAtADIANwBAAEYATQBVAGAAZwBqAGwAbgBxAHMAdQB3AIQAjgDLANAA2AKOApAClQKgAqkCtwK7AsICywLQAt0C4ALyAvUC+gAAAAAAAAIBAAAAAAAAACgAAAAAAAAAAAAAAAAAAAL8}} @inproceedings{beaqlejs, Author = {Kraft, Sebastian and Z{\"o}lzer, Udo}, @@ -102,17 +100,17 @@ Keywords = {perceptual evaluation}, Title = {{BeaqleJS}: {HTML5} and {JavaScript} based framework for the subjective evaluation of audio quality}, Year = {2014}, - Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QQC4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3pvbHplcjIwMTRiZWFxbGVqcy5wZGbSFwsYGVdOUy5kYXRhTxEBzgAAAAABzgACAAAMTWFjaW50b3NoIEhEAAAAAAAAAAAAAAAAAAAA0FxPd0grAAAACl8RFnpvbHplcjIwMTRiZWFxbGVqcy5wZGYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACnAK7RX7izAAAAAAAAAAAABAAEAAAJIAAAAAAAAAAAAAAAAAAAAAZQYXBlcnMAEAAIAADQXEFnAAAAEQAIAADRX6qjAAAAAQAUAApfEQAKXrsACl0CAAlEXgACk9UAAgBTTWFjaW50b3NoIEhEOlVzZXJzOgBCcmVjaHQ6AEdvb2dsZSBEcml2ZToARG9jdW1lbnRzOgBQYXBlcnM6AHpvbHplcjIwMTRiZWFxbGVqcy5wZGYAAA4ALgAWAHoAbwBsAHoAZQByADIAMAAxADQAYgBlAGEAcQBsAGUAagBzAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgBBVXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3pvbHplcjIwMTRiZWFxbGVqcy5wZGYAABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4A0QDWAN4CsAKyArcCwgLLAtkC3QLkAu0C8gL/AwIDFAMXAxwAAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAADHg==}} + Bdsk-File-1 = {YnBsaXN0MDDUAQIDBAUGJCVYJHZlcnNpb25YJG9iamVjdHNZJGFyY2hpdmVyVCR0b3ASAAGGoKgHCBMUFRYaIVUkbnVsbNMJCgsMDxJXTlMua2V5c1pOUy5vYmplY3RzViRjbGFzc6INDoACgAOiEBGABIAFgAdccmVsYXRpdmVQYXRoWWFsaWFzRGF0YV8QQC4uLy4uLy4uLy4uL0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3pvbHplcjIwMTRiZWFxbGVqcy5wZGbSFwsYGVdOUy5kYXRhTxEBzgAAAAABzgACAAAMTWFjaW50b3NoIEhEAAAAAAAAAAAAAAAAAAAA0Fxdh0grAAAACl8RFnpvbHplcjIwMTRiZWFxbGVqcy5wZGYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACnAK7RX8bDAAAAAAAAAAAABAAEAAAJIAAAAAAAAAAAAAAAAAAAAAZQYXBlcnMAEAAIAADQXEFnAAAAEQAIAADRX6qjAAAAAQAUAApfEQAKXrsACl0CAAlEXgACk9UAAgBTTWFjaW50b3NoIEhEOlVzZXJzOgBCcmVjaHQ6AEdvb2dsZSBEcml2ZToARG9jdW1lbnRzOgBQYXBlcnM6AHpvbHplcjIwMTRiZWFxbGVqcy5wZGYAAA4ALgAWAHoAbwBsAHoAZQByADIAMAAxADQAYgBlAGEAcQBsAGUAagBzAC4AcABkAGYADwAaAAwATQBhAGMAaQBuAHQAbwBzAGgAIABIAEQAEgBBVXNlcnMvQnJlY2h0L0dvb2dsZSBEcml2ZS9Eb2N1bWVudHMvUGFwZXJzL3pvbHplcjIwMTRiZWFxbGVqcy5wZGYAABMAAS8AABUAAgAN//8AAIAG0hscHR5aJGNsYXNzbmFtZVgkY2xhc3Nlc11OU011dGFibGVEYXRhox0fIFZOU0RhdGFYTlNPYmplY3TSGxwiI1xOU0RpY3Rpb25hcnmiIiBfEA9OU0tleWVkQXJjaGl2ZXLRJidUcm9vdIABAAgAEQAaACMALQAyADcAQABGAE0AVQBgAGcAagBsAG4AcQBzAHUAdwCEAI4A0QDWAN4CsAKyArcCwgLLAtkC3QLkAu0C8gL/AwIDFAMXAxwAAAAAAAACAQAAAAAAAAAoAAAAAAAAAAAAAAAAAAADHg==}} @article{lipshitz1981great, Author = {Lipshitz, Stanley P and Vanderkooy, John}, Date-Added = {2015-09-23 09:09:51 +0000}, - Date-Modified = {2015-09-23 09:09:51 +0000}, + Date-Modified = {2016-02-29 10:27:05 +0000}, Journal = {Journal of the AES}, Number = {7/8}, Pages = {482--491}, Publisher = {Audio Engineering Society}, - Title = {The great debate: Subjective evaluation}, + Title = {{The Great Debate}: Subjective evaluation}, Volume = {29}, Year = {1981}} @@ -139,12 +137,12 @@ @article{pascoe1983evaluation, Author = {Pascoe, Gregory C and Attkisson, C Clifford}, Date-Added = {2015-09-23 08:59:38 +0000}, - Date-Modified = {2015-09-23 08:59:38 +0000}, + Date-Modified = {2016-02-29 10:41:55 +0000}, Journal = {Evaluation and program planning}, Number = {3}, Pages = {335--347}, Publisher = {Elsevier}, - Title = {The evaluation ranking scale: a new methodology for assessing satisfaction}, + Title = {The evaluation ranking scale: {A} new methodology for assessing satisfaction}, Volume = {6}, Year = {1983}} @@ -169,9 +167,9 @@ @article{likert1932technique, Author = {Likert, Rensis}, Date-Added = {2015-09-23 08:49:36 +0000}, - Date-Modified = {2015-09-23 08:49:36 +0000}, - Journal = {Archives of psychology}, - Title = {A technique for the measurement of attitudes.}, + Date-Modified = {2016-02-29 10:42:48 +0000}, + Journal = {Archives of Psychology}, + Title = {A technique for the measurement of attitudes}, Year = {1932}} @book{nunnally1967psychometric, @@ -184,41 +182,42 @@ Year = {1967}} @article{recommendation19971116, - Author = {{ITURBS Recommendation}}, Date-Added = {2015-09-23 08:36:37 +0000}, - Date-Modified = {2015-09-23 08:36:37 +0000}, - Journal = {International Telecommunication Union, Geneva}, - Title = {1116-1: Methods for the subjective assessment of small impairments in audio systems including multichannel sound systems}, + Date-Modified = {2016-02-29 10:37:34 +0000}, + Journal = {International Telecommunication Union}, + Keywords = {standard}, + Title = {{Recommendation ITU-R BS. 1116-1}: {M}ethods for the subjective assessment of small impairments in audio systems including multichannel sound systems}, Year = {1997}} @article{recommendation20031534, - Author = {{ITURBS Recommendation}}, Date-Added = {2015-09-23 08:34:26 +0000}, - Date-Modified = {2015-09-23 08:34:41 +0000}, + Date-Modified = {2016-02-29 10:38:07 +0000}, Journal = {International Telecommunication Union}, - Title = {BS. 1534-1: Method for the subjective assessment of intermediate quality levels of coding systems}, + Keywords = {standard}, + Title = {Recommendation {ITU}-{R} {BS}.1534-1: {M}ethod for the subjective assessment of intermediate quality levels of coding systems}, Year = {2003}} - + @article{loudness201510, - Author ={{ITURBS Recommendation}}, + Date-Modified = {2016-02-29 10:38:52 +0000}, Journal = {International Telecommunication Union}, - Title = {BS. 1770 : Algorithms to measure audio programme loudness and true-peak audio level}, + Keywords = {standard}, + Title = {Recommendation {ITU}-{R} {BS}.1770-4: {A}lgorithms to measure audio programme loudness and true-peak audio level}, Year = {2015}} @article{recommendation2001bs, - Author = {{ITUR Recommendation}}, Date-Added = {2015-09-23 08:33:38 +0000}, - Date-Modified = {2015-09-23 08:33:38 +0000}, - Journal = {International Telecommunications Union, Geneva}, - Title = {BS. 1534-1. Method for the Subjective Assessment of Intermediate Sound Quality (MUSHRA)}, + Date-Modified = {2016-02-29 10:40:39 +0000}, + Journal = {International Telecommunication Union}, + Keywords = {standard}, + Title = {{Recommendation ITU-R BS. 1534-1}: {M}ethod for the Subjective Assessment of Intermediate Sound Quality ({MUSHRA})}, Year = {2001}} @article{rec1997bs, - Author = {{ITUR Recommendation}}, Date-Added = {2015-09-23 08:32:42 +0000}, - Date-Modified = {2015-09-23 08:32:42 +0000}, - Journal = {International Telecommunications Union}, - Title = {BS. 562-3,`Subjective Assessment of Sound Quality'}, + Date-Modified = {2016-02-29 10:40:47 +0000}, + Journal = {International Telecommunication Union}, + Keywords = {standard}, + Title = {{Recommendation ITU-R BS. 562-3}: {S}ubjective Assessment of Sound Quality}, Year = {1997}} @article{peryam1952advanced, @@ -233,28 +232,26 @@ Year = {1952}} @article{rec1996p, - Author = {{ITUT Recommendation}}, Date-Added = {2015-09-23 08:30:24 +0000}, - Date-Modified = {2015-09-23 08:30:24 +0000}, - Journal = {International Telecommunication Union, Geneva}, - Title = {P. 800: Methods for subjective determination of transmission quality}, + Date-Modified = {2016-02-29 10:40:26 +0000}, + Journal = {International Telecommunication Union}, + Keywords = {standard}, + Title = {{Recommendation ITU-T P. 800}: {M}ethods for subjective determination of transmission quality}, Year = {1996}} @inproceedings{hultigen, Author = {Gribben, Christopher and Lee, Hyunkook}, Booktitle = {AES Convention 138}, Date-Added = {2015-09-23 08:11:17 +0000}, - Date-Modified = {2015-09-29 16:23:17 +0000}, - Organization = {Audio Engineering Society}, - Title = {Toward the Development of a Universal Listening Test Interface Generator in Max}, + Date-Modified = {2016-02-29 10:29:19 +0000}, + Title = {Toward the Development of a Universal Listening Test Interface Generator in {Max}}, Year = {2015}} @conference{waet, Author = {Nicholas Jillings and David Moffat and De Man, Brecht and Joshua D. Reiss}, Booktitle = {12th Sound and Music Computing Conference}, Date-Added = {2015-09-22 16:48:27 +0000}, - Date-Modified = {2015-09-22 16:48:33 +0000}, - Month = {July}, + Date-Modified = {2016-02-29 10:30:22 +0000}, Read = {1}, Title = {Web {A}udio {E}valuation {T}ool: {A} browser-based listening test environment}, Year = {2015}} diff -r 80e6cf42a17b -r 49d27886aa03 docs/WAC2016/WAC2016.pdf Binary file docs/WAC2016/WAC2016.pdf has changed diff -r 80e6cf42a17b -r 49d27886aa03 docs/WAC2016/WAC2016.tex --- a/docs/WAC2016/WAC2016.tex Thu Feb 25 11:57:24 2016 +0000 +++ b/docs/WAC2016/WAC2016.tex Mon Feb 29 11:45:22 2016 +0100 @@ -13,6 +13,21 @@ \newcommand*\rot{\rotatebox{90}} +% Make clickable footnote (Brecht) +\newcommand{\hyperfootnote}[1][]{\def\ArgI{{#1}}\hyperfootnoteRelay} + % relay to new command to make extra optional command possible +\newcommand\hyperfootnoteRelay[2][]{\href{#1#2}{\ArgI}\footnote{\href{#1#2}{#2}}} + % the first optional argument is now in \ArgI, the second is in #1 + +% Takes at most 3 parameters (see http://www.tex.ac.uk/FAQ-twooptarg.html for info on multiple optional parameters) +% If first parameter isn't given, it's value is '' (empty string in text before footnote reference) +% If second parameter isn't given, it's value is '' (string before visible URL, e.g. 'http://') +% Makes a clickable footnote (alternatively: \url{}) with optional reference in the text as well +% Use 1: \hyperfootnote{www.mywebsite.com}: creates a footnote consisting of a clickable URL +% Use 2: \hyperfootnote[My website]{www.mywebsite.com}: creates a clickable piece of text in the text ('My website') plus a footnote consisting of a clickable URL +% Note: requires the hyperref package. +% Note: using xspace package to add/absorb spaces when necessary (e.g. to avoid a space between the footnote number and a punctuation mark) + %% DOI %\doi{10.475/123_4} % @@ -133,9 +148,8 @@ \maketitle \begin{abstract} - -Perceptual listening tests are commonplace in audio research and a vital form of evaluation. Many tools exist to run such tests, however many operate one test type and are therefore limited whilst most require proprietary software. Using Web Audio the Web Audio Evaluation Tool (WAET) addresses these concerns by having one toolbox which can be configured to run many different tests, perform it through a web browser and without needing proprietary software or computer programming knowledge. In this paper the role of the Web Audio API in giving WAET key functionalities are shown. The paper also highlights less common features, available to web based tools, such as easy remote testing environment and in-browser analytics. - +Perceptual listening tests are commonplace in audio research and a vital form of evaluation. % ? +While a large number of tools exist to run such tests, many feature just one test type, are platform dependent, run on proprietary software, or require considerable configuration and programming. Using Web Audio, the Web Audio Evaluation Tool (WAET) addresses these concerns by having one toolbox which can be configured to run many different tests, perform it through a web browser and without needing proprietary software or computer programming knowledge. In this paper the role of the Web Audio API in giving WAET key functionalities are shown. The paper also highlights less common features, available to web based tools, such as easy remote testing environment and in-browser analytics. \end{abstract} @@ -143,7 +157,7 @@ % Listening tests/perceptual audio evaluation: what are they, why are they important % As opposed to limited scope of WAC15 paper: also musical features, realism of sound effects / sound synthesis, performance of source separation and other algorithms... - Perceptual evaluation of audio, using listening tests, is a powerful way to assess anything from audio codec quality to realism of sound synthesis to the performance of source separation, automated music production and other auditory evaluations. + Perceptual evaluation of audio, using listening tests, is a powerful way to assess anything from audio codec quality over realism of sound synthesis to the performance of source separation, automated music production and other auditory evaluations. In less technical areas, the framework of a listening test can be used to measure emotional response to music or test cognitive abilities. % maybe some references? If there's space. @@ -151,14 +165,14 @@ % Why difficult? Challenges? What constitutes a good interface? % Technical, interfaces, user friendliness, reliability - Several applications for performing perceptual listening tests currently exist presented in~\Cref{tab:toolboxes}. Many rely on proprietary, 3rd party software such as MATLAB and MAX, making them less attractive for many. With the exception of the existing JavaScript-based toolboxes, remote deployment (web-based test hosting and result collection) is not possible. + Several applications for performing perceptual listening tests currently exist, see~\Cref{tab:toolboxes}. Many rely on proprietary, third party software such as MATLAB and Max, making them less attractive for many. With the exception of the existing JavaScript-based toolboxes, remote deployment (web-based test hosting and result collection) is not possible. - HULTI-GEN~\cite{hultigen} is a single example of a toolbox that presents the user with a large number of different test interfaces and customisation, without requiring knowledge of any programming language. The Web Audio Evaluation Toolbox (WAET), presented here, stands out for the same reasons and does not require proprietary software or a specific platform. It also provides a wide range of interface and test types in one user friendly environment. Furthermore any test based on the default test types can be configured in the browser as well. Note that the design of an effective listening test further poses many challenges unrelated to interface design, which are beyond the scope of this paper \cite{bech}. + HULTI-GEN~\cite{hultigen} is an example of a toolbox that presents the user with a large number of different test interfaces and customisation, without requiring knowledge of any programming language. The Web Audio Evaluation Toolbox (WAET), presented here, stands out for the same reasons but in addition does not require proprietary software or a specific platform. It also provides a wide range of interface and test types in one user friendly environment. Furthermore, any test based on the default test types can be configured in the browser as well. Note that the design of an effective listening test further poses many challenges unrelated to interface design, which are beyond the scope of this paper \cite{bech}. % Why in the browser? The Web Audio API provides important features including sample level manipulation of audio streams \cite{schoeffler2015mushra} and synchronous and flexible playback. Operating in the browser allows leveraging the flexible JavaScript language and native support for web documents, such as the extensible markup language (XML) which is used for configuration and test result files. Using the web also reduces deployment requirements to a basic web server with extra functionality, such as test collection and automatic processing, using PHP. As recruiting participants can be very time-consuming, and as for some tests a large number of participants is needed, browser-based tests can enable participants in multiple locations to perform the test simultaneously \cite{schoeffler2015mushra}. - Both BeaqleJS \cite{beaqlejs} and mushraJS\footnote{https://github.com/akaroice/mushraJS} also operate in the browser. However, BeaqleJS does not make use of the Web Audio API and therefore lacks arbitrary manipulation of audio stream samples, and neither offer an adequately wide choice of test designs for them to be useful to many researchers. %requires programming knowledge?... + Both BeaqleJS \cite{beaqlejs} and \hyperfootnote[mushraJS][https://]{github.com/akaroice/mushraJS} also operate in the browser. However, BeaqleJS does not make use of the Web Audio API and therefore lacks arbitrary manipulation of audio stream samples, and neither offer an adequately wide choice of test designs for them to be useful to many researchers. %requires programming knowledge?... % only browser-based? \begin{table*}[ht] @@ -191,8 +205,10 @@ n-Alternative Forced Choice & & & & & & \checkmark & & \\ \hline \end{tabular} \end{center} + \vspace{-.5cm} \label{tab:toolboxes} \end{table*} + % %Selling points: remote tests, visualisaton, create your own test in the browser, many interfaces, few/no dependencies, flexibility @@ -240,19 +256,19 @@ Although WAET uses a sparse subset of the Web Audio API functionality, its performance comes directly from it. Listening tests can convey large amounts of information other than obtaining the perceptual relationship between the audio fragments. With WAET it is possible to track which parts of the audio fragments were listened to and when, at what point in the audio stream the participant switched to a different fragment, and how a fragment's rating was adjusted over time within a session, to name a few. Not only does this allow evaluation of a wealth of perceptual aspects, but it also helps detect poor participants whose results are potentially not representative. - One of the key initial design parameters for WAET was to make the tool as open as possible to non-programmers and to this end all of the user modifiable options are included in a single XML document, referred to as the specification document, that can be written manually (or modifying an existing document or template) or using the included test creator. The test creator can modify existing specification documents or generate new ones in a user friendly environment. This simplifies the creation of elements by visualising the data structure with explanatory text. + One of the key initial design parameters for WAET was to make the tool as open as possible to non-programmers. To this end, all of the user modifiable options are included in a single XML document, referred to as the specification document, that can be written manually (or modifying an existing document or template) or using the included test creator. The test creator can modify existing specification documents or generate new ones in an intuitive yet powerful HTML GUI. This simplifies the creation of elements by visualising the data structure with explanatory text. %Describe and/or visualise audioholder-audioelement-... structure. - The specification document contains the URL of the audio fragments for each test page. These fragments are downloaded asynchronously in the test and decoded offline by the Web Audio offline decoder. The LUFS integrated loudness of the buffers are calculated \cite{loudness201510} and stored to enable on-the-fly loudness normalisation. Equally if the playback uses synchronous looping, the buffers are zero-padded accordingly. Performing these in the browser removes any pre-processing. The resulting buffers are assigned to a custom Audio Objects node which tracks the fragment buffer, the Web Audio \textit{bufferSourceNode}, and other specification attributes including its ID, the interface object(s) associated with the fragment and any metric or data collection objects. The Audio Object is controlled by an over-arching custom Audio Engine node allowing for session wide control of the Audio Objects. + The specification document contains the URL of the audio fragments for each test page. These fragments are downloaded asynchronously in the test and decoded offline by the Web Audio offline decoder. The LUFS integrated loudness of the buffers are calculated \cite{loudness201510} and stored to enable on-the-fly loudness normalisation. If the playback uses synchronous looping, the buffers are zero-padded accordingly. Performing these in the browser removes any need for pre-processing. The resulting buffers are assigned to a custom Audio Objects node which tracks the fragment buffer, the Web Audio \textit{bufferSourceNode}, and other specification attributes including its ID, the interface object(s) associated with the fragment and any metric or data collection objects. The Audio Object is controlled by an over-arching custom Audio Engine node allowing for session wide control of the Audio Objects. - The only significant issue with this model is the \textit{bufferNode} in the Web Audio API, implemented in the standard as a `use once' object. Once the node has been played it must be discarded as it cannot be instructed to play again. Therefore on each play request the \textit{bufferSourceNode} must be created and then linked with the stored \textit{bufferNode}. This is an odd behaviour with no alternative except to use the HTML5 audio element, but they do not have the ability to synchronously start on a given time and therefore not suited. + The only significant issue with this model is the \textit{bufferNode} in the Web Audio API, implemented in the standard as a `use once' object. Once the node has been played, it must be discarded as it cannot be instructed to play again. Therefore on each play request the \textit{bufferSourceNode} must be created and then linked with the stored \textit{bufferNode}. This is an odd behaviour with no alternative except to use the HTML5 audio element, but they do not have the ability to synchronously start on a given time and therefore not suited. - In the test, each buffer node is connected to a gain node configured by the loudness normalisation and any user specified gain. Therefore it is possible to perform a `Method of Adjustment' test where an interface could directly manipulate these gain nodes. These gain nodes are used for cross-fading between samples when operating in synchronous playback. Cross-fading can either be fade-out fade-in or a true cross-fade. This is achieved by using the AudioParam controls to provide linear ramping from 0 to the calculated playback level. There is also an optional `Master Volume' slider which can be shown on the test GUI which modifies a gain node before the destination. The controls' position is tracked providing extra test use validation. This is not indicative of the final volume exiting the speakers, not least because the browser cannot read the system volume. Therefore its use should only be considered in a lab environment to ensure results are representative. + In the test, each buffer node is connected to a gain node configured by the loudness normalisation and any user specified gain. Therefore it is possible to perform a `Method of Adjustment' test where an interface could directly manipulate these gain nodes. These gain nodes are used for cross-fading between samples when operating in synchronous playback. Cross-fading can either be fade-out followed by a fade-in, or a true cross-fade. This is achieved by using the AudioParam controls to provide linear ramping from 0 to the calculated playback level. There is also an optional `Master Volume' slider which can be shown on the test GUI to modify a gain node before the destination. The control's position is tracked providing extra test use validation. This is not indicative of the final volume exiting the speakers, though, not least because the browser cannot read the system volume. Therefore its use should only be considered in a lab environment to ensure results are representative. %Which type of files? WAV, anything else? Perhaps not exhaustive list, but say something along the lines of 'whatever browser supports'. Compatability? - The media files supported depend on the browser level support for the initial decoding of information and is the same as the browser support for the HTML5 audio element. The most widely supported media file is the wave (.WAV) format which is accepted by every browser supporting the Web Audio API. Most browsers support floating point WAV except Firefox. To resolve this the tool includes its own wave file decoder to extract the samples. The toolbox works in any browser which supports the Web Audio API and HTML 5. + The media files supported depend on the browser level support for the initial decoding of information and is the same as the browser support for the HTML5 audio element. The most widely supported media file is the wave (.WAV) format which is accepted by every browser supporting the Web Audio API. Most browsers support floating point WAV except Firefox. To resolve this, the tool includes its own wave file decoder to extract the samples. The toolbox works in any browser which supports the Web Audio API and HTML 5. - All the collected session data is returned in an XML document structured similarly to the configuration document, where test pages contain the audio elements with their trace collection, results, comments and any interface specific data points. + All collected session data is returned in an XML document structured similarly to the configuration document, where test pages contain the audio elements with their trace collection, results, comments and any interface-specific data points. \section{Remote tests} % with previous? \label{sec:remote} @@ -261,10 +277,10 @@ Furthermore, a fully browser-based test, where the collection of the results is automatic, is more efficient and technically reliable even when the test still takes place under lab conditions. The following features allow easy and effective remote testing: - \begin{description}[noitemsep,nolistsep] - \item[PHP script to collect result XML files] and store on central server. - \item[Randomly pick a specified number of pages] to ensure an equal and randomised spread of the different pages (`audioHolders') across participants. - \item[Calibration of the sound system (and participant)] by a perceptual pre-test to gather information about the frequency response and speaker configuration - this can be supplemented with a survey. + \begin{itemize}[noitemsep,nolistsep] % replaced description list with itemize as it went outside the margins + \item \textbf{PHP script to collect result XML files} and store on central server. + \item \textbf{Randomly pick a specified number of pages} to ensure an equal and randomised spread of the different pages across participants. + \item \textbf{Calibration of the sound system (and participant)} by a perceptual pre-test to gather information about the frequency response and speaker configuration - this can be supplemented with a survey. % In theory calibration could be applied anywhere?? % \item Functionality to participate multiple times % \begin{itemize}[noitemsep,nolistsep] @@ -275,28 +291,29 @@ % \item Pick `new' audioholders if available % \item Copy survey information first time to new XMLs % \end{itemize} - \item[Intermediate saves] for tests which were interrupted or unfinished. - \item[Collect IP address information] for geographic location, through PHP function which grabs address and appends to XML file. - \item[Collect Browser and Display information] to the extent it is available and reliable. - \end{description} + \item \textbf{Intermediate saves} for tests which were interrupted or unfinished. + \item \textbf{Collect IP address information} for geographic location, through PHP function which grabs address and appends to XML file. + \item \textbf{Collect browser and display information} to the extent it is available and reliable. + \end{itemize} \section{Interfaces} % title? 'Front end'? % Dave \label{sec:interfaces} -The purpose of this listening test framework is to allow any user the maximum flexibility to design a listening test for their exact application with minimum effort. To this end, a large range of standard listening test interfaces have been implemented including: % pretty much the same wording as two sentences earlier +The purpose of this listening test framework is to allow any user the maximum flexibility to design a listening test for their exact application with minimum effort. To this end, a large range of standard listening test interfaces have been implemented, including \begin{itemize}[noitemsep,nolistsep] - \item AB Test~\cite{lipshitz1981great}: Two stimuli presented at a time, participant selects a preferred stimulus. - \item ABC/HR (ITU-R BS. 1116)~\cite{recommendation19971116} (Mean Opinion Score: MOS): each stimulus has a continuous scale (5-1), labeled as Imperceptible, Perceptible but not annoying, slightly annoying, annoying, very annoying. + \item AB Test~\cite{lipshitz1981great}: Two stimuli presented simultaneously, participant selects a preferred stimulus. + \item ABC/HR (ITU-R BS. 1116)~\cite{recommendation19971116} (Mean Opinion Score: MOS): each stimulus has a continuous scale (5-1), labeled as Imperceptible, Perceptible but not annoying, Slightly annoying, Annoying, Very annoying. \item -50 to 50 Bipolar with Ref: each stimulus has a continuous scale -50 to 50 with default values as 0 in middle and a reference. \item Absolute Category Rating (ACR) Scale~\cite{rec1996p}: Likert but labels are Bad, Poor, Fair, Good, Excellent \item ABX Test~\cite{clark1982high}: Two stimuli are presented along with a reference and the participant has to select a preferred stimulus, often the closest to the reference. - \item APE style \cite{ape}: Multiple stimuli as points on a 2D plane for inter-sample rating (eg. Valence Arousal) - \item Comparison Category Rating (CCR) Scale~\cite{rec1996p}: ACR \& DCR but 7 point scale: Much Better, Better, Slightly Better, About the same, slightly worse, worse, much worse. There is also a provided reference. - \item Degredation Category Rating (DCR) Scale~\cite{rec1996p}: ABC \& Likert but labels are (5) Inaudible, (4) Audible but not annoying, (3) slightly annoying, (2) annoying, (1) very annoying. + \item APE style \cite{ape}: Multiple stimuli on one or more axes for inter-sample rating. + \item APE style 2D \cite{ape}: Multiple stimuli on a 2D plane for inter-sample rating (e.g. Valence Arousal). + \item Comparison Category Rating (CCR) Scale~\cite{rec1996p}: ACR \& DCR but 7 point scale, with reference: Much better, Better, Slightly better, About the same, Slightly worse, Worse, Much worse. + \item Degredation Category Rating (DCR) Scale~\cite{rec1996p}: ABC \& Likert but labels are (5) Inaudible, (4) Audible but not annoying, (3) Slightly annoying, (2) Annoying, (1) Very annoying. \item ITU-R 5 Point Continuous Impairment Scale~\cite{rec1997bs}: Same as ABC/HR but with a reference. - \item Likert scale~\cite{likert1932technique}: each stimuli has a five point scale with values: Strongly Agree, Agree, Neutral, Disagree and Strongly Disagree. - \item MUSHRA (ITU-R BS. 1534)~\cite{recommendation20031534} + \item Likert scale~\cite{likert1932technique}: each stimulus has a five point scale with values: Strongly agree, Agree, Neutral, Disagree and Strongly disagree. + \item MUSHRA (ITU-R BS. 1534)~\cite{recommendation20031534} \begin{comment} \begin{itemize}[noitemsep,nolistsep] \item Multiple stimuli are presented and rated on a continuous scale, which includes a reference, hidden reference and hidden anchors. @@ -304,12 +321,12 @@ \end{comment} \item Pairwise Comparison (Better/Worse)~\cite{david1963method}: every stimulus is rated as being either better or worse than the reference. \item Rank Scale~\cite{pascoe1983evaluation}: stimuli ranked on single horizontal scale, where they are ordered in preference order. - \item 9 Point Hedonic Category Rating Scale~\cite{peryam1952advanced}: each stimuli has a seven point scale with values: Like Extremely, Like Very Much, Like Moderate, Like Slightly, Neither Like nor Dislike, dislike Extremely, dislike Very Much, dislike Moderate, dislike Slightly. There is also a provided reference. + \item 9 Point Hedonic Category Rating Scale~\cite{peryam1952advanced}: each stimulus has a seven point scale with values: Like extremely, Like very much, Like moderate, Like slightly, Neither like nor dislike, Dislike extremely, Dislike very much, Dislike moderate, Dislike slightly. There is also a provided reference. \end{itemize} - It is possible to include any number of references, hidden references and hidden anchors into all of these listening test formats. + It is possible to include any number of references, hidden references, hidden anchors and comment fields into all of these listening test formats. - Because of the design to have separate core code and interface modules, it is possible for a 3rd party interface to be built with minimal effort. The repository includes a boilerplate (blank.js) and documentation on which functions must be called and the specific functions they expect your interface to perform. The core includes an `Interface' object which includes object prototypes for the on-page comment boxes (including those with radio or checkbox responses), start and stop buttons and the playhead / transport bars. + Because of the design to have separate core code and interface modules, it is possible for a third party interface to be built with minimal effort. The repository includes a boilerplate (blank.js) and documentation on which functions must be called and the specific functions they expect your interface to perform. The core includes an `Interface' object which includes object prototypes for the on-page comment boxes (including those with radio or checkbox responses), start and stop buttons and the playhead / transport bars. %%%% \begin{itemize}[noitemsep,nolistsep] %%%% \item (APE style) \cite{ape} @@ -368,19 +385,13 @@ \label{sec:analysis} % don't mention Python scripts There are several benefits to providing basic analysis tools in the browser: they allow diagnosing problems, with the interface or with the test subject; they may be sufficient for many researchers' purposes; and test subjects may enjoy seeing an overview of their own results and/or results thus far at the end of their tests. - \begin{figure}[bh] - \centering - \includegraphics[width=.5\textwidth]{boxplot.png} - %\caption{This timeline of a single subject's listening test shows playback of fragments (red segments) and marker movements on the rating axis in function of time. } - \caption{Box and whisker plot showing the aggregated numerical ratings of six stimuli by a group of subjects.} - \label{fig:timeline} - \end{figure} + For this reason, we include a proof-of-concept web page with: \begin{itemize}[noitemsep,nolistsep] - \item All audioholder IDs, file names, subject IDs, audio element IDs, ... in the collected XMLs so far (\texttt{saves/*.xml}) + \item All page IDs, file names, subject IDs, audio element IDs, ... in the collected XMLs so far \item Selection of subjects and/or test samples to zoom in on a subset of the data %Check/uncheck each of the above for analysis (e.g. zoom in on a certain song, or exclude a subset of subjects) \item Embedded audio to hear corresponding test samples % (follow path in XML setup file, which is also embedded in the XML result file) - \item Scatter plot, confidence plot and box plot of rating values (see Figure ) + \item Scatter plot, confidence plot and box plot of rating values (see Figure \ref{fig:boxplot}) \item Timeline for a specific subject %(see Figure \ref{fig:timeline})%, perhaps re-playing the experiment in X times realtime. (If actual realtime, you could replay the audio...) \item Distribution plots of any radio button and number questions in pre- and post-test survey %(drop-down menu with `pretest', `posttest', ...; then drop-down menu with question `IDs' like `gender', `age', ...; make pie chart/histogram of these values over selected range of XMLs) \item All `comments' on a specific audioelement @@ -388,16 +399,23 @@ %\item Validation of setup XMLs (easily spot `errors', like duplicate IDs or URLs, missing/dangling tags, ...) \end{itemize} + \begin{figure}[tbh] + \centering + \includegraphics[width=.5\textwidth]{boxplot.png} + %\caption{This timeline of a single subject's listening test shows playback of fragments (red segments) and marker movements on the rating axis in function of time. } + \caption{Box and whisker plot showing the aggregated numerical ratings of six stimuli by a group of subjects.} + \label{fig:boxplot} + \end{figure} %A subset of the above would already be nice for this paper. -\section{Concluding remarks and future work} +\section{Concluding remarks} \label{sec:conclusion} We have developed a browser-based tool for the design and deployment of listening tests, requiring no programming experience or proprietary software. Following the predictions or guidelines in \cite{schoeffler2015mushra}, it supports remote testing, cross-fading between audio streams, collecting information about the system, among others. - Whereas many other types of interfaces do exist, we felt that supporting e.g. a range of `method of adjustment' tests would be beyond the scope of a tool that aims to be versatile enough while not claiming to support any custom experiment one might want to set up. Rather, it supports many non-adaptive listening test up to multi-stimulus, multi-attribute evaluation including references, anchors, text boxes, radio buttons and/or checkboxes, with arbitrary placement of the various UI elements. + Whereas many other types of interfaces do exist, we felt that supporting e.g.~a range of `method of adjustment' tests would be beyond the scope of a tool that aims to be versatile enough while not claiming to support any custom experiment one might want to set up. Rather, it supports intuitive creation of non-adaptive listening tests up to multi-stimulus, multi-attribute evaluation including references, anchors, text boxes, radio buttons and/or checkboxes, with arbitrary placement of the various UI elements, and many standard test `presets' already available. - The code and documentation can be downloaded from our online repository available at \url{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}. + The code and documentation can be downloaded from the \hyperfootnote[SoundSoftware repository][https://]{code.soundsoftware.ac.uk/projects/webaudioevaluationtool}. % remote % language support (not explicitly stated) % crossfades