# HG changeset patch # User Lucas Thompson # Date 1481503435 0 # Node ID 57980a1a579d4186875a98fd003aa6bf773b5a4c # Parent 05e9cec6d20a1434eea212e761ba4cb33cbe54ec# Parent 270f59ef3b83ebc2979dd9ebbab164b5cd755520 Merge pull request #2 from LucasThompson/feature/basic-offline-extraction Feature/basic offline extraction diff -r 05e9cec6d20a -r 57980a1a579d src/app/app.component.html --- a/src/app/app.component.html Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/app.component.html Mon Dec 12 00:43:55 2016 +0000 @@ -22,9 +22,13 @@ - + - + diff -r 05e9cec6d20a -r 57980a1a579d src/app/app.component.ts --- a/src/app/app.component.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/app.component.ts Mon Dec 12 00:43:55 2016 +0000 @@ -1,5 +1,7 @@ import {Component} from '@angular/core'; import {AudioPlayerService} from "./services/audio-player/audio-player.service"; +import {FeatureExtractionService} from "./services/feature-extraction/feature-extraction.service"; +import {ExtractorOutputInfo} from "./feature-extraction-menu/feature-extraction-menu.component"; @Component({ selector: 'app-root', @@ -8,10 +10,15 @@ }) export class AppComponent { audioBuffer: AudioBuffer; // TODO consider revising + canExtract: boolean; - constructor(private audioService: AudioPlayerService) {} + constructor(private audioService: AudioPlayerService, + private piperService: FeatureExtractionService) { + this.canExtract = false; + } onFileOpened(file: File) { + this.canExtract = false; const reader: FileReader = new FileReader(); const mimeType = file.type; reader.onload = (event: any) => { @@ -21,8 +28,27 @@ // TODO use a rxjs/Subject instead? this.audioService.decodeAudioData(event.target.result).then(audioBuffer => { this.audioBuffer = audioBuffer; + if (this.audioBuffer) + this.canExtract = true; }); }; reader.readAsArrayBuffer(file); } + + extractFeatures(outputInfo: ExtractorOutputInfo): void { + if (!this.canExtract || !outputInfo) return; + this.canExtract = false; + this.piperService.collect({ + audioData: [...Array(this.audioBuffer.numberOfChannels).keys()] + .map(i => this.audioBuffer.getChannelData(i)), + audioFormat: { + sampleRate: this.audioBuffer.sampleRate, + channelCount: this.audioBuffer.numberOfChannels + }, + key: outputInfo.extractorKey, + outputId: outputInfo.outputId + }).then(() => { + this.canExtract = true; + }).catch(err => console.error(err)); + } } diff -r 05e9cec6d20a -r 57980a1a579d src/app/feature-extraction-menu/feature-extraction-menu.component.css --- a/src/app/feature-extraction-menu/feature-extraction-menu.component.css Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/feature-extraction-menu/feature-extraction-menu.component.css Mon Dec 12 00:43:55 2016 +0000 @@ -0,0 +1,5 @@ +#extractor-outputs { + max-width: 80%; + display: block; + margin: 0 auto; +} diff -r 05e9cec6d20a -r 57980a1a579d src/app/feature-extraction-menu/feature-extraction-menu.component.html --- a/src/app/feature-extraction-menu/feature-extraction-menu.component.html Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/feature-extraction-menu/feature-extraction-menu.component.html Mon Dec 12 00:43:55 2016 +0000 @@ -1,7 +1,12 @@ - - - +

+ +

diff -r 05e9cec6d20a -r 57980a1a579d src/app/feature-extraction-menu/feature-extraction-menu.component.ts --- a/src/app/feature-extraction-menu/feature-extraction-menu.component.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/feature-extraction-menu/feature-extraction-menu.component.ts Mon Dec 12 00:43:55 2016 +0000 @@ -1,8 +1,10 @@ -import {Component, OnInit} from '@angular/core'; +import {Component, OnInit, Output, EventEmitter, Input} from '@angular/core'; import {FeatureExtractionService} from "../services/feature-extraction/feature-extraction.service"; -interface ExtractorInfo { - key: string; +export interface ExtractorOutputInfo { + extractorKey: string; + combinedKey: string; + outputId: string; name: string; } @@ -13,20 +15,53 @@ }) export class FeatureExtractionMenuComponent implements OnInit { - extractors: ExtractorInfo[]; + @Input() + set disabled(isDisabled: boolean) { + this.isDisabled = isDisabled; + } + + get disabled() { + return this.isDisabled; + } + + @Output() requestOutput: EventEmitter; + + private isDisabled: boolean; + private extractorsMap: Map; + extractors: Iterable; constructor(private piperService: FeatureExtractionService) { + this.extractorsMap = new Map(); this.extractors = []; + this.requestOutput = new EventEmitter(); + this.isDisabled = true; } ngOnInit() { this.piperService.list().then(available => { - available.available.forEach(staticData => this.extractors.push({ - key: staticData.key, - name: staticData.basic.name - }) - ); + const maxCharacterLimit = 50; + available.available.forEach(staticData => { + const isSingleOutputExtractor = staticData.basicOutputInfo.length === 1; + staticData.basicOutputInfo.forEach(output => { + const combinedKey = `${staticData.key}:${output.identifier}`; + this.extractorsMap.set(combinedKey, { + extractorKey: staticData.key, + combinedKey: combinedKey, + name: ( + isSingleOutputExtractor + ? staticData.basic.name + : `${staticData.basic.name}: ${output.name}` + ).substr(0, maxCharacterLimit) + '...', + outputId: output.identifier + }); + }); + }); + this.extractors = [...this.extractorsMap.values()]; }); } + extract(combinedKey: string): void { + this.requestOutput.emit(this.extractorsMap.get(combinedKey)); + } + } diff -r 05e9cec6d20a -r 57980a1a579d src/app/playback-control/playback-control.component.ts --- a/src/app/playback-control/playback-control.component.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/playback-control/playback-control.component.ts Mon Dec 12 00:43:55 2016 +0000 @@ -13,9 +13,7 @@ private featureExtractionService: FeatureExtractionService) { } - ngOnInit() { - this.featureExtractionService.testMessageStream(); - } + ngOnInit() {} emitPlayPause() { this.audioService.togglePlaying(); diff -r 05e9cec6d20a -r 57980a1a579d src/app/services/audio-player/audio-player.service.ts --- a/src/app/services/audio-player/audio-player.service.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/services/audio-player/audio-player.service.ts Mon Dec 12 00:43:55 2016 +0000 @@ -1,10 +1,29 @@ import {Injectable, Inject} from '@angular/core'; +import {Subject} from "rxjs/Subject"; +import {Observable} from "rxjs"; @Injectable() export class AudioPlayerService { + private currentObjectUrl: string; + private playingStateChange: Subject; + playingStateChange$: Observable; + private seeked: Subject; + seeked$: Observable; + constructor(@Inject(HTMLAudioElement) private audioElement: HTMLAudioElement /* TODO probably shouldn't play audio this way */, @Inject('AudioContext') private audioContext: AudioContext) { + this.currentObjectUrl = ''; + this.playingStateChange = new Subject(); + this.playingStateChange$ = this.playingStateChange.asObservable(); + this.seeked = new Subject(); + this.seeked$ = this.seeked.asObservable(); + this.audioElement.addEventListener('ended', () => { + this.playingStateChange.next(this.isPlaying()); + }); + this.audioElement.addEventListener('seeked', () => { + this.seeked.next(this.audioElement.currentTime); + }); } getCurrentTime(): number { @@ -20,12 +39,18 @@ } loadAudioFromUrl(url: string): void { + if (this.currentObjectUrl) + URL.revokeObjectURL(this.currentObjectUrl); + this.currentObjectUrl = url; this.audioElement.pause(); this.audioElement.src = url; } togglePlaying(): void { - this.isPlaying() ? this.audioElement.pause() : this.audioElement.play(); + if (this.audioElement.readyState >= 2) { + this.isPlaying() ? this.audioElement.pause() : this.audioElement.play(); + this.playingStateChange.next(this.isPlaying()); + } } setVolume(value: number): void { diff -r 05e9cec6d20a -r 57980a1a579d src/app/services/feature-extraction/FeatureExtractionWorker.ts --- a/src/app/services/feature-extraction/FeatureExtractionWorker.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/services/feature-extraction/FeatureExtractionWorker.ts Mon Dec 12 00:43:55 2016 +0000 @@ -2,11 +2,10 @@ * Created by lucas on 01/12/2016. */ -import {ListResponse, EmscriptenProxy} from 'piper'; -import {PiperSimpleClient} from 'piper/HigherLevelUtilities'; +import { EmscriptenProxy } from 'piper'; +import { PiperSimpleClient } from 'piper/HigherLevelUtilities'; import { VampExamplePlugins } from 'piper/ext/VampExamplePluginsModule'; - // TODO TypeScript has a .d.ts file for webworkers, but for some reason it clashes with the typings for dom and causes compiler errors interface WorkerGlobalScope { onmessage: (this: this, ev: MessageEvent) => any; @@ -23,16 +22,25 @@ constructor(workerScope: WorkerGlobalScope) { this.workerScope = workerScope; - let counter = 0; - setInterval(() => this.workerScope.postMessage(counter++), 1000); this.piperClient = new PiperSimpleClient(new EmscriptenProxy(VampExamplePlugins())); this.workerScope.onmessage = (ev: MessageEvent) => { + const sendResponse = (result) => { + this.workerScope.postMessage({ + method: ev.data.method, + result: result + }); + }; switch (ev.data.method) { case 'list': - this.piperClient.list({}).then(this.workerScope.postMessage); + this.piperClient.list({}).then(sendResponse); + break; + case 'process': + this.piperClient.process(ev.data.params).then(sendResponse); + break; + case 'collect': + this.piperClient.collect(ev.data.params).then(sendResponse).catch(err => console.error(err)); } }; } - } diff -r 05e9cec6d20a -r 57980a1a579d src/app/services/feature-extraction/feature-extraction.service.ts --- a/src/app/services/feature-extraction/feature-extraction.service.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/services/feature-extraction/feature-extraction.service.ts Mon Dec 12 00:43:55 2016 +0000 @@ -1,35 +1,74 @@ -import { Injectable } from '@angular/core'; -import {ListResponse} from "piper"; +import {Injectable} from '@angular/core'; +import { + ListResponse, ListRequest +} from "piper"; +import { + SimpleRequest, SimpleResponse +} from "piper/HigherLevelUtilities"; +import {Subject} from "rxjs/Subject"; +import {Observable} from "rxjs"; +interface RequestMessage { + method: string; + params: RequestType; +} + +interface ResponseMessage { + method: string; + result: ResponseType; +} @Injectable() export class FeatureExtractionService { private worker: Worker; - + private featuresExtracted: Subject; + featuresExtracted$: Observable; constructor() { this.worker = new Worker('bootstrap-feature-extraction-worker.js'); - } - - testMessageStream() { - this.worker.addEventListener('message', ev => console.log(ev.data)); - this.worker.postMessage('anything'); + this.featuresExtracted = new Subject(); + this.featuresExtracted$ = this.featuresExtracted.asObservable(); } list(): Promise { - return this.request({method: 'list'}, (ev: MessageEvent) => ev.data.available !== undefined); + return this.request( + {method: 'list', params: {}}, + (ev: MessageEvent) => ev.data.result.available !== undefined + ).then(msg => msg.result); } - private request(request: Req, predicate: (ev: MessageEvent) => boolean): Promise { + process(request: SimpleRequest): Promise { + return this.request( + {method: 'process', params: request}, + (ev: MessageEvent) => ev.data.method === 'process' + ).then(msg => { + this.featuresExtracted.next(msg.result); + return msg.result; + }); + } + + collect(request: SimpleRequest): Promise { + return this.request( + {method: 'collect', params: request}, + (ev: MessageEvent) => ev.data.method === 'collect' + ).then(msg => { + this.featuresExtracted.next(msg.result); + return msg.result; + }); + } + + private request(request: RequestMessage, + predicate: (ev: MessageEvent) => boolean) + : Promise> { return new Promise(res => { - const listener = (ev: MessageEvent ) => { + const listener = (ev: MessageEvent) => { this.worker.removeEventListener('message', listener); if (predicate(ev)) res(ev.data); }; this.worker.addEventListener('message', listener); this.worker.postMessage(request); - }); + }).catch(err => console.error(err)); } } diff -r 05e9cec6d20a -r 57980a1a579d src/app/waveform/waveform.component.ts --- a/src/app/waveform/waveform.component.ts Fri Dec 02 17:03:29 2016 +0000 +++ b/src/app/waveform/waveform.component.ts Mon Dec 12 00:43:55 2016 +0000 @@ -1,21 +1,40 @@ import { - Component, OnInit, ViewChild, ElementRef, Input, AfterViewInit, NgZone + Component, OnInit, ViewChild, ElementRef, Input, AfterViewInit, NgZone, + OnDestroy } from '@angular/core'; import {AudioPlayerService} from "../services/audio-player/audio-player.service"; import wavesUI from 'waves-ui'; +import { + FeatureExtractionService +} from "../services/feature-extraction/feature-extraction.service"; +import {Subscription} from "rxjs"; +import { + FeatureCollection, + FixedSpacedFeatures, SimpleResponse +} from "piper/HigherLevelUtilities"; +import {toSeconds} from "piper"; +import {FeatureList} from "piper/Feature"; type Timeline = any; // TODO what type actually is it.. start a .d.ts for waves-ui? +type Layer = any; +type Track = any; +type DisposableIndex = number; +type Colour = string; @Component({ selector: 'app-waveform', templateUrl: './waveform.component.html', styleUrls: ['./waveform.component.css'] }) -export class WaveformComponent implements OnInit, AfterViewInit { +export class WaveformComponent implements OnInit, AfterViewInit, OnDestroy { @ViewChild('track') trackDiv: ElementRef; - private _audioBuffer: AudioBuffer = undefined; + private _audioBuffer: AudioBuffer; + private timeline: Timeline; + private cursorLayer: any; + private disposableLayers: Layer[]; + private colouredLayers: Map; @Input() set audioBuffer(buffer: AudioBuffer) { @@ -28,12 +47,56 @@ return this._audioBuffer; } + private featureExtractionSubscription: Subscription; + private playingStateSubscription: Subscription; + private seekedSubscription: Subscription; + private isPlaying: boolean; + constructor(private audioService: AudioPlayerService, - public ngZone: NgZone) {} - ngOnInit() {} + private piperService: FeatureExtractionService, + public ngZone: NgZone) { + this.colouredLayers = new Map(); + this.disposableLayers = []; + this._audioBuffer = undefined; + this.timeline = undefined; + this.cursorLayer = undefined; + this.isPlaying = false; + const colours = function* () { + const circularColours = [ + 'black', + 'red', + 'green', + 'purple', + 'orange' + ]; + let index = 0; + const nColours = circularColours.length; + while (true) { + yield circularColours[index = ++index % nColours]; + } + }(); + + this.featureExtractionSubscription = piperService.featuresExtracted$.subscribe( + features => { + this.renderFeatures(features, colours.next().value); + }); + this.playingStateSubscription = audioService.playingStateChange$.subscribe( + isPlaying => { + this.isPlaying = isPlaying; + if (this.isPlaying) + this.animate(); + }); + this.seekedSubscription = audioService.seeked$.subscribe(() => { + if (!this.isPlaying) + this.animate(); + }); + } + + ngOnInit() { + } ngAfterViewInit(): void { - this.renderTimeline(); + this.timeline = this.renderTimeline(); } renderTimeline(duration: number = 1.0): Timeline { @@ -45,51 +108,246 @@ const timeline = new wavesUI.core.Timeline(pixelsPerSecond, width); timeline.timeContext.offset = 0.5 * timeline.timeContext.visibleDuration; timeline.createTrack(track, height, 'main'); + return timeline; + } + renderWaveform(buffer: AudioBuffer): void { + const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height; + const mainTrack = this.timeline.getTrackById('main'); + if (this.timeline) { + // resize + const width = this.trackDiv.nativeElement.getBoundingClientRect().width; + + // loop through layers and remove them, waves-ui provides methods for this but it seems to not work properly + const timeContextChildren = this.timeline.timeContext._children; + + for (let i = 0, length = this.disposableLayers.length; i < length; ++i) { + let layer = this.disposableLayers.pop(); + mainTrack.remove(layer); + + const index = timeContextChildren.indexOf(layer.timeContext); + if (index >= 0) + timeContextChildren.splice(index, 1); + layer.destroy(); + } + this.colouredLayers.clear(); + + this.timeline.visibleWidth = width; + this.timeline.pixelsPerSecond = width / buffer.duration; + mainTrack.height = height; + } else { + this.timeline = this.renderTimeline(buffer.duration) + } // time axis const timeAxis = new wavesUI.helpers.TimeAxisLayer({ height: height, color: 'gray' }); + this.addLayer(timeAxis, mainTrack, this.timeline.timeContext, true); - timeline.addLayer(timeAxis, 'main', 'default', true); - return timeline; - } - - renderWaveform(buffer: AudioBuffer): void { - const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height; - const timeline: Timeline = this.renderTimeline(buffer.duration); const waveformLayer = new wavesUI.helpers.WaveformLayer(buffer, { top: 10, height: height * 0.9, color: 'darkblue' }); - (timeline as any).addLayer(waveformLayer, 'main'); + this.addLayer(waveformLayer, mainTrack, this.timeline.timeContext); - const cursorLayer = new wavesUI.helpers.CursorLayer({ + this.cursorLayer = new wavesUI.helpers.CursorLayer({ height: height }); - timeline.addLayer(cursorLayer, 'main'); - timeline.state = new wavesUI.states.CenteredZoomState(timeline); + this.addLayer(this.cursorLayer, mainTrack, this.timeline.timeContext); + this.timeline.state = new wavesUI.states.CenteredZoomState(this.timeline); + mainTrack.render(); + mainTrack.update(); + this.animate(); + } + + // TODO refactor - this doesn't belong here + private renderFeatures(extracted: SimpleResponse, colour: Colour): void { + if (!extracted.hasOwnProperty('features') || !extracted.hasOwnProperty('outputDescriptor')) return; + if (!extracted.features.hasOwnProperty('shape') || !extracted.features.hasOwnProperty('data')) return; + const features: FeatureCollection = (extracted.features as FeatureCollection); + const outputDescriptor = extracted.outputDescriptor; + const height = this.trackDiv.nativeElement.getBoundingClientRect().height; + const mainTrack = this.timeline.getTrackById('main'); + + // TODO refactor all of this + switch (features.shape) { + case 'vector': { + const stepDuration = (features as FixedSpacedFeatures).stepDuration; + const featureData = (features.data as Float32Array); + const normalisationFactor = 1.0 / + featureData.reduce( + (currentMax, feature) => Math.max(currentMax, feature), + -Infinity + ); + const plotData = [...featureData].map((feature, i) => { + return { + cx: i * stepDuration, + cy: feature * normalisationFactor + }; + }); + let breakpointLayer = new wavesUI.helpers.BreakpointLayer(plotData, { + color: colour, + height: height + }); + this.colouredLayers.set(this.addLayer( + breakpointLayer, + mainTrack, + this.timeline.timeContext + ), colour); + break; + } + case 'list': { + const featureData = (features.data as FeatureList); + // TODO look at output descriptor instead of directly inspecting features + const hasDuration = outputDescriptor.configured.hasDuration; + const isMarker = !hasDuration + && outputDescriptor.configured.binCount === 0 + && featureData[0].featureValues == null; + const isRegion = hasDuration + && featureData[0].timestamp != null; + + // TODO refactor, this is incomprehensible + if (isMarker) { + const plotData = featureData.map(feature => { + return {x: toSeconds(feature.timestamp)} + }); + let markerLayer = new wavesUI.helpers.MarkerLayer(plotData, { + height: height, + color: colour, + }); + this.colouredLayers.set(this.addLayer( + markerLayer, + mainTrack, + this.timeline.timeContext + ), colour); + } else if (isRegion) { + const isBarRegion = featureData[0].featureValues.length === 1; + const getSegmentArgs = () => { + if (isBarRegion) { + const min = featureData.reduce((min, feature) => + Math.min(min, feature.featureValues[0]), + Infinity + ); + + const max = featureData.reduce((max, feature) => + Math.max(max, feature.featureValues[0]), + -Infinity + ); + + return [ + featureData.map(feature => { + return { + x: toSeconds(feature.timestamp), + y: feature.featureValues[0], + width: toSeconds(feature.duration), + height: 0.05 * max, + color: colour, + opacity: 0.8 + } + }), + {yDomain: [min, max + 0.05 * max], height: height} as any + ] + } else { + return [featureData.map(feature => { + return { + x: toSeconds(feature.timestamp), + width: toSeconds(feature.duration), + color: colour, + opacity: 0.8 + } + }), {height: height}]; + } + }; + + let segmentLayer = new wavesUI.helpers.SegmentLayer( + ...getSegmentArgs() + ); + this.colouredLayers.set(this.addLayer( + segmentLayer, + mainTrack, + this.timeline.timeContext + ), colour); + } + + break; + } + } + + this.timeline.tracks.update(); + } + + private animate(): void { this.ngZone.runOutsideAngular(() => { // listen for time passing... - // TODO this gets the fans going on large files... worth fixing? or waiting to write a better component? - // or, can this be updated in a more efficient manner? const updateSeekingCursor = () => { - cursorLayer.currentPosition = this.audioService.getCurrentTime(); - cursorLayer.update(); - if (timeline.timeContext.offset + this.audioService.getCurrentTime() >= timeline.timeContext.visibleDuration) { - timeline.timeContext.offset -= timeline.timeContext.visibleDuration; - timeline.tracks.update(); + const currentTime = this.audioService.getCurrentTime(); + this.cursorLayer.currentPosition = currentTime; + this.cursorLayer.update(); + + const currentOffset = this.timeline.timeContext.offset; + const offsetTimestamp = currentOffset + + currentTime; + + const visibleDuration = this.timeline.timeContext.visibleDuration; + // TODO reduce duplication between directions and make more declarative + // this kinda logic should also be tested + const mustPageForward = offsetTimestamp > visibleDuration; + const mustPageBackward = currentTime < -currentOffset; + + if (mustPageForward) { + const hasSkippedMultiplePages = offsetTimestamp - visibleDuration > visibleDuration; + + this.timeline.timeContext.offset = hasSkippedMultiplePages + ? -currentTime + 0.5 * visibleDuration + : currentOffset - visibleDuration; + this.timeline.tracks.update(); } - if (-this.audioService.getCurrentTime() > timeline.timeContext.offset) { - timeline.timeContext.offset += timeline.timeContext.visibleDuration; - timeline.tracks.update(); + + if (mustPageBackward) { + const hasSkippedMultiplePages = currentTime + visibleDuration < -currentOffset; + this.timeline.timeContext.offset = hasSkippedMultiplePages + ? -currentTime + 0.5 * visibleDuration + : currentOffset + visibleDuration; + this.timeline.tracks.update(); } - requestAnimationFrame(updateSeekingCursor); + + if (this.isPlaying) + requestAnimationFrame(updateSeekingCursor); }; updateSeekingCursor(); }); } + private addLayer(layer: Layer, track: Track, timeContext: any, isAxis: boolean = false): DisposableIndex { + timeContext.zoom = 1.0; + if (!layer.timeContext) { + layer.setTimeContext(isAxis ? + timeContext : new wavesUI.core.LayerTimeContext(timeContext)); + } + track.add(layer); + layer.render(); + layer.update(); + return this.disposableLayers.push(layer) - 1; + } + + private static changeColour(layer: Layer, colour: string): void { + const butcherShapes = (shape) => { + shape.install({color: () => colour}); + shape.params.color = colour; + shape.update(layer._renderingContext, layer.data); + }; + + layer._$itemCommonShapeMap.forEach(butcherShapes); + layer._$itemShapeMap.forEach(butcherShapes); + layer.render(); + layer.update(); + } + + ngOnDestroy(): void { + this.featureExtractionSubscription.unsubscribe(); + this.playingStateSubscription.unsubscribe(); + this.seekedSubscription.unsubscribe(); + } }