annotate src/app/waveform/waveform.component.ts @ 196:aa1c92c553cb

A few different @Input flags allowing for using component for just a waveform or features or both, turning off seeking and allowing more than one feature to be extracted to the component. Very messy, desperately needs refactoring.
author Lucas Thompson <dev@lucas.im>
date Fri, 24 Mar 2017 11:00:54 +0000
parents a50feba0d7f0
children bdfd4b4b7130
rev   line source
dev@10 1 import {
dev@51 2 Component, OnInit, ViewChild, ElementRef, Input, AfterViewInit, NgZone,
dev@51 3 OnDestroy
dev@10 4 } from '@angular/core';
dev@196 5 import {
dev@196 6 AudioPlayerService, AudioResource,
dev@196 7 AudioResourceError
dev@196 8 } from "../services/audio-player/audio-player.service";
dev@36 9 import wavesUI from 'waves-ui';
dev@63 10 import {
dev@64 11 FeatureExtractionService
dev@63 12 } from "../services/feature-extraction/feature-extraction.service";
dev@51 13 import {Subscription} from "rxjs";
dev@63 14 import {
dev@63 15 FeatureCollection,
dev@64 16 FixedSpacedFeatures, SimpleResponse
dev@63 17 } from "piper/HigherLevelUtilities";
dev@53 18 import {toSeconds} from "piper";
dev@67 19 import {FeatureList, Feature} from "piper/Feature";
dev@81 20 import * as Hammer from 'hammerjs';
dev@129 21 import {WavesSpectrogramLayer} from "../spectrogram/Spectrogram";
dev@8 22
dev@54 23 type Layer = any;
dev@54 24 type Track = any;
dev@59 25 type Colour = string;
dev@6 26
dev@6 27 @Component({
dev@6 28 selector: 'app-waveform',
dev@6 29 templateUrl: './waveform.component.html',
dev@6 30 styleUrls: ['./waveform.component.css']
dev@6 31 })
dev@51 32 export class WaveformComponent implements OnInit, AfterViewInit, OnDestroy {
dev@20 33
dev@8 34 @ViewChild('track') trackDiv: ElementRef;
dev@6 35
dev@189 36 @Input() timeline: Timeline;
dev@189 37 @Input() trackIdPrefix: string;
dev@196 38 @Input() set isSubscribedToExtractionService(isSubscribed: boolean) {
dev@196 39 if (isSubscribed) {
dev@196 40 if (this.featureExtractionSubscription) {
dev@196 41 return;
dev@196 42 }
dev@16 43
dev@196 44 const colours = function* () {
dev@196 45 const circularColours = [
dev@196 46 'black',
dev@196 47 'red',
dev@196 48 'green',
dev@196 49 'purple',
dev@196 50 'orange'
dev@196 51 ];
dev@196 52 let index = 0;
dev@196 53 const nColours = circularColours.length;
dev@196 54 while (true) {
dev@196 55 yield circularColours[index = ++index % nColours];
dev@196 56 }
dev@196 57 }();
dev@196 58
dev@196 59 this.featureExtractionSubscription =
dev@196 60 this.piperService.featuresExtracted$.subscribe(
dev@196 61 features => {
dev@196 62 this.renderFeatures(features, colours.next().value);
dev@196 63 });
dev@196 64 } else {
dev@196 65 if (this.featureExtractionSubscription) {
dev@196 66 this.featureExtractionSubscription.unsubscribe();
dev@196 67 }
dev@196 68 }
dev@196 69 }
dev@196 70 @Input() set isSubscribedToAudioService(isSubscribed: boolean) {
dev@196 71 this._isSubscribedToAudioService = isSubscribed;
dev@196 72 if (isSubscribed) {
dev@196 73 if (this.onAudioDataSubscription) {
dev@196 74 return;
dev@196 75 }
dev@196 76
dev@196 77 this.onAudioDataSubscription =
dev@196 78 this.audioService.audioLoaded$.subscribe(res => {
dev@196 79 const wasError = (res as AudioResourceError).message != null;
dev@196 80
dev@196 81 if (wasError) {
dev@196 82 console.warn('No audio, display error?');
dev@196 83 } else {
dev@196 84 this.audioBuffer = (res as AudioResource).samples;
dev@196 85 }
dev@196 86 });
dev@196 87 } else {
dev@196 88 if (this.onAudioDataSubscription) {
dev@196 89 this.onAudioDataSubscription.unsubscribe();
dev@196 90 }
dev@196 91 }
dev@196 92 }
dev@196 93
dev@196 94 get isSubscribedToAudioService(): boolean {
dev@196 95 return this._isSubscribedToAudioService;
dev@196 96 }
dev@196 97
dev@196 98 @Input() set isOneShotExtractor(isOneShot: boolean) {
dev@196 99 this._isOneShotExtractor = isOneShot;
dev@196 100 }
dev@196 101
dev@196 102 get isOneShotExtractor(): boolean {
dev@196 103 return this._isOneShotExtractor;
dev@196 104 }
dev@196 105
dev@196 106 @Input() set isSeeking(isSeeking: boolean) {
dev@196 107 this._isSeeking = isSeeking;
dev@196 108 if (isSeeking) {
dev@196 109 if (this.seekedSubscription) {
dev@196 110 return;
dev@196 111 }
dev@196 112 if(this.playingStateSubscription) {
dev@196 113 return;
dev@196 114 }
dev@196 115
dev@196 116 this.seekedSubscription = this.audioService.seeked$.subscribe(() => {
dev@196 117 if (!this.isPlaying)
dev@196 118 this.animate();
dev@196 119 });
dev@196 120 this.playingStateSubscription =
dev@196 121 this.audioService.playingStateChange$.subscribe(
dev@196 122 isPlaying => {
dev@196 123 this.isPlaying = isPlaying;
dev@196 124 if (this.isPlaying)
dev@196 125 this.animate();
dev@196 126 });
dev@196 127 } else {
dev@196 128 if (this.isPlaying) {
dev@196 129 this.isPlaying = false;
dev@196 130 }
dev@196 131 if (this.playingStateSubscription) {
dev@196 132 this.playingStateSubscription.unsubscribe();
dev@196 133 }
dev@196 134 if (this.seekedSubscription) {
dev@196 135 this.seekedSubscription.unsubscribe();
dev@196 136 }
dev@196 137 }
dev@196 138 }
dev@196 139
dev@196 140 get isSeeking(): boolean {
dev@196 141 return this._isSeeking;
dev@196 142 }
dev@196 143
dev@16 144 set audioBuffer(buffer: AudioBuffer) {
dev@16 145 this._audioBuffer = buffer || undefined;
cannam@117 146 if (this.audioBuffer) {
dev@20 147 this.renderWaveform(this.audioBuffer);
dev@180 148 // this.renderSpectrogram(this.audioBuffer);
cannam@117 149 }
dev@16 150 }
dev@16 151
dev@16 152 get audioBuffer(): AudioBuffer {
dev@16 153 return this._audioBuffer;
dev@16 154 }
dev@16 155
dev@196 156 private _audioBuffer: AudioBuffer;
dev@196 157 private _isSubscribedToAudioService: boolean;
dev@196 158 private _isOneShotExtractor: boolean;
dev@196 159 private _isSeeking: boolean;
dev@196 160 private cursorLayer: any;
dev@196 161 private layers: Layer[];
dev@51 162 private featureExtractionSubscription: Subscription;
dev@53 163 private playingStateSubscription: Subscription;
dev@53 164 private seekedSubscription: Subscription;
dev@196 165 private onAudioDataSubscription: Subscription;
dev@53 166 private isPlaying: boolean;
dev@110 167 private offsetAtPanStart: number;
dev@110 168 private initialZoom: number;
dev@110 169 private initialDistance: number;
dev@155 170 private zoomOnMouseDown: number;
dev@157 171 private offsetOnMouseDown: number;
dev@196 172 private hasShot: boolean;
dev@196 173 private isLoading: boolean;
dev@51 174
dev@31 175 constructor(private audioService: AudioPlayerService,
dev@51 176 private piperService: FeatureExtractionService,
dev@51 177 public ngZone: NgZone) {
dev@196 178 this.isSubscribedToAudioService = true;
dev@196 179 this.isSeeking = true;
dev@185 180 this.layers = [];
dev@196 181 this.audioBuffer = undefined;
dev@54 182 this.timeline = undefined;
dev@54 183 this.cursorLayer = undefined;
dev@53 184 this.isPlaying = false;
dev@196 185 this.isLoading = true;
dev@51 186 }
dev@51 187
dev@53 188 ngOnInit() {
dev@53 189 }
dev@10 190
dev@10 191 ngAfterViewInit(): void {
dev@189 192 this.trackIdPrefix = this.trackIdPrefix || "default";
dev@196 193 if (this.timeline) {
dev@196 194 this.renderTimeline(null, true, true);
dev@196 195 } else {
dev@196 196 this.renderTimeline();
dev@196 197 }
dev@20 198 }
dev@20 199
dev@196 200 renderTimeline(duration: number = 1.0,
dev@196 201 useExistingDuration: boolean = false,
dev@196 202 isInitialRender: boolean = false): Timeline {
dev@18 203 const track: HTMLElement = this.trackDiv.nativeElement;
dev@20 204 track.innerHTML = "";
dev@18 205 const height: number = track.getBoundingClientRect().height;
dev@18 206 const width: number = track.getBoundingClientRect().width;
dev@18 207 const pixelsPerSecond = width / duration;
dev@196 208 const hasExistingTimeline = this.timeline instanceof wavesUI.core.Timeline;
dev@196 209
dev@196 210 if (hasExistingTimeline) {
dev@196 211 if (!useExistingDuration) {
dev@196 212 this.timeline.pixelsPerSecond = pixelsPerSecond;
dev@196 213 this.timeline.visibleWidth = width;
dev@196 214 }
dev@180 215 } else {
dev@180 216 this.timeline = new wavesUI.core.Timeline(pixelsPerSecond, width);
dev@180 217 }
dev@196 218 const waveTrack = this.timeline.createTrack(
dev@196 219 track,
dev@196 220 height,
dev@196 221 `wave-${this.trackIdPrefix}`
dev@196 222 );
dev@196 223 if (isInitialRender && hasExistingTimeline) {
dev@196 224 // time axis
dev@196 225 const timeAxis = new wavesUI.helpers.TimeAxisLayer({
dev@196 226 height: height,
dev@196 227 color: '#b0b0b0'
dev@196 228 });
dev@196 229 this.addLayer(timeAxis, waveTrack, this.timeline.timeContext, true);
dev@196 230 this.cursorLayer = new wavesUI.helpers.CursorLayer({
dev@196 231 height: height
dev@196 232 });
dev@196 233 this.addLayer(this.cursorLayer, waveTrack, this.timeline.timeContext);
dev@196 234 }
dev@196 235 if ('ontouchstart' in window) {
dev@196 236 interface Point {
dev@196 237 x: number;
dev@196 238 y: number;
dev@196 239 }
dev@196 240
dev@196 241 let zoomGestureJustEnded: boolean = false;
dev@196 242
dev@196 243 const pixelToExponent: Function = wavesUI.utils.scales.linear()
dev@196 244 .domain([0, 100]) // 100px => factor 2
dev@196 245 .range([0, 1]);
dev@196 246
dev@196 247 const calculateDistance: (p1: Point, p2: Point) => number = (p1, p2) => {
dev@196 248 return Math.pow(
dev@196 249 Math.pow(p2.x - p1.x, 2) +
dev@196 250 Math.pow(p2.y - p1.y, 2), 0.5);
dev@196 251 };
dev@196 252
dev@196 253 const hammertime = new Hammer(this.trackDiv.nativeElement);
dev@196 254 const scroll = (ev) => {
dev@196 255 if (zoomGestureJustEnded) {
dev@196 256 zoomGestureJustEnded = false;
dev@196 257 console.log("Skip this event: likely a single touch dangling from pinch");
dev@196 258 return;
dev@196 259 }
dev@196 260 this.timeline.timeContext.offset = this.offsetAtPanStart +
dev@196 261 this.timeline.timeContext.timeToPixel.invert(ev.deltaX);
dev@196 262 this.timeline.tracks.update();
dev@196 263 };
dev@196 264
dev@196 265 const zoom = (ev) => {
dev@196 266 const minZoom = this.timeline.state.minZoom;
dev@196 267 const maxZoom = this.timeline.state.maxZoom;
dev@196 268 const distance = calculateDistance({
dev@196 269 x: ev.pointers[0].clientX,
dev@196 270 y: ev.pointers[0].clientY
dev@196 271 }, {
dev@196 272 x: ev.pointers[1].clientX,
dev@196 273 y: ev.pointers[1].clientY
dev@196 274 });
dev@196 275
dev@196 276 const lastCenterTime =
dev@196 277 this.timeline.timeContext.timeToPixel.invert(ev.center.x);
dev@196 278
dev@196 279 const exponent = pixelToExponent(distance - this.initialDistance);
dev@196 280 const targetZoom = this.initialZoom * Math.pow(2, exponent);
dev@196 281
dev@196 282 this.timeline.timeContext.zoom =
dev@196 283 Math.min(Math.max(targetZoom, minZoom), maxZoom);
dev@196 284
dev@196 285 const newCenterTime =
dev@196 286 this.timeline.timeContext.timeToPixel.invert(ev.center.x);
dev@196 287
dev@196 288 this.timeline.timeContext.offset += newCenterTime - lastCenterTime;
dev@196 289 this.timeline.tracks.update();
dev@196 290 };
dev@196 291 hammertime.get('pinch').set({ enable: true });
dev@196 292 hammertime.on('panstart', () => {
dev@196 293 this.offsetAtPanStart = this.timeline.timeContext.offset;
dev@196 294 });
dev@196 295 hammertime.on('panleft', scroll);
dev@196 296 hammertime.on('panright', scroll);
dev@196 297 hammertime.on('pinchstart', (e) => {
dev@196 298 this.initialZoom = this.timeline.timeContext.zoom;
dev@196 299
dev@196 300 this.initialDistance = calculateDistance({
dev@196 301 x: e.pointers[0].clientX,
dev@196 302 y: e.pointers[0].clientY
dev@196 303 }, {
dev@196 304 x: e.pointers[1].clientX,
dev@196 305 y: e.pointers[1].clientY
dev@196 306 });
dev@196 307 });
dev@196 308 hammertime.on('pinch', zoom);
dev@196 309 hammertime.on('pinchend', () => {
dev@196 310 zoomGestureJustEnded = true;
dev@196 311 });
dev@196 312 }
dev@189 313 // this.timeline.createTrack(track, height/2, `wave-${this.trackIdPrefix}`);
dev@189 314 // this.timeline.createTrack(track, height/2, `grid-${this.trackIdPrefix}`);
dev@54 315 }
dev@18 316
cannam@108 317 estimatePercentile(matrix, percentile) {
cannam@108 318 // our sample is not evenly distributed across the whole data set:
cannam@108 319 // it is guaranteed to include at least one sample from every
cannam@108 320 // column, and could sample some values more than once. But it
cannam@108 321 // should be good enough in most cases (todo: show this)
cannam@109 322 if (matrix.length === 0) {
cannam@109 323 return 0.0;
cannam@109 324 }
cannam@108 325 const w = matrix.length;
cannam@108 326 const h = matrix[0].length;
cannam@108 327 const n = w * h;
cannam@109 328 const m = (n > 50000 ? 50000 : n); // should base that on the %ile
cannam@108 329 let m_per = Math.floor(m / w);
cannam@108 330 if (m_per < 1) m_per = 1;
cannam@108 331 let sample = [];
cannam@108 332 for (let x = 0; x < w; ++x) {
cannam@108 333 for (let i = 0; i < m_per; ++i) {
cannam@108 334 const y = Math.floor(Math.random() * h);
cannam@109 335 const value = matrix[x][y];
cannam@109 336 if (!isNaN(value) && value !== Infinity) {
cannam@109 337 sample.push(value);
cannam@109 338 }
cannam@108 339 }
cannam@108 340 }
cannam@109 341 if (sample.length === 0) {
cannam@109 342 console.log("WARNING: No samples gathered, even though we hoped for " +
cannam@109 343 (m_per * w) + " of them");
cannam@109 344 return 0.0;
cannam@109 345 }
cannam@108 346 sample.sort((a,b) => { return a - b; });
cannam@108 347 const ix = Math.floor((sample.length * percentile) / 100);
cannam@108 348 console.log("Estimating " + percentile + "-%ile of " +
cannam@108 349 n + "-sample dataset (" + w + " x " + h + ") as value " + ix +
cannam@108 350 " of sorted " + sample.length + "-sample subset");
cannam@108 351 const estimate = sample[ix];
cannam@108 352 console.log("Estimate is: " + estimate + " (where min sampled value = " +
cannam@108 353 sample[0] + " and max = " + sample[sample.length-1] + ")");
cannam@108 354 return estimate;
cannam@108 355 }
cannam@108 356
cannam@108 357 interpolatingMapper(hexColours) {
cannam@108 358 const colours = hexColours.map(n => {
cannam@108 359 const i = parseInt(n, 16);
cannam@118 360 return [ ((i >> 16) & 255) / 255.0,
cannam@118 361 ((i >> 8) & 255) / 255.0,
cannam@118 362 ((i) & 255) / 255.0 ];
cannam@108 363 });
cannam@108 364 const last = colours.length - 1;
cannam@108 365 return (value => {
cannam@108 366 const m = value * last;
cannam@108 367 if (m >= last) {
cannam@108 368 return colours[last];
cannam@108 369 }
cannam@108 370 if (m <= 0) {
cannam@108 371 return colours[0];
cannam@108 372 }
cannam@108 373 const base = Math.floor(m);
cannam@108 374 const prop0 = base + 1.0 - m;
cannam@108 375 const prop1 = m - base;
cannam@108 376 const c0 = colours[base];
cannam@108 377 const c1 = colours[base+1];
cannam@118 378 return [ c0[0] * prop0 + c1[0] * prop1,
cannam@118 379 c0[1] * prop0 + c1[1] * prop1,
cannam@118 380 c0[2] * prop0 + c1[2] * prop1 ];
cannam@108 381 });
cannam@108 382 }
dev@110 383
cannam@108 384 iceMapper() {
dev@110 385 let hexColours = [
cannam@108 386 // Based on ColorBrewer ylGnBu
cannam@108 387 "ffffff", "ffff00", "f7fcf0", "e0f3db", "ccebc5", "a8ddb5",
cannam@108 388 "7bccc4", "4eb3d3", "2b8cbe", "0868ac", "084081", "042040"
cannam@108 389 ];
cannam@108 390 hexColours.reverse();
cannam@108 391 return this.interpolatingMapper(hexColours);
cannam@108 392 }
dev@110 393
cannam@118 394 hsv2rgb(h, s, v) { // all values in range [0, 1]
cannam@118 395 const i = Math.floor(h * 6);
cannam@118 396 const f = h * 6 - i;
cannam@118 397 const p = v * (1 - s);
cannam@118 398 const q = v * (1 - f * s);
cannam@118 399 const t = v * (1 - (1 - f) * s);
cannam@118 400 let r = 0, g = 0, b = 0;
cannam@118 401 switch (i % 6) {
cannam@118 402 case 0: r = v, g = t, b = p; break;
cannam@118 403 case 1: r = q, g = v, b = p; break;
cannam@118 404 case 2: r = p, g = v, b = t; break;
cannam@118 405 case 3: r = p, g = q, b = v; break;
cannam@118 406 case 4: r = t, g = p, b = v; break;
cannam@118 407 case 5: r = v, g = p, b = q; break;
cannam@118 408 }
cannam@118 409 return [ r, g, b ];
cannam@118 410 }
dev@122 411
cannam@118 412 greenMapper() {
cannam@118 413 const blue = 0.6666;
cannam@118 414 const pieslice = 0.3333;
cannam@118 415 return (value => {
cannam@118 416 const h = blue - value * 2.0 * pieslice;
cannam@118 417 const s = 0.5 + value / 2.0;
cannam@118 418 const v = value;
cannam@118 419 return this.hsv2rgb(h, s, v);
cannam@118 420 });
cannam@118 421 }
cannam@118 422
cannam@118 423 sunsetMapper() {
cannam@118 424 return (value => {
cannam@118 425 let r = (value - 0.24) * 2.38;
cannam@118 426 let g = (value - 0.64) * 2.777;
cannam@118 427 let b = (3.6 * value);
cannam@118 428 if (value > 0.277) b = 2.0 - b;
cannam@118 429 return [ r, g, b ];
cannam@118 430 });
cannam@118 431 }
cannam@118 432
dev@122 433 clearTimeline(): void {
dev@122 434 // loop through layers and remove them, waves-ui provides methods for this but it seems to not work properly
dev@122 435 const timeContextChildren = this.timeline.timeContext._children;
dev@122 436 for (let track of this.timeline.tracks) {
dev@122 437 if (track.layers.length === 0) { continue; }
dev@122 438 const trackLayers = Array.from(track.layers);
dev@122 439 while (trackLayers.length) {
dev@122 440 let layer: Layer = trackLayers.pop();
dev@185 441 if (this.layers.includes(layer)) {
dev@185 442 track.remove(layer);
dev@185 443 this.layers.splice(this.layers.indexOf(layer), 1);
dev@185 444 const index = timeContextChildren.indexOf(layer.timeContext);
dev@185 445 if (index >= 0) {
dev@185 446 timeContextChildren.splice(index, 1);
dev@185 447 }
dev@185 448 layer.destroy();
dev@122 449 }
dev@122 450 }
dev@122 451 }
dev@122 452 }
dev@122 453
dev@54 454 renderWaveform(buffer: AudioBuffer): void {
dev@180 455 // const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height / 2;
dev@180 456 const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height;
dev@189 457 const waveTrack = this.timeline.getTrackById(`wave-${this.trackIdPrefix}`);
dev@54 458 if (this.timeline) {
dev@54 459 // resize
dev@54 460 const width = this.trackDiv.nativeElement.getBoundingClientRect().width;
dev@55 461
dev@122 462 this.clearTimeline();
dev@59 463
dev@54 464 this.timeline.visibleWidth = width;
dev@54 465 this.timeline.pixelsPerSecond = width / buffer.duration;
cannam@117 466 waveTrack.height = height;
dev@54 467 } else {
dev@180 468 this.renderTimeline(buffer.duration)
dev@54 469 }
dev@83 470 this.timeline.timeContext.offset = 0.5 * this.timeline.timeContext.visibleDuration;
cannam@106 471
dev@18 472 // time axis
dev@18 473 const timeAxis = new wavesUI.helpers.TimeAxisLayer({
dev@18 474 height: height,
cannam@106 475 color: '#b0b0b0'
dev@18 476 });
cannam@117 477 this.addLayer(timeAxis, waveTrack, this.timeline.timeContext, true);
dev@18 478
cannam@161 479 const nchannels = buffer.numberOfChannels;
cannam@161 480 const totalWaveHeight = height * 0.9;
cannam@161 481 const waveHeight = totalWaveHeight / nchannels;
dev@189 482
cannam@161 483 for (let ch = 0; ch < nchannels; ++ch) {
cannam@161 484 console.log("about to construct a waveform layer for channel " + ch);
cannam@161 485 const waveformLayer = new wavesUI.helpers.WaveformLayer(buffer, {
cannam@161 486 top: (height - totalWaveHeight)/2 + waveHeight * ch,
cannam@161 487 height: waveHeight,
cannam@161 488 color: 'darkblue',
cannam@161 489 channel: ch
cannam@161 490 });
cannam@161 491 this.addLayer(waveformLayer, waveTrack, this.timeline.timeContext);
cannam@161 492 }
cannam@117 493
dev@53 494 this.cursorLayer = new wavesUI.helpers.CursorLayer({
dev@31 495 height: height
dev@31 496 });
cannam@117 497 this.addLayer(this.cursorLayer, waveTrack, this.timeline.timeContext);
dev@51 498 this.timeline.state = new wavesUI.states.CenteredZoomState(this.timeline);
cannam@117 499 waveTrack.render();
cannam@117 500 waveTrack.update();
dev@81 501
dev@196 502 this.isLoading = false;
dev@53 503 this.animate();
dev@53 504 }
dev@53 505
cannam@117 506 renderSpectrogram(buffer: AudioBuffer): void {
cannam@117 507 const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height / 2;
dev@189 508 const gridTrack = this.timeline.getTrackById(`grid-${this.trackIdPrefix}`);
cannam@117 509
dev@129 510 const spectrogramLayer = new WavesSpectrogramLayer(buffer, {
cannam@118 511 top: height * 0.05,
cannam@117 512 height: height * 0.9,
cannam@117 513 stepSize: 512,
dev@129 514 blockSize: 1024,
cannam@118 515 normalise: 'none',
cannam@118 516 mapper: this.sunsetMapper()
cannam@117 517 });
cannam@117 518 this.addLayer(spectrogramLayer, gridTrack, this.timeline.timeContext);
cannam@117 519
cannam@117 520 this.timeline.tracks.update();
cannam@117 521 }
cannam@117 522
dev@53 523 // TODO refactor - this doesn't belong here
dev@64 524 private renderFeatures(extracted: SimpleResponse, colour: Colour): void {
dev@196 525 if (this.isOneShotExtractor && !this.hasShot) {
dev@196 526 this.featureExtractionSubscription.unsubscribe();
dev@196 527 this.hasShot = true;
dev@196 528 }
dev@196 529
dev@64 530 if (!extracted.hasOwnProperty('features') || !extracted.hasOwnProperty('outputDescriptor')) return;
dev@64 531 if (!extracted.features.hasOwnProperty('shape') || !extracted.features.hasOwnProperty('data')) return;
dev@64 532 const features: FeatureCollection = (extracted.features as FeatureCollection);
dev@64 533 const outputDescriptor = extracted.outputDescriptor;
dev@196 534 // const height = this.trackDiv.nativeElement.getBoundingClientRect().height / 2;
dev@196 535 const height = this.trackDiv.nativeElement.getBoundingClientRect().height;
dev@189 536 const waveTrack = this.timeline.getTrackById(`wave-${this.trackIdPrefix}`);
dev@64 537
dev@64 538 // TODO refactor all of this
dev@63 539 switch (features.shape) {
dev@64 540 case 'vector': {
dev@63 541 const stepDuration = (features as FixedSpacedFeatures).stepDuration;
dev@63 542 const featureData = (features.data as Float32Array);
dev@68 543 if (featureData.length === 0) return;
dev@63 544 const normalisationFactor = 1.0 /
dev@63 545 featureData.reduce(
dev@63 546 (currentMax, feature) => Math.max(currentMax, feature),
dev@63 547 -Infinity
dev@63 548 );
dev@67 549
dev@63 550 const plotData = [...featureData].map((feature, i) => {
dev@63 551 return {
dev@63 552 cx: i * stepDuration,
dev@63 553 cy: feature * normalisationFactor
dev@63 554 };
dev@63 555 });
dev@67 556
dev@105 557 let lineLayer = new wavesUI.helpers.LineLayer(plotData, {
dev@63 558 color: colour,
dev@64 559 height: height
dev@63 560 });
dev@122 561 this.addLayer(
dev@105 562 lineLayer,
cannam@117 563 waveTrack,
dev@63 564 this.timeline.timeContext
dev@122 565 );
dev@63 566 break;
dev@64 567 }
dev@64 568 case 'list': {
dev@64 569 const featureData = (features.data as FeatureList);
dev@68 570 if (featureData.length === 0) return;
dev@64 571 // TODO look at output descriptor instead of directly inspecting features
dev@64 572 const hasDuration = outputDescriptor.configured.hasDuration;
dev@64 573 const isMarker = !hasDuration
dev@64 574 && outputDescriptor.configured.binCount === 0
dev@64 575 && featureData[0].featureValues == null;
dev@64 576 const isRegion = hasDuration
dev@64 577 && featureData[0].timestamp != null;
cannam@149 578 console.log("Have list features: length " + featureData.length +
cannam@149 579 ", isMarker " + isMarker + ", isRegion " + isRegion +
cannam@149 580 ", hasDuration " + hasDuration);
dev@64 581 // TODO refactor, this is incomprehensible
dev@64 582 if (isMarker) {
dev@64 583 const plotData = featureData.map(feature => {
cannam@152 584 return {
cannam@152 585 time: toSeconds(feature.timestamp),
cannam@152 586 label: feature.label
cannam@152 587 }
dev@64 588 });
cannam@149 589 let featureLayer = new wavesUI.helpers.TickLayer(plotData, {
dev@64 590 height: height,
dev@64 591 color: colour,
cannam@152 592 labelPosition: 'bottom',
cannam@152 593 shadeSegments: true
dev@64 594 });
dev@122 595 this.addLayer(
cannam@149 596 featureLayer,
cannam@117 597 waveTrack,
dev@64 598 this.timeline.timeContext
dev@122 599 );
dev@64 600 } else if (isRegion) {
cannam@149 601 console.log("Output is of region type");
dev@67 602 const binCount = outputDescriptor.configured.binCount || 0;
dev@67 603 const isBarRegion = featureData[0].featureValues.length >= 1 || binCount >= 1 ;
dev@64 604 const getSegmentArgs = () => {
dev@64 605 if (isBarRegion) {
dev@64 606
dev@67 607 // TODO refactor - this is messy
dev@67 608 interface FoldsToNumber<T> {
dev@67 609 reduce(fn: (previousValue: number,
dev@67 610 currentValue: T,
dev@67 611 currentIndex: number,
dev@67 612 array: ArrayLike<T>) => number,
dev@67 613 initialValue?: number): number;
dev@67 614 }
dev@64 615
dev@67 616 // TODO potentially change impl., i.e avoid reduce
dev@67 617 const findMin = <T>(arr: FoldsToNumber<T>, getElement: (x: T) => number): number => {
dev@67 618 return arr.reduce((min, val) => Math.min(min, getElement(val)), Infinity);
dev@67 619 };
dev@67 620
dev@67 621 const findMax = <T>(arr: FoldsToNumber<T>, getElement: (x: T) => number): number => {
dev@67 622 return arr.reduce((min, val) => Math.max(min, getElement(val)), -Infinity);
dev@67 623 };
dev@67 624
dev@67 625 const min = findMin<Feature>(featureData, (x: Feature) => {
dev@67 626 return findMin<number>(x.featureValues, y => y);
dev@67 627 });
dev@67 628
dev@67 629 const max = findMax<Feature>(featureData, (x: Feature) => {
dev@67 630 return findMax<number>(x.featureValues, y => y);
dev@67 631 });
dev@67 632
dev@67 633 const barHeight = 1.0 / height;
dev@64 634 return [
dev@67 635 featureData.reduce((bars, feature) => {
dev@67 636 const staticProperties = {
dev@64 637 x: toSeconds(feature.timestamp),
dev@64 638 width: toSeconds(feature.duration),
dev@67 639 height: min + barHeight,
dev@64 640 color: colour,
dev@64 641 opacity: 0.8
dev@67 642 };
dev@67 643 // TODO avoid copying Float32Array to an array - map is problematic here
dev@67 644 return bars.concat([...feature.featureValues]
dev@67 645 .map(val => Object.assign({}, staticProperties, {y: val})))
dev@67 646 }, []),
dev@67 647 {yDomain: [min, max + barHeight], height: height} as any
dev@67 648 ];
dev@64 649 } else {
dev@64 650 return [featureData.map(feature => {
dev@64 651 return {
dev@64 652 x: toSeconds(feature.timestamp),
dev@64 653 width: toSeconds(feature.duration),
dev@64 654 color: colour,
dev@64 655 opacity: 0.8
dev@64 656 }
dev@64 657 }), {height: height}];
dev@64 658 }
dev@64 659 };
dev@64 660
dev@64 661 let segmentLayer = new wavesUI.helpers.SegmentLayer(
dev@64 662 ...getSegmentArgs()
dev@64 663 );
dev@122 664 this.addLayer(
dev@64 665 segmentLayer,
cannam@117 666 waveTrack,
dev@64 667 this.timeline.timeContext
dev@122 668 );
dev@64 669 }
dev@64 670 break;
dev@64 671 }
cannam@106 672 case 'matrix': {
cannam@108 673 const stepDuration = (features as FixedSpacedFeatures).stepDuration;
cannam@120 674 //!!! + start time
cannam@108 675 const matrixData = (features.data as Float32Array[]);
cannam@108 676 if (matrixData.length === 0) return;
cannam@109 677 console.log("matrix data length = " + matrixData.length);
cannam@109 678 console.log("height of first column = " + matrixData[0].length);
cannam@109 679 const targetValue = this.estimatePercentile(matrixData, 95);
cannam@108 680 const gain = (targetValue > 0.0 ? (1.0 / targetValue) : 1.0);
cannam@108 681 console.log("setting gain to " + gain);
cannam@120 682 const matrixEntity =
cannam@120 683 new wavesUI.utils.PrefilledMatrixEntity(matrixData,
cannam@120 684 0, // startTime
cannam@120 685 stepDuration);
cannam@108 686 let matrixLayer = new wavesUI.helpers.MatrixLayer(matrixEntity, {
cannam@108 687 gain,
cannam@118 688 height: height * 0.9,
cannam@118 689 top: height * 0.05,
cannam@109 690 normalise: 'none',
cannam@108 691 mapper: this.iceMapper()
cannam@108 692 });
dev@122 693 this.addLayer(
cannam@108 694 matrixLayer,
cannam@117 695 waveTrack,
cannam@108 696 this.timeline.timeContext
dev@122 697 );
cannam@108 698 break;
cannam@106 699 }
dev@67 700 default:
cannam@106 701 console.log("Cannot render an appropriate layer for feature shape '" +
cannam@106 702 features.shape + "'");
dev@63 703 }
dev@59 704
dev@196 705 this.isLoading = false;
dev@56 706 this.timeline.tracks.update();
dev@53 707 }
dev@53 708
dev@53 709 private animate(): void {
dev@196 710 if (!this.isSeeking) return;
dev@196 711
dev@31 712 this.ngZone.runOutsideAngular(() => {
dev@31 713 // listen for time passing...
dev@31 714 const updateSeekingCursor = () => {
dev@53 715 const currentTime = this.audioService.getCurrentTime();
dev@53 716 this.cursorLayer.currentPosition = currentTime;
dev@53 717 this.cursorLayer.update();
dev@53 718
dev@53 719 const currentOffset = this.timeline.timeContext.offset;
dev@53 720 const offsetTimestamp = currentOffset
dev@53 721 + currentTime;
dev@53 722
dev@53 723 const visibleDuration = this.timeline.timeContext.visibleDuration;
dev@53 724 // TODO reduce duplication between directions and make more declarative
dev@53 725 // this kinda logic should also be tested
dev@53 726 const mustPageForward = offsetTimestamp > visibleDuration;
dev@53 727 const mustPageBackward = currentTime < -currentOffset;
dev@53 728
dev@53 729 if (mustPageForward) {
dev@53 730 const hasSkippedMultiplePages = offsetTimestamp - visibleDuration > visibleDuration;
dev@53 731
cannam@106 732 this.timeline.timeContext.offset = hasSkippedMultiplePages ?
cannam@106 733 -currentTime + 0.5 * visibleDuration :
cannam@106 734 currentOffset - visibleDuration;
dev@51 735 this.timeline.tracks.update();
dev@34 736 }
dev@53 737
dev@53 738 if (mustPageBackward) {
dev@53 739 const hasSkippedMultiplePages = currentTime + visibleDuration < -currentOffset;
cannam@106 740 this.timeline.timeContext.offset = hasSkippedMultiplePages ?
cannam@106 741 -currentTime + 0.5 * visibleDuration :
cannam@106 742 currentOffset + visibleDuration;
dev@51 743 this.timeline.tracks.update();
dev@34 744 }
dev@53 745
dev@53 746 if (this.isPlaying)
dev@53 747 requestAnimationFrame(updateSeekingCursor);
dev@31 748 };
dev@31 749 updateSeekingCursor();
dev@31 750 });
dev@6 751 }
dev@16 752
dev@122 753 private addLayer(layer: Layer, track: Track, timeContext: any, isAxis: boolean = false): void {
dev@54 754 timeContext.zoom = 1.0;
dev@54 755 if (!layer.timeContext) {
dev@54 756 layer.setTimeContext(isAxis ?
dev@54 757 timeContext : new wavesUI.core.LayerTimeContext(timeContext));
dev@54 758 }
dev@54 759 track.add(layer);
dev@185 760 this.layers.push(layer);
dev@54 761 layer.render();
dev@54 762 layer.update();
dev@122 763 if (this.cursorLayer && track.$layout.contains(this.cursorLayer.$el)) {
dev@112 764 track.$layout.appendChild(this.cursorLayer.$el);
dev@112 765 }
dev@59 766 }
dev@59 767
dev@59 768 private static changeColour(layer: Layer, colour: string): void {
dev@59 769 const butcherShapes = (shape) => {
dev@59 770 shape.install({color: () => colour});
dev@59 771 shape.params.color = colour;
dev@59 772 shape.update(layer._renderingContext, layer.data);
dev@59 773 };
dev@59 774
dev@59 775 layer._$itemCommonShapeMap.forEach(butcherShapes);
dev@59 776 layer._$itemShapeMap.forEach(butcherShapes);
dev@59 777 layer.render();
dev@59 778 layer.update();
dev@54 779 }
dev@54 780
dev@51 781 ngOnDestroy(): void {
dev@196 782 if (this.featureExtractionSubscription)
dev@196 783 this.featureExtractionSubscription.unsubscribe();
dev@196 784 if (this.playingStateSubscription)
dev@196 785 this.playingStateSubscription.unsubscribe();
dev@196 786 if (this.seekedSubscription)
dev@196 787 this.seekedSubscription.unsubscribe();
dev@196 788 if (this.onAudioDataSubscription)
dev@196 789 this.onAudioDataSubscription.unsubscribe();
dev@51 790 }
dev@154 791
dev@155 792 seekStart(): void {
dev@155 793 this.zoomOnMouseDown = this.timeline.timeContext.zoom;
dev@157 794 this.offsetOnMouseDown = this.timeline.timeContext.offset;
dev@155 795 }
dev@155 796
dev@155 797 seekEnd(x: number): void {
dev@157 798 const hasSameZoom: boolean = this.zoomOnMouseDown ===
dev@157 799 this.timeline.timeContext.zoom;
dev@157 800 const hasSameOffset: boolean = this.offsetOnMouseDown ===
dev@157 801 this.timeline.timeContext.offset;
dev@157 802 if (hasSameZoom && hasSameOffset) {
dev@155 803 this.seek(x);
dev@155 804 }
dev@155 805 }
dev@155 806
dev@154 807 seek(x: number): void {
dev@154 808 if (this.timeline) {
dev@154 809 const timeContext: any = this.timeline.timeContext;
dev@196 810 if (this.isSeeking) {
dev@196 811 this.audioService.seekTo(
dev@196 812 timeContext.timeToPixel.invert(x)- timeContext.offset
dev@196 813 );
dev@196 814 }
dev@154 815 }
dev@154 816 }
dev@6 817 }