annotate src/app/waveform/waveform.component.ts @ 221:f3ff5ee994a1

With card layout, these can be full-height again
author Chris Cannam <cannam@all-day-breakfast.com>
date Thu, 30 Mar 2017 16:46:48 +0100
parents e72e74453581
children 990b0fc68653 0833ddde6a83
rev   line source
dev@10 1 import {
dev@51 2 Component, OnInit, ViewChild, ElementRef, Input, AfterViewInit, NgZone,
dev@51 3 OnDestroy
dev@10 4 } from '@angular/core';
dev@196 5 import {
dev@196 6 AudioPlayerService, AudioResource,
dev@196 7 AudioResourceError
dev@196 8 } from "../services/audio-player/audio-player.service";
dev@36 9 import wavesUI from 'waves-ui';
dev@63 10 import {
dev@64 11 FeatureExtractionService
dev@63 12 } from "../services/feature-extraction/feature-extraction.service";
dev@51 13 import {Subscription} from "rxjs";
dev@63 14 import {
dev@63 15 FeatureCollection,
dev@64 16 FixedSpacedFeatures, SimpleResponse
dev@63 17 } from "piper/HigherLevelUtilities";
dev@53 18 import {toSeconds} from "piper";
dev@67 19 import {FeatureList, Feature} from "piper/Feature";
dev@81 20 import * as Hammer from 'hammerjs';
dev@129 21 import {WavesSpectrogramLayer} from "../spectrogram/Spectrogram";
dev@8 22
dev@54 23 type Layer = any;
dev@54 24 type Track = any;
dev@59 25 type Colour = string;
dev@6 26
dev@6 27 @Component({
dev@6 28 selector: 'app-waveform',
dev@6 29 templateUrl: './waveform.component.html',
dev@6 30 styleUrls: ['./waveform.component.css']
dev@6 31 })
dev@51 32 export class WaveformComponent implements OnInit, AfterViewInit, OnDestroy {
dev@20 33
dev@8 34 @ViewChild('track') trackDiv: ElementRef;
dev@6 35
dev@189 36 @Input() timeline: Timeline;
dev@189 37 @Input() trackIdPrefix: string;
dev@196 38 @Input() set isSubscribedToExtractionService(isSubscribed: boolean) {
dev@196 39 if (isSubscribed) {
dev@196 40 if (this.featureExtractionSubscription) {
dev@196 41 return;
dev@196 42 }
dev@16 43
dev@196 44 const colours = function* () {
dev@196 45 const circularColours = [
dev@196 46 'black',
dev@196 47 'red',
dev@196 48 'green',
dev@196 49 'purple',
dev@196 50 'orange'
dev@196 51 ];
dev@196 52 let index = 0;
dev@196 53 const nColours = circularColours.length;
dev@196 54 while (true) {
dev@196 55 yield circularColours[index = ++index % nColours];
dev@196 56 }
dev@196 57 }();
dev@196 58
dev@196 59 this.featureExtractionSubscription =
dev@196 60 this.piperService.featuresExtracted$.subscribe(
dev@196 61 features => {
dev@196 62 this.renderFeatures(features, colours.next().value);
dev@196 63 });
dev@196 64 } else {
dev@196 65 if (this.featureExtractionSubscription) {
dev@196 66 this.featureExtractionSubscription.unsubscribe();
dev@196 67 }
dev@196 68 }
dev@196 69 }
dev@196 70 @Input() set isSubscribedToAudioService(isSubscribed: boolean) {
dev@196 71 this._isSubscribedToAudioService = isSubscribed;
dev@196 72 if (isSubscribed) {
dev@196 73 if (this.onAudioDataSubscription) {
dev@196 74 return;
dev@196 75 }
dev@196 76
dev@196 77 this.onAudioDataSubscription =
dev@196 78 this.audioService.audioLoaded$.subscribe(res => {
dev@196 79 const wasError = (res as AudioResourceError).message != null;
dev@196 80
dev@196 81 if (wasError) {
dev@196 82 console.warn('No audio, display error?');
dev@196 83 } else {
dev@196 84 this.audioBuffer = (res as AudioResource).samples;
dev@196 85 }
dev@196 86 });
dev@196 87 } else {
dev@196 88 if (this.onAudioDataSubscription) {
dev@196 89 this.onAudioDataSubscription.unsubscribe();
dev@196 90 }
dev@196 91 }
dev@196 92 }
dev@196 93
dev@196 94 get isSubscribedToAudioService(): boolean {
dev@196 95 return this._isSubscribedToAudioService;
dev@196 96 }
dev@196 97
dev@196 98 @Input() set isOneShotExtractor(isOneShot: boolean) {
dev@196 99 this._isOneShotExtractor = isOneShot;
dev@196 100 }
dev@196 101
dev@196 102 get isOneShotExtractor(): boolean {
dev@196 103 return this._isOneShotExtractor;
dev@196 104 }
dev@196 105
dev@196 106 @Input() set isSeeking(isSeeking: boolean) {
dev@196 107 this._isSeeking = isSeeking;
dev@196 108 if (isSeeking) {
dev@196 109 if (this.seekedSubscription) {
dev@196 110 return;
dev@196 111 }
dev@196 112 if(this.playingStateSubscription) {
dev@196 113 return;
dev@196 114 }
dev@196 115
dev@196 116 this.seekedSubscription = this.audioService.seeked$.subscribe(() => {
dev@196 117 if (!this.isPlaying)
dev@196 118 this.animate();
dev@196 119 });
dev@196 120 this.playingStateSubscription =
dev@196 121 this.audioService.playingStateChange$.subscribe(
dev@196 122 isPlaying => {
dev@196 123 this.isPlaying = isPlaying;
dev@196 124 if (this.isPlaying)
dev@196 125 this.animate();
dev@196 126 });
dev@196 127 } else {
dev@196 128 if (this.isPlaying) {
dev@196 129 this.isPlaying = false;
dev@196 130 }
dev@196 131 if (this.playingStateSubscription) {
dev@196 132 this.playingStateSubscription.unsubscribe();
dev@196 133 }
dev@196 134 if (this.seekedSubscription) {
dev@196 135 this.seekedSubscription.unsubscribe();
dev@196 136 }
dev@196 137 }
dev@196 138 }
dev@196 139
dev@196 140 get isSeeking(): boolean {
dev@196 141 return this._isSeeking;
dev@196 142 }
dev@196 143
dev@16 144 set audioBuffer(buffer: AudioBuffer) {
dev@16 145 this._audioBuffer = buffer || undefined;
cannam@117 146 if (this.audioBuffer) {
dev@20 147 this.renderWaveform(this.audioBuffer);
dev@180 148 // this.renderSpectrogram(this.audioBuffer);
cannam@117 149 }
dev@16 150 }
dev@16 151
dev@16 152 get audioBuffer(): AudioBuffer {
dev@16 153 return this._audioBuffer;
dev@16 154 }
dev@16 155
dev@196 156 private _audioBuffer: AudioBuffer;
dev@196 157 private _isSubscribedToAudioService: boolean;
dev@196 158 private _isOneShotExtractor: boolean;
dev@196 159 private _isSeeking: boolean;
dev@196 160 private cursorLayer: any;
dev@196 161 private layers: Layer[];
dev@51 162 private featureExtractionSubscription: Subscription;
dev@53 163 private playingStateSubscription: Subscription;
dev@53 164 private seekedSubscription: Subscription;
dev@196 165 private onAudioDataSubscription: Subscription;
dev@53 166 private isPlaying: boolean;
dev@155 167 private zoomOnMouseDown: number;
dev@157 168 private offsetOnMouseDown: number;
dev@196 169 private hasShot: boolean;
dev@196 170 private isLoading: boolean;
dev@51 171
dev@31 172 constructor(private audioService: AudioPlayerService,
dev@51 173 private piperService: FeatureExtractionService,
dev@51 174 public ngZone: NgZone) {
dev@196 175 this.isSubscribedToAudioService = true;
dev@196 176 this.isSeeking = true;
dev@185 177 this.layers = [];
dev@196 178 this.audioBuffer = undefined;
dev@54 179 this.timeline = undefined;
dev@54 180 this.cursorLayer = undefined;
dev@53 181 this.isPlaying = false;
dev@196 182 this.isLoading = true;
dev@51 183 }
dev@51 184
dev@53 185 ngOnInit() {
dev@53 186 }
dev@10 187
dev@10 188 ngAfterViewInit(): void {
dev@189 189 this.trackIdPrefix = this.trackIdPrefix || "default";
dev@196 190 if (this.timeline) {
dev@196 191 this.renderTimeline(null, true, true);
dev@196 192 } else {
dev@196 193 this.renderTimeline();
dev@196 194 }
dev@20 195 }
dev@20 196
dev@196 197 renderTimeline(duration: number = 1.0,
dev@196 198 useExistingDuration: boolean = false,
dev@196 199 isInitialRender: boolean = false): Timeline {
dev@18 200 const track: HTMLElement = this.trackDiv.nativeElement;
dev@20 201 track.innerHTML = "";
dev@18 202 const height: number = track.getBoundingClientRect().height;
dev@18 203 const width: number = track.getBoundingClientRect().width;
dev@18 204 const pixelsPerSecond = width / duration;
dev@196 205 const hasExistingTimeline = this.timeline instanceof wavesUI.core.Timeline;
dev@196 206
dev@196 207 if (hasExistingTimeline) {
dev@196 208 if (!useExistingDuration) {
dev@196 209 this.timeline.pixelsPerSecond = pixelsPerSecond;
dev@196 210 this.timeline.visibleWidth = width;
dev@196 211 }
dev@180 212 } else {
dev@180 213 this.timeline = new wavesUI.core.Timeline(pixelsPerSecond, width);
dev@180 214 }
dev@196 215 const waveTrack = this.timeline.createTrack(
dev@196 216 track,
dev@196 217 height,
dev@196 218 `wave-${this.trackIdPrefix}`
dev@196 219 );
dev@196 220 if (isInitialRender && hasExistingTimeline) {
dev@196 221 // time axis
dev@196 222 const timeAxis = new wavesUI.helpers.TimeAxisLayer({
dev@196 223 height: height,
dev@196 224 color: '#b0b0b0'
dev@196 225 });
dev@196 226 this.addLayer(timeAxis, waveTrack, this.timeline.timeContext, true);
dev@196 227 this.cursorLayer = new wavesUI.helpers.CursorLayer({
dev@196 228 height: height
dev@196 229 });
dev@196 230 this.addLayer(this.cursorLayer, waveTrack, this.timeline.timeContext);
dev@196 231 }
dev@196 232 if ('ontouchstart' in window) {
dev@196 233 interface Point {
dev@196 234 x: number;
dev@196 235 y: number;
dev@196 236 }
dev@196 237
dev@196 238 let zoomGestureJustEnded: boolean = false;
dev@196 239
dev@196 240 const pixelToExponent: Function = wavesUI.utils.scales.linear()
dev@196 241 .domain([0, 100]) // 100px => factor 2
dev@196 242 .range([0, 1]);
dev@196 243
dev@196 244 const calculateDistance: (p1: Point, p2: Point) => number = (p1, p2) => {
dev@196 245 return Math.pow(
dev@196 246 Math.pow(p2.x - p1.x, 2) +
dev@196 247 Math.pow(p2.y - p1.y, 2), 0.5);
dev@196 248 };
dev@196 249
dev@205 250 const calculateMidPoint: (p1: Point, p2: Point) => Point = (p1, p2) => {
dev@205 251 return {
dev@205 252 x: 0.5 * (p1.x + p2.x),
dev@205 253 y: 0.5 * (p1.y + p2.y)
dev@205 254 };
dev@205 255 };
dev@205 256
dev@205 257 const hammertime = new Hammer.Manager(this.trackDiv.nativeElement, {
dev@205 258 recognizers: [
dev@205 259 [Hammer.Pan, { direction: Hammer.DIRECTION_HORIZONTAL }]
dev@205 260 ]
dev@205 261 });
dev@204 262
dev@204 263 // it seems HammerJs binds the event to the window?
dev@204 264 // causing these events to propagate to other components?
dev@204 265 const componentTimeline = this.timeline;
dev@204 266 let initialZoom;
dev@204 267 let initialDistance;
dev@204 268 let offsetAtPanStart;
dev@205 269 let startX;
dev@205 270 let isZooming;
dev@204 271
dev@196 272 const scroll = (ev) => {
dev@205 273 if (ev.center.x - startX === 0) return;
dev@196 274 if (zoomGestureJustEnded) {
dev@196 275 zoomGestureJustEnded = false;
dev@196 276 console.log("Skip this event: likely a single touch dangling from pinch");
dev@196 277 return;
dev@196 278 }
dev@204 279 componentTimeline.timeContext.offset = offsetAtPanStart +
dev@204 280 componentTimeline.timeContext.timeToPixel.invert(ev.deltaX);
dev@204 281 componentTimeline.tracks.update();
dev@196 282 };
dev@196 283
dev@196 284 const zoom = (ev) => {
dev@218 285 if (ev.touches.length < 2) return;
dev@214 286 ev.preventDefault();
dev@204 287 const minZoom = componentTimeline.state.minZoom;
dev@204 288 const maxZoom = componentTimeline.state.maxZoom;
dev@205 289 const p1: Point = {
dev@218 290 x: ev.touches[0].clientX,
dev@218 291 y: ev.touches[0].clientY
dev@205 292 };
dev@205 293 const p2: Point = {
dev@218 294 x: ev.touches[1].clientX,
dev@218 295 y: ev.touches[1].clientY
dev@205 296 };
dev@205 297 const distance = calculateDistance(p1, p2);
dev@205 298 const midPoint = calculateMidPoint(p1, p2);
dev@196 299
dev@196 300 const lastCenterTime =
dev@205 301 componentTimeline.timeContext.timeToPixel.invert(midPoint.x);
dev@196 302
dev@204 303 const exponent = pixelToExponent(distance - initialDistance);
dev@204 304 const targetZoom = initialZoom * Math.pow(2, exponent);
dev@196 305
dev@204 306 componentTimeline.timeContext.zoom =
dev@196 307 Math.min(Math.max(targetZoom, minZoom), maxZoom);
dev@196 308
dev@196 309 const newCenterTime =
dev@205 310 componentTimeline.timeContext.timeToPixel.invert(midPoint.x);
dev@196 311
dev@204 312 componentTimeline.timeContext.offset += newCenterTime - lastCenterTime;
dev@204 313 componentTimeline.tracks.update();
dev@196 314 };
dev@205 315 hammertime.on('panstart', (ev) => {
dev@204 316 offsetAtPanStart = componentTimeline.timeContext.offset;
dev@205 317 startX = ev.center.x;
dev@196 318 });
dev@196 319 hammertime.on('panleft', scroll);
dev@196 320 hammertime.on('panright', scroll);
dev@205 321
dev@205 322
dev@205 323 const element: HTMLElement = this.trackDiv.nativeElement;
dev@205 324 element.addEventListener('touchstart', (e) => {
dev@218 325 if (e.touches.length < 2) return;
dev@205 326 isZooming = true;
dev@204 327 initialZoom = componentTimeline.timeContext.zoom;
dev@196 328
dev@204 329 initialDistance = calculateDistance({
dev@218 330 x: e.touches[0].clientX,
dev@218 331 y: e.touches[0].clientY
dev@196 332 }, {
dev@218 333 x: e.touches[1].clientX,
dev@218 334 y: e.touches[1].clientY
dev@196 335 });
dev@196 336 });
dev@205 337 element.addEventListener('touchend', () => {
dev@205 338 if (isZooming) {
dev@205 339 isZooming = false;
dev@205 340 zoomGestureJustEnded = true;
dev@205 341 }
dev@205 342 });
dev@205 343 element.addEventListener('touchmove', zoom);
dev@196 344 }
dev@189 345 // this.timeline.createTrack(track, height/2, `wave-${this.trackIdPrefix}`);
dev@189 346 // this.timeline.createTrack(track, height/2, `grid-${this.trackIdPrefix}`);
dev@54 347 }
dev@18 348
cannam@108 349 estimatePercentile(matrix, percentile) {
cannam@108 350 // our sample is not evenly distributed across the whole data set:
cannam@108 351 // it is guaranteed to include at least one sample from every
cannam@108 352 // column, and could sample some values more than once. But it
cannam@108 353 // should be good enough in most cases (todo: show this)
cannam@109 354 if (matrix.length === 0) {
cannam@109 355 return 0.0;
cannam@109 356 }
cannam@108 357 const w = matrix.length;
cannam@108 358 const h = matrix[0].length;
cannam@108 359 const n = w * h;
cannam@109 360 const m = (n > 50000 ? 50000 : n); // should base that on the %ile
cannam@108 361 let m_per = Math.floor(m / w);
cannam@108 362 if (m_per < 1) m_per = 1;
cannam@108 363 let sample = [];
cannam@108 364 for (let x = 0; x < w; ++x) {
cannam@108 365 for (let i = 0; i < m_per; ++i) {
cannam@108 366 const y = Math.floor(Math.random() * h);
cannam@109 367 const value = matrix[x][y];
cannam@109 368 if (!isNaN(value) && value !== Infinity) {
cannam@109 369 sample.push(value);
cannam@109 370 }
cannam@108 371 }
cannam@108 372 }
cannam@109 373 if (sample.length === 0) {
cannam@109 374 console.log("WARNING: No samples gathered, even though we hoped for " +
cannam@109 375 (m_per * w) + " of them");
cannam@109 376 return 0.0;
cannam@109 377 }
cannam@108 378 sample.sort((a,b) => { return a - b; });
cannam@108 379 const ix = Math.floor((sample.length * percentile) / 100);
cannam@108 380 console.log("Estimating " + percentile + "-%ile of " +
cannam@108 381 n + "-sample dataset (" + w + " x " + h + ") as value " + ix +
cannam@108 382 " of sorted " + sample.length + "-sample subset");
cannam@108 383 const estimate = sample[ix];
cannam@108 384 console.log("Estimate is: " + estimate + " (where min sampled value = " +
cannam@108 385 sample[0] + " and max = " + sample[sample.length-1] + ")");
cannam@108 386 return estimate;
cannam@108 387 }
cannam@108 388
cannam@108 389 interpolatingMapper(hexColours) {
cannam@108 390 const colours = hexColours.map(n => {
cannam@108 391 const i = parseInt(n, 16);
cannam@118 392 return [ ((i >> 16) & 255) / 255.0,
cannam@118 393 ((i >> 8) & 255) / 255.0,
cannam@118 394 ((i) & 255) / 255.0 ];
cannam@108 395 });
cannam@108 396 const last = colours.length - 1;
cannam@108 397 return (value => {
cannam@108 398 const m = value * last;
cannam@108 399 if (m >= last) {
cannam@108 400 return colours[last];
cannam@108 401 }
cannam@108 402 if (m <= 0) {
cannam@108 403 return colours[0];
cannam@108 404 }
cannam@108 405 const base = Math.floor(m);
cannam@108 406 const prop0 = base + 1.0 - m;
cannam@108 407 const prop1 = m - base;
cannam@108 408 const c0 = colours[base];
cannam@108 409 const c1 = colours[base+1];
cannam@118 410 return [ c0[0] * prop0 + c1[0] * prop1,
cannam@118 411 c0[1] * prop0 + c1[1] * prop1,
cannam@118 412 c0[2] * prop0 + c1[2] * prop1 ];
cannam@108 413 });
cannam@108 414 }
dev@110 415
cannam@108 416 iceMapper() {
dev@110 417 let hexColours = [
cannam@108 418 // Based on ColorBrewer ylGnBu
cannam@108 419 "ffffff", "ffff00", "f7fcf0", "e0f3db", "ccebc5", "a8ddb5",
cannam@108 420 "7bccc4", "4eb3d3", "2b8cbe", "0868ac", "084081", "042040"
cannam@108 421 ];
cannam@108 422 hexColours.reverse();
cannam@108 423 return this.interpolatingMapper(hexColours);
cannam@108 424 }
dev@110 425
cannam@118 426 hsv2rgb(h, s, v) { // all values in range [0, 1]
cannam@118 427 const i = Math.floor(h * 6);
cannam@118 428 const f = h * 6 - i;
cannam@118 429 const p = v * (1 - s);
cannam@118 430 const q = v * (1 - f * s);
cannam@118 431 const t = v * (1 - (1 - f) * s);
cannam@118 432 let r = 0, g = 0, b = 0;
cannam@118 433 switch (i % 6) {
cannam@118 434 case 0: r = v, g = t, b = p; break;
cannam@118 435 case 1: r = q, g = v, b = p; break;
cannam@118 436 case 2: r = p, g = v, b = t; break;
cannam@118 437 case 3: r = p, g = q, b = v; break;
cannam@118 438 case 4: r = t, g = p, b = v; break;
cannam@118 439 case 5: r = v, g = p, b = q; break;
cannam@118 440 }
cannam@118 441 return [ r, g, b ];
cannam@118 442 }
dev@122 443
cannam@118 444 greenMapper() {
cannam@118 445 const blue = 0.6666;
cannam@118 446 const pieslice = 0.3333;
cannam@118 447 return (value => {
cannam@118 448 const h = blue - value * 2.0 * pieslice;
cannam@118 449 const s = 0.5 + value / 2.0;
cannam@118 450 const v = value;
cannam@118 451 return this.hsv2rgb(h, s, v);
cannam@118 452 });
cannam@118 453 }
cannam@118 454
cannam@118 455 sunsetMapper() {
cannam@118 456 return (value => {
cannam@118 457 let r = (value - 0.24) * 2.38;
cannam@118 458 let g = (value - 0.64) * 2.777;
cannam@118 459 let b = (3.6 * value);
cannam@118 460 if (value > 0.277) b = 2.0 - b;
cannam@118 461 return [ r, g, b ];
cannam@118 462 });
cannam@118 463 }
cannam@118 464
dev@122 465 clearTimeline(): void {
dev@122 466 // loop through layers and remove them, waves-ui provides methods for this but it seems to not work properly
dev@122 467 const timeContextChildren = this.timeline.timeContext._children;
dev@122 468 for (let track of this.timeline.tracks) {
dev@122 469 if (track.layers.length === 0) { continue; }
dev@122 470 const trackLayers = Array.from(track.layers);
dev@122 471 while (trackLayers.length) {
dev@122 472 let layer: Layer = trackLayers.pop();
dev@185 473 if (this.layers.includes(layer)) {
dev@185 474 track.remove(layer);
dev@185 475 this.layers.splice(this.layers.indexOf(layer), 1);
dev@185 476 const index = timeContextChildren.indexOf(layer.timeContext);
dev@185 477 if (index >= 0) {
dev@185 478 timeContextChildren.splice(index, 1);
dev@185 479 }
dev@185 480 layer.destroy();
dev@122 481 }
dev@122 482 }
dev@122 483 }
dev@122 484 }
dev@122 485
dev@54 486 renderWaveform(buffer: AudioBuffer): void {
dev@180 487 // const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height / 2;
dev@180 488 const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height;
dev@189 489 const waveTrack = this.timeline.getTrackById(`wave-${this.trackIdPrefix}`);
dev@54 490 if (this.timeline) {
dev@54 491 // resize
dev@54 492 const width = this.trackDiv.nativeElement.getBoundingClientRect().width;
dev@55 493
dev@122 494 this.clearTimeline();
dev@59 495
dev@54 496 this.timeline.visibleWidth = width;
dev@54 497 this.timeline.pixelsPerSecond = width / buffer.duration;
cannam@117 498 waveTrack.height = height;
dev@54 499 } else {
dev@180 500 this.renderTimeline(buffer.duration)
dev@54 501 }
dev@83 502 this.timeline.timeContext.offset = 0.5 * this.timeline.timeContext.visibleDuration;
cannam@106 503
dev@18 504 // time axis
dev@18 505 const timeAxis = new wavesUI.helpers.TimeAxisLayer({
dev@18 506 height: height,
cannam@106 507 color: '#b0b0b0'
dev@18 508 });
cannam@117 509 this.addLayer(timeAxis, waveTrack, this.timeline.timeContext, true);
dev@18 510
cannam@161 511 const nchannels = buffer.numberOfChannels;
cannam@161 512 const totalWaveHeight = height * 0.9;
cannam@161 513 const waveHeight = totalWaveHeight / nchannels;
dev@189 514
cannam@161 515 for (let ch = 0; ch < nchannels; ++ch) {
cannam@161 516 console.log("about to construct a waveform layer for channel " + ch);
cannam@161 517 const waveformLayer = new wavesUI.helpers.WaveformLayer(buffer, {
cannam@161 518 top: (height - totalWaveHeight)/2 + waveHeight * ch,
cannam@161 519 height: waveHeight,
cannam@161 520 color: 'darkblue',
cannam@161 521 channel: ch
cannam@161 522 });
cannam@161 523 this.addLayer(waveformLayer, waveTrack, this.timeline.timeContext);
cannam@161 524 }
cannam@117 525
dev@53 526 this.cursorLayer = new wavesUI.helpers.CursorLayer({
dev@31 527 height: height
dev@31 528 });
cannam@117 529 this.addLayer(this.cursorLayer, waveTrack, this.timeline.timeContext);
dev@51 530 this.timeline.state = new wavesUI.states.CenteredZoomState(this.timeline);
cannam@117 531 waveTrack.render();
cannam@117 532 waveTrack.update();
dev@81 533
dev@196 534 this.isLoading = false;
dev@53 535 this.animate();
dev@53 536 }
dev@53 537
cannam@117 538 renderSpectrogram(buffer: AudioBuffer): void {
cannam@117 539 const height: number = this.trackDiv.nativeElement.getBoundingClientRect().height / 2;
dev@189 540 const gridTrack = this.timeline.getTrackById(`grid-${this.trackIdPrefix}`);
cannam@117 541
dev@129 542 const spectrogramLayer = new WavesSpectrogramLayer(buffer, {
cannam@221 543 top: 0,
cannam@221 544 height: height,
cannam@117 545 stepSize: 512,
dev@129 546 blockSize: 1024,
cannam@118 547 normalise: 'none',
cannam@118 548 mapper: this.sunsetMapper()
cannam@117 549 });
cannam@117 550 this.addLayer(spectrogramLayer, gridTrack, this.timeline.timeContext);
cannam@117 551
cannam@117 552 this.timeline.tracks.update();
cannam@117 553 }
cannam@117 554
dev@53 555 // TODO refactor - this doesn't belong here
dev@64 556 private renderFeatures(extracted: SimpleResponse, colour: Colour): void {
dev@196 557 if (this.isOneShotExtractor && !this.hasShot) {
dev@196 558 this.featureExtractionSubscription.unsubscribe();
dev@196 559 this.hasShot = true;
dev@196 560 }
dev@196 561
dev@64 562 if (!extracted.hasOwnProperty('features') || !extracted.hasOwnProperty('outputDescriptor')) return;
dev@64 563 if (!extracted.features.hasOwnProperty('shape') || !extracted.features.hasOwnProperty('data')) return;
dev@64 564 const features: FeatureCollection = (extracted.features as FeatureCollection);
dev@64 565 const outputDescriptor = extracted.outputDescriptor;
dev@196 566 // const height = this.trackDiv.nativeElement.getBoundingClientRect().height / 2;
dev@196 567 const height = this.trackDiv.nativeElement.getBoundingClientRect().height;
dev@189 568 const waveTrack = this.timeline.getTrackById(`wave-${this.trackIdPrefix}`);
dev@64 569
dev@64 570 // TODO refactor all of this
dev@63 571 switch (features.shape) {
dev@64 572 case 'vector': {
dev@63 573 const stepDuration = (features as FixedSpacedFeatures).stepDuration;
dev@63 574 const featureData = (features.data as Float32Array);
dev@68 575 if (featureData.length === 0) return;
dev@63 576 const normalisationFactor = 1.0 /
dev@63 577 featureData.reduce(
dev@63 578 (currentMax, feature) => Math.max(currentMax, feature),
dev@63 579 -Infinity
dev@63 580 );
dev@67 581
dev@63 582 const plotData = [...featureData].map((feature, i) => {
dev@63 583 return {
dev@63 584 cx: i * stepDuration,
dev@63 585 cy: feature * normalisationFactor
dev@63 586 };
dev@63 587 });
dev@67 588
dev@105 589 let lineLayer = new wavesUI.helpers.LineLayer(plotData, {
dev@63 590 color: colour,
dev@64 591 height: height
dev@63 592 });
dev@122 593 this.addLayer(
dev@105 594 lineLayer,
cannam@117 595 waveTrack,
dev@63 596 this.timeline.timeContext
dev@122 597 );
dev@63 598 break;
dev@64 599 }
dev@64 600 case 'list': {
dev@64 601 const featureData = (features.data as FeatureList);
dev@68 602 if (featureData.length === 0) return;
dev@64 603 // TODO look at output descriptor instead of directly inspecting features
dev@64 604 const hasDuration = outputDescriptor.configured.hasDuration;
dev@64 605 const isMarker = !hasDuration
dev@64 606 && outputDescriptor.configured.binCount === 0
dev@64 607 && featureData[0].featureValues == null;
dev@64 608 const isRegion = hasDuration
dev@64 609 && featureData[0].timestamp != null;
cannam@149 610 console.log("Have list features: length " + featureData.length +
cannam@149 611 ", isMarker " + isMarker + ", isRegion " + isRegion +
cannam@149 612 ", hasDuration " + hasDuration);
dev@64 613 // TODO refactor, this is incomprehensible
dev@64 614 if (isMarker) {
dev@64 615 const plotData = featureData.map(feature => {
cannam@152 616 return {
cannam@152 617 time: toSeconds(feature.timestamp),
cannam@152 618 label: feature.label
cannam@152 619 }
dev@64 620 });
cannam@149 621 let featureLayer = new wavesUI.helpers.TickLayer(plotData, {
dev@64 622 height: height,
dev@64 623 color: colour,
cannam@152 624 labelPosition: 'bottom',
cannam@152 625 shadeSegments: true
dev@64 626 });
dev@122 627 this.addLayer(
cannam@149 628 featureLayer,
cannam@117 629 waveTrack,
dev@64 630 this.timeline.timeContext
dev@122 631 );
dev@64 632 } else if (isRegion) {
cannam@149 633 console.log("Output is of region type");
dev@67 634 const binCount = outputDescriptor.configured.binCount || 0;
dev@67 635 const isBarRegion = featureData[0].featureValues.length >= 1 || binCount >= 1 ;
dev@64 636 const getSegmentArgs = () => {
dev@64 637 if (isBarRegion) {
dev@64 638
dev@67 639 // TODO refactor - this is messy
dev@67 640 interface FoldsToNumber<T> {
dev@67 641 reduce(fn: (previousValue: number,
dev@67 642 currentValue: T,
dev@67 643 currentIndex: number,
dev@67 644 array: ArrayLike<T>) => number,
dev@67 645 initialValue?: number): number;
dev@67 646 }
dev@64 647
dev@67 648 // TODO potentially change impl., i.e avoid reduce
dev@67 649 const findMin = <T>(arr: FoldsToNumber<T>, getElement: (x: T) => number): number => {
dev@67 650 return arr.reduce((min, val) => Math.min(min, getElement(val)), Infinity);
dev@67 651 };
dev@67 652
dev@67 653 const findMax = <T>(arr: FoldsToNumber<T>, getElement: (x: T) => number): number => {
dev@67 654 return arr.reduce((min, val) => Math.max(min, getElement(val)), -Infinity);
dev@67 655 };
dev@67 656
dev@67 657 const min = findMin<Feature>(featureData, (x: Feature) => {
dev@67 658 return findMin<number>(x.featureValues, y => y);
dev@67 659 });
dev@67 660
dev@67 661 const max = findMax<Feature>(featureData, (x: Feature) => {
dev@67 662 return findMax<number>(x.featureValues, y => y);
dev@67 663 });
dev@67 664
dev@67 665 const barHeight = 1.0 / height;
dev@64 666 return [
dev@67 667 featureData.reduce((bars, feature) => {
dev@67 668 const staticProperties = {
dev@64 669 x: toSeconds(feature.timestamp),
dev@64 670 width: toSeconds(feature.duration),
dev@67 671 height: min + barHeight,
dev@64 672 color: colour,
dev@64 673 opacity: 0.8
dev@67 674 };
dev@67 675 // TODO avoid copying Float32Array to an array - map is problematic here
dev@67 676 return bars.concat([...feature.featureValues]
dev@67 677 .map(val => Object.assign({}, staticProperties, {y: val})))
dev@67 678 }, []),
dev@67 679 {yDomain: [min, max + barHeight], height: height} as any
dev@67 680 ];
dev@64 681 } else {
dev@64 682 return [featureData.map(feature => {
dev@64 683 return {
dev@64 684 x: toSeconds(feature.timestamp),
dev@64 685 width: toSeconds(feature.duration),
dev@64 686 color: colour,
dev@64 687 opacity: 0.8
dev@64 688 }
dev@64 689 }), {height: height}];
dev@64 690 }
dev@64 691 };
dev@64 692
dev@64 693 let segmentLayer = new wavesUI.helpers.SegmentLayer(
dev@64 694 ...getSegmentArgs()
dev@64 695 );
dev@122 696 this.addLayer(
dev@64 697 segmentLayer,
cannam@117 698 waveTrack,
dev@64 699 this.timeline.timeContext
dev@122 700 );
dev@64 701 }
dev@64 702 break;
dev@64 703 }
cannam@106 704 case 'matrix': {
cannam@108 705 const stepDuration = (features as FixedSpacedFeatures).stepDuration;
cannam@120 706 //!!! + start time
cannam@108 707 const matrixData = (features.data as Float32Array[]);
cannam@108 708 if (matrixData.length === 0) return;
cannam@109 709 console.log("matrix data length = " + matrixData.length);
cannam@109 710 console.log("height of first column = " + matrixData[0].length);
cannam@109 711 const targetValue = this.estimatePercentile(matrixData, 95);
cannam@108 712 const gain = (targetValue > 0.0 ? (1.0 / targetValue) : 1.0);
cannam@108 713 console.log("setting gain to " + gain);
cannam@120 714 const matrixEntity =
cannam@120 715 new wavesUI.utils.PrefilledMatrixEntity(matrixData,
cannam@120 716 0, // startTime
cannam@120 717 stepDuration);
cannam@108 718 let matrixLayer = new wavesUI.helpers.MatrixLayer(matrixEntity, {
cannam@108 719 gain,
cannam@221 720 top: 0,
cannam@221 721 height: height,
cannam@109 722 normalise: 'none',
cannam@108 723 mapper: this.iceMapper()
cannam@108 724 });
dev@122 725 this.addLayer(
cannam@108 726 matrixLayer,
cannam@117 727 waveTrack,
cannam@108 728 this.timeline.timeContext
dev@122 729 );
cannam@108 730 break;
cannam@106 731 }
dev@67 732 default:
cannam@106 733 console.log("Cannot render an appropriate layer for feature shape '" +
cannam@106 734 features.shape + "'");
dev@63 735 }
dev@59 736
dev@196 737 this.isLoading = false;
dev@56 738 this.timeline.tracks.update();
dev@53 739 }
dev@53 740
dev@53 741 private animate(): void {
dev@196 742 if (!this.isSeeking) return;
dev@196 743
dev@31 744 this.ngZone.runOutsideAngular(() => {
dev@31 745 // listen for time passing...
dev@31 746 const updateSeekingCursor = () => {
dev@53 747 const currentTime = this.audioService.getCurrentTime();
dev@53 748 this.cursorLayer.currentPosition = currentTime;
dev@53 749 this.cursorLayer.update();
dev@53 750
dev@53 751 const currentOffset = this.timeline.timeContext.offset;
dev@53 752 const offsetTimestamp = currentOffset
dev@53 753 + currentTime;
dev@53 754
dev@53 755 const visibleDuration = this.timeline.timeContext.visibleDuration;
dev@53 756 // TODO reduce duplication between directions and make more declarative
dev@53 757 // this kinda logic should also be tested
dev@53 758 const mustPageForward = offsetTimestamp > visibleDuration;
dev@53 759 const mustPageBackward = currentTime < -currentOffset;
dev@53 760
dev@53 761 if (mustPageForward) {
dev@53 762 const hasSkippedMultiplePages = offsetTimestamp - visibleDuration > visibleDuration;
dev@53 763
cannam@106 764 this.timeline.timeContext.offset = hasSkippedMultiplePages ?
cannam@106 765 -currentTime + 0.5 * visibleDuration :
cannam@106 766 currentOffset - visibleDuration;
dev@51 767 this.timeline.tracks.update();
dev@34 768 }
dev@53 769
dev@53 770 if (mustPageBackward) {
dev@53 771 const hasSkippedMultiplePages = currentTime + visibleDuration < -currentOffset;
cannam@106 772 this.timeline.timeContext.offset = hasSkippedMultiplePages ?
cannam@106 773 -currentTime + 0.5 * visibleDuration :
cannam@106 774 currentOffset + visibleDuration;
dev@51 775 this.timeline.tracks.update();
dev@34 776 }
dev@53 777
dev@53 778 if (this.isPlaying)
dev@53 779 requestAnimationFrame(updateSeekingCursor);
dev@31 780 };
dev@31 781 updateSeekingCursor();
dev@31 782 });
dev@6 783 }
dev@16 784
dev@122 785 private addLayer(layer: Layer, track: Track, timeContext: any, isAxis: boolean = false): void {
dev@54 786 timeContext.zoom = 1.0;
dev@54 787 if (!layer.timeContext) {
dev@54 788 layer.setTimeContext(isAxis ?
dev@54 789 timeContext : new wavesUI.core.LayerTimeContext(timeContext));
dev@54 790 }
dev@54 791 track.add(layer);
dev@185 792 this.layers.push(layer);
dev@54 793 layer.render();
dev@54 794 layer.update();
dev@122 795 if (this.cursorLayer && track.$layout.contains(this.cursorLayer.$el)) {
dev@112 796 track.$layout.appendChild(this.cursorLayer.$el);
dev@112 797 }
dev@59 798 }
dev@59 799
dev@59 800 private static changeColour(layer: Layer, colour: string): void {
dev@59 801 const butcherShapes = (shape) => {
dev@59 802 shape.install({color: () => colour});
dev@59 803 shape.params.color = colour;
dev@59 804 shape.update(layer._renderingContext, layer.data);
dev@59 805 };
dev@59 806
dev@59 807 layer._$itemCommonShapeMap.forEach(butcherShapes);
dev@59 808 layer._$itemShapeMap.forEach(butcherShapes);
dev@59 809 layer.render();
dev@59 810 layer.update();
dev@54 811 }
dev@54 812
dev@51 813 ngOnDestroy(): void {
dev@196 814 if (this.featureExtractionSubscription)
dev@196 815 this.featureExtractionSubscription.unsubscribe();
dev@196 816 if (this.playingStateSubscription)
dev@196 817 this.playingStateSubscription.unsubscribe();
dev@196 818 if (this.seekedSubscription)
dev@196 819 this.seekedSubscription.unsubscribe();
dev@196 820 if (this.onAudioDataSubscription)
dev@196 821 this.onAudioDataSubscription.unsubscribe();
dev@51 822 }
dev@154 823
dev@155 824 seekStart(): void {
dev@155 825 this.zoomOnMouseDown = this.timeline.timeContext.zoom;
dev@157 826 this.offsetOnMouseDown = this.timeline.timeContext.offset;
dev@155 827 }
dev@155 828
dev@155 829 seekEnd(x: number): void {
dev@157 830 const hasSameZoom: boolean = this.zoomOnMouseDown ===
dev@157 831 this.timeline.timeContext.zoom;
dev@157 832 const hasSameOffset: boolean = this.offsetOnMouseDown ===
dev@157 833 this.timeline.timeContext.offset;
dev@157 834 if (hasSameZoom && hasSameOffset) {
dev@155 835 this.seek(x);
dev@155 836 }
dev@155 837 }
dev@155 838
dev@154 839 seek(x: number): void {
dev@154 840 if (this.timeline) {
dev@154 841 const timeContext: any = this.timeline.timeContext;
dev@196 842 if (this.isSeeking) {
dev@196 843 this.audioService.seekTo(
dev@196 844 timeContext.timeToPixel.invert(x)- timeContext.offset
dev@196 845 );
dev@196 846 }
dev@154 847 }
dev@154 848 }
dev@6 849 }