Mercurial > hg > match
changeset 0:9feddf959b6b
* Import MATCH v0.9.2
author | Chris Cannam |
---|---|
date | Fri, 08 Oct 2010 16:02:41 +0100 |
parents | |
children | 88f50ba37174 |
files | META-INF/MANIFEST.MF at/ofai/music/match/AligningAudioPlayer.java at/ofai/music/match/AudioFile.java at/ofai/music/match/Finder.java at/ofai/music/match/FixedPoint.java at/ofai/music/match/GUI.java at/ofai/music/match/Help.java at/ofai/music/match/MatrixFrame.java at/ofai/music/match/Path.java at/ofai/music/match/PerformanceMatcher.java at/ofai/music/match/ScrollingMatrix.java at/ofai/music/match/WormHandler.java |
diffstat | 12 files changed, 4976 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/META-INF/MANIFEST.MF Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,4 @@ +Manifest-Version: 1.0 +Created-By: 1.4.0 (Sun Microsystems Inc.) +Main-Class: at.ofai.music.match.PerformanceMatcher +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/AligningAudioPlayer.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,670 @@ +package at.ofai.music.match; + +import java.util.LinkedList; +import java.util.ListIterator; + +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; +import javax.swing.event.ChangeListener; +import javax.swing.event.ChangeEvent; +import javax.swing.JFileChooser; +import javax.swing.JOptionPane; +import java.io.File; +import java.io.FileWriter; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.BufferedReader; + +public class AligningAudioPlayer implements ChangeListener, Runnable { + + protected GUI gui; + protected PerformanceMatcher pm1, pm2; + protected ScrollingMatrix sm; + protected Matcher matcher; + protected Thread matchThread; + protected LinkedList<AudioFile> files; + protected LinkedList<Long> marks; + protected JFileChooser jfc; + protected long currentPosition, requestedPosition; + protected boolean playFromMark; + protected boolean stopRequested; + protected AudioFile currentFile, requestedFile; + protected boolean playing; + protected SourceDataLine audioOut; + protected int outputBufferSize; + protected long audioLength; + protected byte[] readBuffer, readBuffer2; +// public static boolean showMatch = false; + protected static final int readBufferSize = 2048; // 46ms@44.1kHz + protected static final int defaultOutputBufferSize = 16384; +// protected static final String[] pmArgs = {"-b","-q"}; // use PM defaults + // ,"-n1","-n4","-s","90","-h",".02","-f",".046"}; + + public AligningAudioPlayer(GUI g, PerformanceMatcher p1, + PerformanceMatcher p2, + ScrollingMatrix s) { + gui = g; + pm1 = p1; + pm2 = p2; + sm = s; + // PerformanceMatcher.processArgs(pm1, pm2, pmArgs); + files = new LinkedList<AudioFile>(); + currentFile = null; + requestedFile = null; + currentPosition = 0; + playFromMark = false; + requestedPosition = -1; + stopRequested = false; + playing = false; + outputBufferSize = 0; + marks = new LinkedList<Long>(); + readBuffer = new byte[readBufferSize]; + readBuffer2 = new byte[readBufferSize]; + matcher = new Matcher(g); + jfc = gui.fileChooser; + new Thread(this).start(); // for audio playback + matchThread = new Thread(matcher); // for aligning audio files + matchThread.start(); + } // constructor + + public int getFileCount() { + return files.size(); + } // getFileCount() + + public void play() { + synchronized(this) { + if (!playing) + notify(); + } + } // play() + + public void pause() { + if (playing) + stopRequested = true; + } // pause() + + public void stop() { + if (playing) { + stopRequested = true; + requestedPosition = 0; + } else + setPosition(0); + } // stop() + + public void togglePlay() { + if (playing) + pause(); + else + play(); + } // togglePlay() + + public void setMode(boolean fromMark) { + playFromMark = fromMark; + } // setMode + + protected void setPosition(long positionRequested) { + if (requestedFile != null) { + currentFile = requestedFile; + requestedFile = null; + } + if (currentFile != null) { + try { + currentPosition = currentFile.setPosition(positionRequested); + if (currentPosition != positionRequested) + System.err.println("setPosition() failed: " + + currentPosition + " instead of " + positionRequested); + // else System.err.println("setPosition: " + currentPosition); + } catch (java.io.IOException e) { + e.printStackTrace(); + } + updateGUI(); + } + } // setPosition() + + protected void updateGUI() { + gui.setSlider(((double)currentPosition) / currentFile.length); + gui.setTimer(currentPosition / currentFile.frameSize / + currentFile.frameRate, currentFile); + } // updateGUI() + + public void skipToNextMark() { + if (currentFile != null) { + ListIterator<Long> i = marks.listIterator(); + long newPosn = currentFile.length; + while (i.hasNext()) { + long l = i.next().longValue(); + if (l > currentPosition) { + newPosn = l; + break; + } + } + if (playing) + requestedPosition = newPosn; + else + setPosition(newPosn); + } + } // skipToNextMark() + + public void skipToPreviousMark() { + if (currentFile != null) { + ListIterator<Long> i = marks.listIterator(); + long newPosn = 0; + while (i.hasNext()) { + long l = i.next().longValue(); + if (l >= currentPosition) + break; + newPosn = l; + } + if (playing) + requestedPosition = newPosn; + else + setPosition(newPosn); + } + } // skipToPreviousMark() + + public void addMark() { + if (currentFile != null) { + long newMark = correctedPosition(); + ListIterator<Long> i = marks.listIterator(); + while (i.hasNext()) { + long l = i.next().longValue(); + if (l == newMark) { + i.remove(); + gui.updateMarks(); + return; + } + if (l > newMark) { + i.previous(); + break; + } + } + i.add(new Long(newMark)); + gui.updateMarks(); + } + } // addMark() + + public long correctedPosition() { + if (audioOut == null) + return currentPosition; + return currentPosition - (outputBufferSize - audioOut.available()); + } // correctedPosition() + + public ListIterator<Long> getMarkListIterator() { + return marks.listIterator(); + } // getMarkListIterator() + + public long getCurrentFileLength() { + if (requestedFile != null) { + try { // avoid race conditions + return requestedFile.length; + } catch (NullPointerException e) {} + } + try { // avoid race conditions + return currentFile.length; + } catch (NullPointerException e) {} + return 0; + } // getCurrentFileLength() + + public void save() { + File f = null; + if ((jfc.showSaveDialog(null) == JFileChooser.APPROVE_OPTION) && + (!jfc.getSelectedFile().exists() || + (JOptionPane.showConfirmDialog(null, + "File " + jfc.getSelectedFile().getAbsolutePath() + + " exists.\nDo you want to replace it?", "Are you sure?", + JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION))) + f = jfc.getSelectedFile(); + if (f == null) + return; + try { + BufferedWriter out = new BufferedWriter(new FileWriter(f)); + ListIterator<AudioFile> iter = files.listIterator(); + while (iter.hasNext()) { + AudioFile af = iter.next(); + out.write("File: " + af.path + "\n"); + if (af == currentFile) { + out.write("Marks: " + marks.size() + "\n"); + ListIterator<Long> iter2 = marks.listIterator(); + while (iter2.hasNext()) + out.write(iter2.next().longValue() + "\n"); + } else + out.write("Marks: -1\n"); + int count = 0; + FixedPoint p = af.fixedPoints; + while (p != null) { + p = p.prev; + count++; + } + out.write("FixedPoints: " + af.orientationX + " " + count+"\n"); + p = af.fixedPoints; + while (p != null) { + out.write(p.x + "\n"); + out.write(p.y + "\n"); + p = p.prev; + } + if (af.isReference) + out.write("0\n0\n0\n0\n"); + else { + out.write(af.thisHopTime + "\n"); + out.write(af.refHopTime + "\n"); + out.write(af.pathLength + "\n"); + for (int i = 0; i < af.pathLength; i++) + out.write(af.thisIndex[i] + "\n"); + out.write(af.pathLength + "\n"); + for (int i = 0; i < af.pathLength; i++) + out.write(af.refIndex[i] + "\n"); + } + } + out.close(); + } catch (java.io.IOException e) { + System.err.println("IOException while saving data"); + } + } // save() + + public void load() { + File f = null; + if (jfc.showOpenDialog(null) == JFileChooser.APPROVE_OPTION) + load(jfc.getSelectedFile().getAbsolutePath()); + } // load() + + public static String checkString(String in, String prefix) { + return checkString(in, prefix, true); + } // checkString() + + public static String checkString(String in, String prefix, boolean exc) { + if ((in != null) && in.startsWith(prefix)) + return in.substring(prefix.length()); + if (!exc) + return null; + throw new IllegalArgumentException("Expecting: "+prefix+"; got: "+in); + } // checkString() + + public void load(String fileName) { + File f = new File(fileName); + if (!f.exists()) { + System.err.println("File " + fileName + " does not exist."); + return; + } + try { + BufferedReader in = new BufferedReader(new FileReader(f)); + gui.setStatus(GUI.LOADING); + clearFiles(); + marks.clear(); + int[] thisIndex, refIndex; + int len; + while (true) { + String line = in.readLine(); + if (line == null) + break; + line = checkString(line, "File: "); + GUI.FileNameSelection selector = gui.addFile(line, false); + AudioFile af = new AudioFile(line, selector); + files.add(af); + line = checkString(in.readLine(), "Marks: "); + len = Integer.parseInt(line); + if (len >= 0) { + setCurrentFile(selector); + for (int i = 0; i < len; i++) + marks.add(new Long(Long.parseLong(in.readLine()))); + } + String tmp = in.readLine(); + line = checkString(tmp, "FixedPoints: ", false); + if (line != null) { + boolean isX = true; + if (line.startsWith("true ")) { + line = line.substring(5); + } else { + isX = false; + line = line.substring(6); + } + int ln = Integer.parseInt(line); + FixedPoint p = null; + FixedPoint q = null; + for (int i = 0; i < ln; i++) { + int x = Integer.parseInt(in.readLine()); + int y = Integer.parseInt(in.readLine()); + if (p == null) + p = FixedPoint.newList(0,0,x,y); // dummy start + else + q = p.insert(x,y); + } + if (q != null) + q.prev = null; // remove dummy start + af.setFixedPoints(p, isX); + line = in.readLine(); + } else + line = tmp; + double thisHopTime = Double.parseDouble(line); + double refHopTime = Double.parseDouble(in.readLine()); + len = Integer.parseInt(in.readLine()); + if (len > 0) { + thisIndex = new int[len]; + for (int i = 0; i < len; i++) + thisIndex[i] = Integer.parseInt(in.readLine()); + } else + thisIndex = null; + len = Integer.parseInt(in.readLine()); + if (len > 0) { + refIndex = new int[len]; + for (int i = 0; i < len; i++) + refIndex[i] = Integer.parseInt(in.readLine()); + af.setMatch(thisIndex, thisHopTime,refIndex,refHopTime,len); + } + selector.setFraction(1.0); + } + gui.setStatus(GUI.READY); + gui.updateMarks(); + } catch (IllegalArgumentException e) { + System.err.println("Load error: " + e); + clearFiles(); + marks.clear(); + } catch (java.io.IOException e) { + System.err.println("IOException while loading data"); + clearFiles(); + marks.clear(); + } + } // load() + + protected void clearFiles() { // not thread-safe + stop(); + while (playing) { + try { Thread.sleep(100); } catch (InterruptedException e) {} + stop(); // in case play() is called while clearing + } + matcher.stop(); + ListIterator<AudioFile> iter = files.listIterator(files.size()); + while (iter.hasPrevious()) { // remove backwards for gui consistency + AudioFile af = iter.previous(); + iter.remove(); + gui.removeFile(af.selector); + } + currentFile = null; // not thread-safe + } // clearFiles() + + public void addFile(String fileName, GUI.FileNameSelection selector) { + AudioFile af = new AudioFile(fileName, selector); + files.add(af); + if (files.size() == 1) { + setCurrentFile(selector); + selector.setFraction(1.0); + } + else + matcher.enqueue(af, files.get(0)); // align in background Thread + } // addFile() + + class Matcher implements Runnable { + + LinkedList<AudioFile> reference; + LinkedList<AudioFile> other; + GUI gui; + + public Matcher(GUI g) { + gui = g; + reference = new LinkedList<AudioFile>(); + other = new LinkedList<AudioFile>(); + } // constructor + + public void enqueue(AudioFile af, AudioFile ref) { + synchronized(this) { + reference.add(ref); + other.add(af); + notify(); + } + } // enqueue() + + public void stop() { + synchronized(this) { + reference.clear(); + other.clear(); + matchThread.interrupt(); + } + } // stop() + + public void run() { + AudioFile af, ref; + while (true) { + synchronized(this) { + if (reference.size() == 0) { + gui.setStatus(GUI.READY); + try { + wait(); + } catch (InterruptedException e) { + continue; // skip remove() since size is still 0 + } + gui.setStatus(GUI.ALIGNING); + } + ref = reference.remove(); + af = other.remove(); + } + match(af, ref); + } + } // run() + + public void match(AudioFile af, AudioFile ref) { + sm.init(); + pm1.setInputFile(ref); + pm2.setInputFile(af); + pm2.setProgressCallback(af.selector); + PerformanceMatcher.doMatch(pm1, pm2, sm); + if (Thread.interrupted()) { + sm.setVisible(false); + return; + } + if (sm.isVisible()) { + sm.updateMatrix(playing); + // if (!playing) + // updateGUI(); // to set the time in the MatrixFrame + } else + sm.updatePaths(false); + } // match() + + } // inner class Matcher + + public void setCurrentFile(int index) { + if (index < files.size()) + setCurrentFile(files.get(index)); + } // setCurrentFile() + + public void setCurrentFile(GUI.FileNameSelection selector) { + AudioFile newFile = null; + for (ListIterator<AudioFile> i = files.listIterator(); i.hasNext(); ) { + AudioFile f = i.next(); + if (f.selector == selector) { + newFile = f; + break; + } + } + setCurrentFile(newFile); + } // setCurrentFile() + + public void setCurrentFile(AudioFile newFile) { + if (newFile == null) + throw new RuntimeException("setCurrentFile(): null"); + if (newFile == currentFile) + return; + newFile.selector.setSelected(true); + if (currentFile != null) { + currentFile.selector.setSelected(false); + // translate marks to new sampling rate and tempo + for (ListIterator<Long> i = marks.listIterator(); i.hasNext(); ) { + long mark = i.next().longValue(); + double time = currentFile.toReferenceTime(mark); + mark = newFile.fromReferenceTime(time); + i.set(new Long(mark)); + } + // translate current playback position + double time = currentFile.toReferenceTime(correctedPosition()); + long newCurrentPosition = newFile.fromReferenceTime(time); + // rewind to previous mark if playing from mark + if (playFromMark) { + ListIterator<Long> i = marks.listIterator(); + long newPosn = 0; + while (i.hasNext()) { + long l = i.next().longValue(); + if (l >= newCurrentPosition) + break; + newPosn = l; + } + newCurrentPosition = newPosn; + } + // change file and position + synchronized(this) { // try to make it thread-safe + if (playing) { + requestedFile = newFile; + requestedPosition = newCurrentPosition; + } else { + currentFile = newFile; + setPosition(newCurrentPosition); + } + } + gui.updateMarks(); + } else + currentFile = newFile; + } // setCurrentFile() + + public void previousFile() { + if (files.size() > 1) { + int index = files.indexOf(currentFile) - 1; + if (index < 0) + index = files.size() - 1; + setCurrentFile(files.get(index)); + } + } // previousFile() + + public void nextFile() { + if (files.size() > 1) { + int index = files.indexOf(currentFile) + 1; + if (index == files.size()) + index = 0; + setCurrentFile(files.get(index)); + } + } // nextFile() + + /** Code for audio playback thread. Implements Runnable interface. */ + public void run() { + int bytesRead, bytesWritten; + while (true) { + try { + if ((currentFile == null) || stopRequested || !playing) { + synchronized(this) { + playing = false; + wait(); + playing = true; + stopRequested = false; + } + if (currentFile == null) + continue; + if (currentPosition == currentFile.length) + setPosition(0); + } + if (audioOut != null) { + audioOut.stop(); + audioOut.flush(); + } + if ((audioOut == null) || + !currentFile.format.matches(audioOut.getFormat())) { + audioOut= AudioSystem.getSourceDataLine(currentFile.format); + audioOut.open(currentFile.format, defaultOutputBufferSize); + outputBufferSize = audioOut.getBufferSize(); + } + audioOut.start(); + while (true) { // PLAY loop + synchronized(this) { + if ((requestedPosition < 0) && !stopRequested) + bytesRead = currentFile.audioIn.read(readBuffer); + else if (stopRequested || + ((requestedPosition >= 0) && + (requestedFile != null) && + !currentFile.format.matches( + requestedFile.format))) { + audioOut.stop(); + audioOut.flush(); // ?correct posn before flush? + if (requestedPosition >= 0) { + setPosition(requestedPosition); + requestedPosition = -1; + } + break; + } else { // requestedPosition >= 0 && format matches + bytesRead = currentFile.audioIn.read(readBuffer); + setPosition(requestedPosition); + requestedPosition = -1; + if (bytesRead == readBuffer.length) { + int read =currentFile.audioIn.read(readBuffer2); + if (read == bytesRead) { // linear crossfade + int sample, sample2; + for (int i = 0; i < read; i += 2) { + if (currentFile.format.isBigEndian()) { + sample = (readBuffer[i+1] & 0xff) | + (readBuffer[i] << 8); + sample2= (readBuffer2[i+1] & 0xff) | + (readBuffer2[i] << 8); + sample = ((read-i) * sample + + i * sample2) / read; + readBuffer[i] = (byte)(sample >> 8); + readBuffer[i+1] = (byte)sample; + } else { + sample = (readBuffer[i] & 0xff) | + (readBuffer[i+1] << 8); + sample2 = (readBuffer2[i] & 0xff) | + (readBuffer2[i+1] << 8); + sample = ((read-i) * sample + + i * sample2) / read; + readBuffer[i+1] = (byte)(sample>>8); + readBuffer[i] = (byte)sample; + } + } + } else { + bytesRead = read; + for (int i = 0; i < read; i++) + readBuffer[i] = readBuffer2[i]; + } + } else + bytesRead =currentFile.audioIn.read(readBuffer); + } + } + bytesWritten = 0; + if (bytesRead > 0) + bytesWritten = audioOut.write(readBuffer, 0,bytesRead); + if (bytesWritten > 0) { + currentPosition += bytesWritten; + updateGUI(); + } + if (bytesWritten < readBufferSize) { + if (currentPosition != currentFile.length) + System.err.println("read error: unexpected EOF"); + stopRequested = true; + break; + } + } + } catch (InterruptedException e) { + playing = false; + e.printStackTrace(); + } catch (LineUnavailableException e) { + playing = false; + e.printStackTrace(); + } catch (java.io.IOException e) { + playing = false; + e.printStackTrace(); + } + } + } // run + + /** Implements ChangeListener interface */ + public void stateChanged(ChangeEvent e) { + int value = gui.playSlider.getValue(); + if ((value == gui.oldSlider) || (currentFile == null)) + return; + if (gui.playSlider.getValueIsAdjusting()) + stopRequested = true; + else { + long newPosn = currentFile.length * value / GUI.maxSlider; + newPosn = newPosn / currentFile.frameSize * currentFile.frameSize; + if (playing) + requestedPosition = newPosn; + else + setPosition(newPosn); + } + } // stateChanged() + +} // class AligningAudioPlayer
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/AudioFile.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,132 @@ +package at.ofai.music.match; + +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.UnsupportedAudioFileException; +import at.ofai.music.util.RandomAccessInputStream; + +public class AudioFile { + + protected String path; + protected GUI.FileNameSelection selector; + private RandomAccessInputStream underlyingStream; + protected AudioInputStream audioIn; + protected AudioFormat format; + protected long length; + protected int frameSize; + protected float frameRate; + protected int[] thisIndex; + protected double thisHopTime; + protected int[] refIndex; + protected double refHopTime; + protected int pathLength; + protected boolean isReference; + protected boolean orientationX; + protected FixedPoint fixedPoints; + + public AudioFile(String pathName, GUI.FileNameSelection jb) { + this(); + path = pathName; + selector = jb; + try { + underlyingStream = new RandomAccessInputStream(pathName); + audioIn = AudioSystem.getAudioInputStream(underlyingStream); + audioIn.mark(0); + underlyingStream.mark(); // after the audio header + format = audioIn.getFormat(); + frameSize = format.getFrameSize(); + frameRate = format.getFrameRate(); + length = audioIn.getFrameLength() * frameSize; + } catch (java.io.IOException e) { // includes FileNotFound + e.printStackTrace(); + } catch (UnsupportedAudioFileException e) { + e.printStackTrace(); + } + } // constructor + + public AudioFile() { // used for worm alignment (no real file!) + thisIndex = refIndex = null; + thisHopTime = refHopTime = 0; + isReference = true; // until aligned + orientationX = true; + fixedPoints = null; + } // default constructor + + public void setMatch(int[] idx1, double ht1, int[] idx2,double ht2,int ln) { + thisIndex = idx1; + thisHopTime = ht1; + refIndex = idx2; + refHopTime = ht2; + pathLength = ln; + isReference = false; // alignment complete: ready for use + } // setMatch() + + public void setFixedPoints(FixedPoint p, boolean isX) { + fixedPoints = p; + orientationX = isX; + } // setFixedPoints() + + public void print() { + for (int i = 0; i < pathLength; i++) { + System.err.print(i + " " + thisIndex[i]+" "+refIndex[i]+" : "); + if (i % 4 == 3) + System.err.println(); + } + System.err.println(); + } // print() + + /** Performs a binary search for a value in an array and returns its index. + * If the value does not exist in the array, the index of the nearest + * element is returned. If the value occurs multiple times in the array, + * the centre index is returned. Note that we can't use + * Arrays.binarySearch() because the array might not be full. + */ + public int search(int[] arr, int val) { + int max = pathLength - 1; + int min = 0; + while (max > min) { + int mid = (max + min) / 2; + if (val > arr[mid]) + min = mid + 1; + else + max = mid; + } // max = MIN_j (arr[j] >= val) i.e. the first equal or next highest + while ((max + 1 < pathLength) && (arr[max + 1] == val)) + max++; + return (min + max) / 2; + } // search() + + public long fromReferenceTime(double time) { + return (long) Math.round(fromReferenceTimeD(time)*frameRate)*frameSize; + } // fromReferenceTime() + + public double fromReferenceTimeD(double time) { + if (!isReference && (pathLength != 0)) { + int refI = (int) Math.round(time / refHopTime); + int index = search(refIndex, refI); + time = thisIndex[index] * thisHopTime; + } + return time; + } // fromReferenceTimeD() + + public double toReferenceTime(long ltime) { + return toReferenceTimeD(ltime / frameSize / frameRate); + } // toReferenceTime() + + public double toReferenceTimeD(double time) { + if (!isReference && (pathLength != 0)) { + int thisI = (int) Math.round(time / thisHopTime); + int index = search(thisIndex, thisI); + time = refIndex[index] * refHopTime; + } + return time; + } // toReferenceTimeD() + + public long setPosition(long position) throws java.io.IOException { + audioIn.reset(); + // must be multiple of frameSize + return underlyingStream.seekFromMark(position / frameSize * frameSize); + } // skip() + +} // class AudioFile
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/Finder.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,218 @@ +package at.ofai.music.match; + +/** Maps cost matrix coordinates into an efficient + * (linear instead of quadratic space) representation. + * Stores result of most recent mapping for fast + * sequential access. + */ +class Finder { + + PerformanceMatcher pm1, pm2; + int index1, index2, bestRow, bestCol; + int[] rowRange; + int[] colRange; + + public Finder(PerformanceMatcher p1, PerformanceMatcher p2) { + if (!p1.firstPM) + System.err.println("Warning: wrong args in Finder()"); + pm1 = p1; + pm2 = p2; + index1 = 0; + index2 = 0; + rowRange = new int[2]; + colRange = new int[2]; + } // constructor + + /** Sets up the instance variables to point to the given coordinate in the + * distance matrix. + * @param i1 frameNumber in the first PerformanceMatcher + * @param i2 frameNumber in the second PerformanceMatcher + * @return true iff the point (i2,i1) is represented in the distance matrix + */ + public boolean find(int i1, int i2) { + if (i1 >= 0) { + index1 = i1; + index2 = i2 - pm1.first[i1]; + } + return (i1 >= 0) && (i2 >= pm1.first[i1]) && (i2 < pm1.last[i1]); + } // find() + + /** Returns the range [lo,hi) of legal column indices for the given row. */ + public void getColRange(int row, int[] range) { + range[0] = pm1.first[row]; + range[1] = pm1.last[row]; + } // getColRange() + + /** Returns the range [lo,hi) of legal row indices for the given column. */ + public void getRowRange(int col, int[] range) { + range[0] = pm2.first[col]; + range[1] = pm2.last[col]; + } // getRowRange() + + public int getExpandDirection(int row, int col) { + return getExpandDirection(row, col, false); + } // getExpandDirection() + + public int getExpandDirection(int row, int col, boolean check) { + int min = getPathCost(row, col); + bestRow = row; + bestCol = col; + getRowRange(col, rowRange); + if (rowRange[1] > row+1) + rowRange[1] = row+1; // don't cheat by looking at future :) + for (int index = rowRange[0]; index < rowRange[1]; index++) { + int tmp = getPathCost(index, col); + if (tmp < min) { + min = tmp; + bestRow = index; + } + } + getColRange(row, colRange); + if (colRange[1] > col+1) + colRange[1] = col+1; // don't cheat by looking at future :) + for (int index = colRange[0]; index < colRange[1]; index++) { + int tmp = getPathCost(row, index); + if (tmp < min) { + min = tmp; + bestCol = index; + bestRow = row; + } + } + // System.err.print(" BEST: " + bestRow + " " + bestCol + " " + check); + // System.err.println(" " + pm1.frameCount + " " + pm2.frameCount); + if (check) { + // System.err.println(find(row+1, col) + " " + find(row, col+1)); + if (!find(row, col+1)) + return PerformanceMatcher.ADVANCE_THIS; + if (!find(row+1, col)) + return PerformanceMatcher.ADVANCE_OTHER; + } + return ((bestRow==row)? PerformanceMatcher.ADVANCE_THIS: 0) | + ((bestCol==col)? PerformanceMatcher.ADVANCE_OTHER: 0); + } // getExpandDirection() + + public byte getDistance(int row, int col) { + if (find(row, col)) + return pm1.distance[row][col - pm1.first[row]]; + throw new IndexOutOfBoundsException("getDistance("+row+","+col+")"); + } // getDistance()/2 + + public void setDistance(int row, int col, byte b) { + if (find(row, col)) + pm1.distance[row][col - pm1.first[row]] = b; + throw new IndexOutOfBoundsException("setDistance("+ + row+","+col+","+b+")"); + } // setDistance() + + public int getPathCost(int row, int col) { + if (find(row, col)) // "1" avoids div by 0 below + return pm1.bestPathCost[row][col - pm1.first[row]]*100/ (1+row+col); + throw new IndexOutOfBoundsException("getPathCost("+row+","+col+")"); + } // getPathCost() + + public int getRawPathCost(int row, int col) { + if (find(row, col)) + return pm1.bestPathCost[row][col - pm1.first[row]]; + throw new IndexOutOfBoundsException("getPathCost("+row+","+col+")"); + } // getRawPathCost() + + public void setPathCost(int row, int col, int i) { + if (find(row, col)) + pm1.bestPathCost[row][col - pm1.first[row]] = i; + throw new IndexOutOfBoundsException("setPathCost("+ + row+","+col+","+i+")"); + } // setPathCost() + + public byte getDistance() { + return pm1.distance[index1][index2]; + } // getDistance()/0 + + public void setDistance(int b) { + pm1.distance[index1][index2] = (byte)b; + } // setDistance() + + public int getPathCost() { + return pm1.bestPathCost[index1][index2]; + } // getPathCost() + + public void setPathCost(int i) { + pm1.bestPathCost[index1][index2] = i; + } // setPathCost() + + /** Calculates a rectangle of the path cost matrix so that the minimum cost + * path between the bottom left and top right corners can be computed. + * Caches previous values to avoid calling find() multiple times, and is + * several times faster as a result. + * @param r1 the bottom of the rectangle to be calculated + * @param c1 the left side of the rectangle to be calculated + * @param r2 the top of the rectangle to be calculated + * @param c2 the right side of the rectangle to be calculated + */ + public void recalculatePathCostMatrix(int r1, int c1, int r2, int c2) { + if (!find(r1,c1)) + throw new IndexOutOfBoundsException(r1+ "," + c1 + " out of range"); +/*/REMOVE + System.err.print("Recalc: " + c1 + "," + r1 + " to " + c2 + "," + r2); + long startTime = System.nanoTime(); + long currentTime; +//--REMOVE */ + int thisRowStart, c; + int prevRowStart = 0, prevRowStop = 0; + for (int r = r1; r <= r2; r++) { + thisRowStart = pm1.first[r]; + if (thisRowStart < c1) + thisRowStart = c1; + for (c = thisRowStart; c <= c2; c++) { + if (find(r,c)) { + int i2 = index2; + int newCost = pm1.distance[r][i2]; + int dir = 0; + if (r > r1) { // not first row + int min = -1; + if ((c > prevRowStart) && (c <= prevRowStop)) { + // diagonal from (r-1,c-1) + min = pm1.bestPathCost[r-1][c-pm1.first[r-1]-1] + + newCost * 2; + dir = PerformanceMatcher.ADVANCE_BOTH; + } + if ((c >= prevRowStart) && (c < prevRowStop)) { + // vertical from (r-1,c) + int cost = pm1.bestPathCost[r-1][c-pm1.first[r-1]] + + newCost; + if ((min == -1) || (cost < min)) { + min = cost; + dir = PerformanceMatcher.ADVANCE_THIS; + } + } + if (c > thisRowStart) { + // horizontal from (r,c-1) + int cost =pm1.bestPathCost[r][i2-1]+newCost; + if ((min == -1) || (cost < min)) { + min = cost; + dir = PerformanceMatcher.ADVANCE_OTHER; + } + } + pm1.bestPathCost[r][i2] = min; + } else if (c > thisRowStart) { // first row + // horizontal from (r,c-1) + pm1.bestPathCost[r][i2] = pm1.bestPathCost[r][i2-1] + + newCost; + dir = PerformanceMatcher.ADVANCE_OTHER; + } + if ((r != r1) || (c != c1)) + pm1.distance[r][i2] = (byte) ((pm1.distance[r][i2] & + PerformanceMatcher.MASK) | dir); + } else + break; // end of row + } + prevRowStart = thisRowStart; + prevRowStop = c; + } +/*/REMOVE + currentTime = System.nanoTime(); + System.err.println(" Time: " + ((currentTime-startTime)/1e6)); + startTime = currentTime; +//--REMOVE */ + } // recalculatePathCostMatrix() + +} // class Finder
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/FixedPoint.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,95 @@ +package at.ofai.music.match; + +/** Maintains a list of fixed points through which all alignment paths must pass. + * This class assumes that the list of points has an unchangeable first and last + * element, corresponding to the beginnings and ends of both files. It is + * implemented as a doubly linked list. + */ +public class FixedPoint { + + protected int x; + protected int y; + protected FixedPoint next, prev; + + private FixedPoint(int x, int y) { + this.x = x; + this.y = y; + next = null; + prev = null; + } // constructor + + private void insert(FixedPoint p) { + p.next = this; + p.prev = prev; + prev.next = p; + prev = p; + } // insert() + + /** Remove the current point from the list containing it. + * It is assumed that this point is not the first or last + * in the list. + */ + public void remove() { // not for the first or last + prev.next = next; + next.prev = prev; + } // remove() + + /** Inserts a new point into the list in sorted (ascending) order + * of both coordinates. The new point will be rejected if it can + * not be inserted into the list such that both the x and y coordinates + * are monotonically non-decreasing. + * @param x The x-coordinate of the new point + * @param y The y-coordinate of the new point + * @return Indicates whether the insertion was successful: + * it will be unsuccessful if the new point would make + * the list non-monotonic + */ + public FixedPoint insert(int x, int y) { + FixedPoint p = new FixedPoint(x,y); + insert(p); + if (p.sort()) + return p; + return null; + } // insert() + + private boolean sort() { + FixedPoint posn = next; + if ((prev.prev != null) && ((x < prev.x) || + ((x == prev.x) && (y < prev.y)))) { + posn = prev; + while ((posn.prev != null) && ((x < posn.prev.x) || + (x == posn.prev.x) && (y < posn.prev.y))) + posn = posn.prev; + } else { + while ((posn.next != null) && ((x > posn.x) || + ((x == posn.x) && (y > posn.y)))) + posn = posn.next; + } + if (posn != next) { + next.prev = prev; + prev.next = next; + posn.insert(this); + } + if ((x < prev.x) || (y < prev.y) || (x > next.x) || (y > next.y)) { + remove(); // paths must be monotonic + return false; + } + return true; + } // sort() + + /** This class assumes a list of points has an unchangeable first and last + * element, corresponding to the beginnings and ends of both files; this + * method creates and initialises such a list with the two end points. + * @param x1 x-coordinate of the start point (usually 0) + * @param y1 y-coordinate of the start point (usually 0) + * @param x2 x-coordinate of the end point (usually length of file x) + * @param y2 y-coordinate of the end point (usually length of file y) + */ + public static FixedPoint newList(int x1, int y1, int x2, int y2) { + FixedPoint p = new FixedPoint(x2, y2); + p.prev = new FixedPoint(x1, y1); + p.prev.next = p; + return p; + } // newList() + +} // class FixedPoint
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/GUI.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,599 @@ +package at.ofai.music.match; + +import java.awt.Color; +import java.awt.Font; +import java.awt.Graphics; +import java.awt.GraphicsConfiguration; +import java.awt.Image; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyEvent; +import java.awt.event.KeyListener; +import java.awt.event.MouseEvent; +import java.awt.event.MouseListener; +import java.awt.event.MouseMotionListener; +import java.util.ListIterator; + +import javax.swing.ImageIcon; +import javax.swing.JButton; +import javax.swing.JComponent; +import javax.swing.JFileChooser; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JSlider; +import javax.swing.WindowConstants; + +import at.ofai.music.util.PSPrinter; + +public class GUI extends JFrame implements ActionListener, + MouseListener, + MouseMotionListener, + KeyListener { + + public static final String version = "0.9.2"; + public static final String title = "MATCH " + version; + protected static final int xSize = 230; + protected static final int ySize = 150; + protected static final int fileNameHeight = 20; + protected static final int buttonWd = 25; + protected static final int buttonHt = 15; + protected static final int maxSlider = 1000; + public static boolean DEBUG = true; + protected static String loadFile = null; + protected static final String READY = "Status: Ready"; + protected static final String LOADING = "Status: Loading"; + protected static final String ALIGNING = "Status: Aligning"; + public static final Color BACKGROUND = Color.black; + public static final Color BACKGROUND2 = Color.gray; // unmatched files + public static final Color FOREGROUND = Color.green; // text + public static final Color HIGHLIGHT = Color.red; // buttons, status + static final long serialVersionUID = 0; + + protected PerformanceMatcher pm1, pm2; + protected AligningAudioPlayer audioPlayer; + protected JFileChooser fileChooser; + protected MarkDisplay markDisplay; + protected Help help; + protected ScrollingMatrix scrollingMatrix; + protected JSlider playSlider; + protected TimePanel timePanel; + protected JLabel readyLabel; + protected JLabel modeLabel; + protected int oldTime; + protected double oldTimeDouble; + protected int oldSlider; + protected int originX, originY; + + protected GUI(PerformanceMatcher p1, PerformanceMatcher p2, + ScrollingMatrix sm, boolean makeVisible) { + super(title); + fileChooser = new JFileChooser(); // NOTE: needed by AAP constructor + audioPlayer = new AligningAudioPlayer(this, p1, p2, sm); + pm1 = p1; + pm2 = p2; + scrollingMatrix = sm; + oldTime = 0; + oldTimeDouble = -1; + oldSlider = 0; + originX = 0; + originY = 0; + setUndecorated(true); + setLayout(null); + setSize(xSize,ySize); // in case no files are given + getContentPane().setBackground(BACKGROUND); + addLabels(); + addTimePanel(); + addSlider(); + addMarkDisplay(); + addButtons(); + setLocation(10,10); + addMouseMotionListener(this); + addMouseListener(this); + addKeyListener(this); + setResizable(false); + setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE); + if (makeVisible) { + setVisible(true); + requestFocusInWindow(); + } + if (loadFile != null) + audioPlayer.load(loadFile); + } // constructor + + protected void addFiles(String[] files, int index) { + for ( ; index < files.length; index++) + addFile(files[index]); + } // addFiles() + + protected void addFile(String pathName) { + addFile(pathName, true); + } // addFile()/1 + + protected FileNameSelection addFile(String pathName, boolean addToPlayer) { + String sep = System.getProperty("file.separator"); + int start = pathName.lastIndexOf(sep); + int end = pathName.lastIndexOf("."); + String baseName; + if (end <= start) + baseName = pathName.substring(start+1); + else + baseName = pathName.substring(start+1, end); + FileNameSelection b = new FileNameSelection(baseName, this); + b.setBounds(10, ySize + audioPlayer.getFileCount() * fileNameHeight, + xSize - 20, fileNameHeight); + if (addToPlayer) { + audioPlayer.addFile(pathName, b); + setSize(xSize, ySize+audioPlayer.getFileCount()*fileNameHeight+10); + } else + setSize(xSize, ySize + (audioPlayer.getFileCount()+1) * + fileNameHeight + 10); + add(b); + validate(); // necessary in Windows, but not Linux + return b; + } // addFile() + + protected void removeFile(FileNameSelection b) { // only remove the last! + remove(b); + setSize(xSize, ySize + audioPlayer.getFileCount() * fileNameHeight +10); + validate(); // necessary in Windows, but not Linux + } // removeFile() + + protected void addSlider() { + playSlider = new JSlider(JSlider.HORIZONTAL, 0, maxSlider, oldSlider); + playSlider.setBounds(10, ySize - buttonHt - 60, xSize - 20, 30); + playSlider.setBackground(BACKGROUND); + playSlider.addChangeListener(audioPlayer); + playSlider.addKeyListener(this); + add(playSlider); + } // addSlider() + + class MarkDisplay extends JComponent { + + static final long serialVersionUID = 0; + + public void paint(Graphics g) { + g.setColor(FOREGROUND); + g.fillRect(0, 0, getWidth(), getHeight()); + long length = audioPlayer.getCurrentFileLength(); + if (length == 0) + return; + ListIterator<Long> i = audioPlayer.getMarkListIterator(); + g.setColor(HIGHLIGHT); + while (i.hasNext()) { + int x = (int) (i.next().longValue() * (getWidth()-1) / length); + g.drawLine(x, 0, x, getHeight()-1); + } + } // paint() + + } // inner class MarkDisplay + + protected void addMarkDisplay() { + final int thumbSize = 14;// est. width in pixels of the slider's pointer + markDisplay = new MarkDisplay(); + markDisplay.setBounds(10 + thumbSize / 2, ySize - buttonHt - 30, + xSize - 20 - thumbSize, 10); + add(markDisplay); + } // addMarkDisplay() + + protected void addLabels() { + addLabel(title, 10, 5, xSize - 30 - buttonWd, 25, FOREGROUND); + readyLabel = addLabel("", 20, 30, 110, 20, HIGHLIGHT); + setStatus(READY); + modeLabel = addLabel("", 20, 50, 110, 20, HIGHLIGHT); + setMode(false); + } // addLabels() + + protected JLabel addLabel(String text, int x,int y,int wd,int ht, Color c) { + JLabel label = new JLabel(text); + label.setForeground(c); + label.setBounds(x, y, wd, ht); + add(label); + return label; + } // addLabel() + + class TimePanel extends JComponent { // simulates LCD timer display + + byte[] digits = {119,36,93,109,46,107,123,37,127,111}; + static final long serialVersionUID = 0; + + public void paint(Graphics g) { + g.setColor(BACKGROUND); + g.fillRect(0, 0, getWidth(), getHeight()); + g.setColor(FOREGROUND); + paintDigit(g, oldTime/600%6, 5); + paintDigit(g, oldTime/60%10, 20); + paintDigit(g, oldTime/10%6, 45); + paintDigit(g, oldTime%10, 60); + g.fillRoundRect(38, 15, 4, 4, 3, 3); + g.fillRoundRect(38, 25, 4, 4, 3, 3); + } // paint() + + public void paintDigit(Graphics g, int d, int x) { + if ((digits[d] & 1) != 0) // top + g.fillRoundRect(x+1, 5, 10, 3, 3, 3); + if ((digits[d] & 2) != 0) // upper left + g.fillRoundRect(x, 6, 3, 10, 3, 3); + if ((digits[d] & 4) != 0) // upper right + g.fillRoundRect(x+10, 6, 3, 10, 3, 3); + if ((digits[d] & 8) != 0) // centre + g.fillRoundRect(x+1, 15, 10, 3, 3, 3); + if ((digits[d] & 16) != 0) // lower left + g.fillRoundRect(x, 16, 3, 10, 3, 3); + if ((digits[d] & 32) != 0) // lower right + g.fillRoundRect(x+10, 16, 3, 10, 3, 3); + if ((digits[d] & 64) != 0) // bottom + g.fillRoundRect(x+1, 25, 10, 3, 3, 3); + } + } // inner class TimePanel + + protected void addTimePanel() { + timePanel = new TimePanel(); + timePanel.setBounds(140, 35, 80, 35); + add(timePanel); + } // addTimePanel() + + protected void addButtons() { + GraphicsConfiguration gc = getGraphicsConfiguration(); + for (int i = 0; i < 9; i++) { + Image image = gc.createCompatibleImage(buttonWd, buttonHt); + Graphics g = image.getGraphics(); + g.setColor(BACKGROUND); + g.fillRect(0,0,buttonWd, buttonHt); + g.setColor(HIGHLIGHT); + g.drawRect(0,0,buttonWd-1, buttonHt-1); + g.setColor(HIGHLIGHT); + int x = 10 + i * (buttonWd + 2); + int y = ySize - buttonHt - 10; + int[] x1, y1; + String text = null; + switch (i) { + case 0: + text = "play"; + x1 = new int[]{8,16,8}; + y1 = new int[]{3,7,11}; + g.fillPolygon(x1,y1,3); + break; + case 1: + text = "pause"; + g.fillRect(9,4,2,7); + g.fillRect(14,4,2,7); + break; + case 2: + text = "stop"; + g.fillRect(9,4,7,7); + break; + case 3: + text = "previous"; + x1 = new int[]{18,12,18}; + y1 = new int[]{3,7,11}; + g.fillPolygon(x1,y1,3); + x1 = new int[]{12,6,12}; + g.fillPolygon(x1,y1,3); + break; + case 4: + text = "mark"; + g.drawLine(9,4,15,10); + g.drawLine(8,7,16,7); + g.drawLine(9,10,15,4); + g.drawLine(12,3,12,11); + break; + case 5: + text = "next"; + x1 = new int[]{6,12,6}; + y1 = new int[]{11,7,3}; + g.fillPolygon(x1,y1,3); + x1 = new int[]{12,18,12}; + g.fillPolygon(x1,y1,3); + break; + case 6: + x = xSize - buttonWd - 10; + text = "load"; + g.fillRect(12,3,2,8); + g.fillRect(9,6,8,2); + break; + case 7: + x = xSize - 2 * buttonWd - 12; + y = 10; + text = "help"; + Font f = g.getFont(); + g.setFont(f.deriveFont(Font.BOLD, 10.0f)); + g.drawString("?", 10, 12); + break; + case 8: + x = xSize - buttonWd - 10; + y = 10; + text = "exit"; + g.drawLine(8,3,15,11); + g.drawLine(9,3,16,11); + g.drawLine(8,11,15,3); + g.drawLine(9,11,16,3); + break; + } + JButton button = new JButton(new ImageIcon(image)); + button.setActionCommand(text); + button.setToolTipText(text); + button.setBorder(null); + button.setBounds(x,y,buttonWd,buttonHt); + button.addActionListener(this); + button.addKeyListener(this); + add(button); + } + } // addButtons + + class FileNameSelection extends JButton { + + String name; + boolean selected; + double fractionMatched; + static final long serialVersionUID = 0; + + public FileNameSelection(String n, GUI a) { + name = n; + selected = false; + fractionMatched = 0; + setActionCommand(name); + setBorder(null); + addActionListener(a); + addKeyListener(a); + } // constructor + + public void setSelected(boolean b) { + selected = b; + repaint(); + } // setSelected() + + public void setFraction(double d) { + fractionMatched = d; + repaint(); + } // setFraction() + + public void paint(Graphics g) { + Font f = g.getFont(); + g.setFont(f.deriveFont(Font.PLAIN, 12.0f)); + int wd = (int)(fractionMatched * getWidth()); + if (fractionMatched != 1.0) { + g.setColor(BACKGROUND2); + g.fillRect(wd, 0, getWidth(), getHeight()); + } + if (fractionMatched != 0.0) { + g.setColor(BACKGROUND); + g.fillRect(0, 0, wd, getHeight()); + } + if (selected) { + g.setColor(HIGHLIGHT); + g.drawRect(0, 0, getWidth()-1, getHeight()-1); + } + g.setColor(FOREGROUND); + g.drawString(name, 5, 14); + } // paint() + + } // inner class FileNameSelection + + protected void setStatus(String status) { + if (readyLabel == null) // match Thread starts before constructor ends + return; + readyLabel.setText(status); + readyLabel.repaint(); + } // setStatus() + + protected void setMode(boolean fromMark) { + audioPlayer.setMode(fromMark); + if (modeLabel == null) // match Thread starts before constructor ends + return; + if (fromMark) + modeLabel.setText("Mode: Repeat"); + else + modeLabel.setText("Mode: Continue"); + modeLabel.repaint(); + } // setMode() + + public void setTimer(double time, AudioFile currentFile) { // in seconds + int newTime = (int) time; + if (newTime != oldTime) { + oldTime = newTime; + timePanel.repaint(); + } + if ((scrollingMatrix != null) && (currentFile != null) && + (Math.abs(oldTimeDouble-time) > 0.1)) { + oldTimeDouble = time; + scrollingMatrix.setTime(time, currentFile); + } + } // setTimer() + + public void setSlider(double position) { // position in [0,1] + int newSlider = (int) (position * maxSlider); + if (newSlider != oldSlider) { + oldSlider = newSlider; + playSlider.setValue(newSlider); + playSlider.repaint(); + } + } // setSlider() + + public void updateMarks() { + markDisplay.repaint(); + } // updateMarks() + + protected void showHelp() { + if (help == null) + help = new Help(); + help.setVisible(true); + } // showHelp() + + protected void loadFile() { + if (fileChooser.showOpenDialog(null) == JFileChooser.APPROVE_OPTION) + addFile(fileChooser.getSelectedFile().getAbsolutePath()); + } // loadFile() + + protected void saveWormFile() { + if ((scrollingMatrix != null) && (scrollingMatrix.wormHandler != null)){ + if (fileChooser.showSaveDialog(null) == JFileChooser.APPROVE_OPTION) + scrollingMatrix.wormHandler.write( + fileChooser.getSelectedFile(), true); + } + } // saveWormFile() + + // interface ActionListener + public void actionPerformed(ActionEvent e) { + String command = e.getActionCommand(); + // System.err.println("Button Pressed: " + command + + // " Modifiers: " + e.getModifiers()); + if (e.getModifiers() == 0) + return; // ignore key-generated actions + if (command.equals("exit")) + System.exit(0); + else if (command.equals("help")) + showHelp(); + else if (command.equals("play")) + audioPlayer.play(); + else if (command.equals("pause")) + audioPlayer.pause(); + else if (command.equals("stop")) + audioPlayer.stop(); + else if (command.equals("previous")) + audioPlayer.skipToPreviousMark(); + else if (command.equals("mark")) + audioPlayer.addMark(); + else if (command.equals("next")) + audioPlayer.skipToNextMark(); + else if (command.equals("load")) + loadFile(); + else if (e.getSource() instanceof FileNameSelection) + audioPlayer.setCurrentFile((FileNameSelection)e.getSource()); + else + System.err.println("Unknown ActionEvent: " + e); + } // actionPerformed + + // interface MouseMotionListener + public void mouseMoved(MouseEvent e) { + // requestFocusInWindow(); // for KeyEvents + } // mouseMoved() + + public void mouseDragged(MouseEvent e) { // move GUI + setLocation(getX() + e.getX() - originX, getY() + e.getY() - originY); + } // mouseDragged() + + // interface MouseListener + public void mouseEntered(MouseEvent e) { + requestFocusInWindow(); // for KeyEvents + } // mouseEntered() + + public void mouseExited(MouseEvent e) {} + + public void mouseClicked(MouseEvent e) {} + + public void mouseReleased(MouseEvent e) { + originX = 0; + originY = 0; + } // mouseReleased() + + public void mousePressed(MouseEvent e) { + originX = e.getX(); + originY = e.getY(); + } // mouseClicked() + + // interface KeyListener + public void keyPressed(KeyEvent e) { + // System.err.println("keyPressed(): " + e); + switch(e.getKeyCode()) { + case KeyEvent.VK_Z: + audioPlayer.play(); + break; + case KeyEvent.VK_X: + audioPlayer.pause(); + break; + case KeyEvent.VK_C: + audioPlayer.stop(); + break; + case KeyEvent.VK_V: + case KeyEvent.VK_LEFT: + case KeyEvent.VK_KP_LEFT: + audioPlayer.skipToPreviousMark(); + break; + case KeyEvent.VK_B: + audioPlayer.addMark(); + break; + case KeyEvent.VK_N: + case KeyEvent.VK_RIGHT: + case KeyEvent.VK_KP_RIGHT: + audioPlayer.skipToNextMark(); + break; + case KeyEvent.VK_M: + loadFile(); + break; + case KeyEvent.VK_DOWN: + case KeyEvent.VK_KP_DOWN: + audioPlayer.nextFile(); + break; + case KeyEvent.VK_UP: + case KeyEvent.VK_KP_UP: + audioPlayer.previousFile(); + break; + case KeyEvent.VK_SPACE: + audioPlayer.togglePlay(); + break; + case KeyEvent.VK_COMMA: + setMode(false); + break; + case KeyEvent.VK_PERIOD: + setMode(true); + break; + case KeyEvent.VK_O: + audioPlayer.clearFiles(); + break; + case KeyEvent.VK_H: + case KeyEvent.VK_SLASH: + showHelp(); + break; + case KeyEvent.VK_Q: + case KeyEvent.VK_ESCAPE: + System.exit(0); + break; + case KeyEvent.VK_1: + audioPlayer.setCurrentFile(0); + break; + case KeyEvent.VK_2: + audioPlayer.setCurrentFile(1); + break; + case KeyEvent.VK_3: + audioPlayer.setCurrentFile(2); + break; + case KeyEvent.VK_4: + audioPlayer.setCurrentFile(3); + break; + case KeyEvent.VK_5: + audioPlayer.setCurrentFile(4); + break; + case KeyEvent.VK_6: + audioPlayer.setCurrentFile(5); + break; + case KeyEvent.VK_7: + audioPlayer.setCurrentFile(6); + break; + case KeyEvent.VK_8: + audioPlayer.setCurrentFile(7); + break; + case KeyEvent.VK_9: + audioPlayer.setCurrentFile(8); + break; + case KeyEvent.VK_0: + audioPlayer.setCurrentFile(9); + break; + case KeyEvent.VK_P: // print + PSPrinter.print(getContentPane()); + break; + case KeyEvent.VK_S: // save + audioPlayer.save(); + break; + case KeyEvent.VK_R: // restore + audioPlayer.load(); + break; + case KeyEvent.VK_W: // save wormfile + saveWormFile(); + break; + } + } // keyPressed() + + public void keyTyped(KeyEvent e) {} // ignore + public void keyReleased(KeyEvent e) {} // ignore + +} // class GUI
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/Help.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,175 @@ +package at.ofai.music.match; + +import java.awt.Font; +import java.awt.Graphics; +import java.awt.GraphicsConfiguration; +import java.awt.Image; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyEvent; +import java.awt.event.KeyListener; +import java.awt.event.MouseEvent; +import java.awt.event.MouseListener; +import java.awt.event.MouseMotionListener; + +import javax.swing.ImageIcon; +import javax.swing.JButton; +import javax.swing.JComponent; +import javax.swing.JFrame; +import javax.swing.JLabel; + +public class Help extends JFrame implements ActionListener, + KeyListener, + MouseListener, + MouseMotionListener { + + static final String[] helpText = { + "The following shortcut keys are defined:", + "'z' - play", + "'x' - pause", + "'c' - stop", + "'v','LEFT' - go to previous mark", + "'b' - add/remove mark at current position", + "'n','RIGHT' - go to next mark", + "'m' - load new audio file", + "'o' - clear all audio files", + "'s' - save session", + "'r' - restore session", + "'w' - write worm file format", + "'p' - print screenshot", + "'SPACE' - toggle play/pause", + "',' - continue mode: plays from current position", + "'.' - repeat mode: plays from previous mark ", + "'UP' - go to previous file", + "'DOWN' - go to next file", + "'1'...'9' - go to file number n", + "'0' - go to file number 10", + "'h','/' - show this help screen", + "'q','ESCAPE' - exit"}; + static final int xSize = 300; + static final int ySize = 18 * (helpText.length + 3); + static final int buttonWd = 25; + static final int buttonHt = 15; + private int originX, originY; + static final long serialVersionUID = 0; + + public static void main(String[] args) { new Help(); } // main() + + protected Help() { + super(GUI.title + " - Help"); + originX = 0; + originY = 0; + setUndecorated(true); + setLayout(null); + setSize(xSize,ySize); + getContentPane().setBackground(GUI.BACKGROUND); + addButton(); + JLabel l = new JLabel(GUI.title + " - Help"); + l.setBackground(GUI.BACKGROUND); + l.setForeground(GUI.FOREGROUND); + l.setBounds(10, 10, xSize - buttonWd - 30, buttonHt + 10); + add(l); + HelpText t = new HelpText(); + t.setBounds(0, buttonHt + 15, xSize, ySize - buttonHt - 15); + add(t); + setLocation(10,10); + addKeyListener(this); + addMouseListener(this); + addMouseMotionListener(this); + setResizable(false); + setVisible(true); + requestFocusInWindow(); + } // constructor + + protected void addButton() { + GraphicsConfiguration gc = getGraphicsConfiguration(); + Image image = gc.createCompatibleImage(buttonWd, buttonHt); + Graphics g = image.getGraphics(); + g.setColor(GUI.BACKGROUND); + g.fillRect(0,0,buttonWd, buttonHt); + g.setColor(GUI.HIGHLIGHT); + g.drawRect(0,0,buttonWd-1, buttonHt-1); + g.setColor(GUI.HIGHLIGHT); + g.drawLine(8,3,15,11); + g.drawLine(9,3,16,11); + g.drawLine(8,11,15,3); + g.drawLine(9,11,16,3); + JButton button = new JButton(new ImageIcon(image)); + String text = "close"; + button.setActionCommand(text); + button.setToolTipText(text); + button.setBorder(null); + button.setBounds(xSize - buttonWd - 10, 10, buttonWd, buttonHt); + button.addActionListener(this); + add(button); + } // addButtons + + class HelpText extends JComponent { + + static final long serialVersionUID = 0; + public void paint(Graphics g) { + Font f = g.getFont(); + g.setFont(f.deriveFont(Font.PLAIN, 12.0f)); + g.setColor(GUI.BACKGROUND); + g.fillRect(0, 0, getWidth(), getHeight()); + g.setColor(GUI.FOREGROUND); + for (int i = 0; i < helpText.length; i++) + g.drawString(helpText[i], 10, 18 * (i + 1)); + } // paint() + + } // inner class HelpText + + // interface ActionListener + public void actionPerformed(ActionEvent e) { + setVisible(false); + } // actionPerformed + + // interface KeyListener + public void keyPressed(KeyEvent e) { + switch(e.getKeyCode()) { + case KeyEvent.VK_X: + case KeyEvent.VK_Q: + case KeyEvent.VK_ESCAPE: + setVisible(false); + break; + // case KeyEvent.VK_DOWN: + // case KeyEvent.VK_KP_DOWN: + // break; + // case KeyEvent.VK_UP: + // case KeyEvent.VK_KP_UP: + // break; + } + } // keyPressed() + + public void keyTyped(KeyEvent e) {} // ignore + public void keyReleased(KeyEvent e) {} // ignore + + // interface MouseListener + public void mouseEntered(MouseEvent e) { + requestFocusInWindow(); // for KeyEvents + } // mouseEntered() + + public void mouseExited(MouseEvent e) {} + + public void mouseClicked(MouseEvent e) {} + + public void mouseReleased(MouseEvent e) { + originX = 0; + originY = 0; + } // mouseReleased() + + public void mousePressed(MouseEvent e) { + originX = e.getX(); + originY = e.getY(); + } // mouseClicked() + + // interface MouseMotionListener + public void mouseMoved(MouseEvent e) { + // requestFocusInWindow(); // for KeyEvents + } // mouseMoved() + + public void mouseDragged(MouseEvent e) { // move GUI + setLocation(getX() + e.getX() - originX, getY() + e.getY() - originY); + } // mouseDragged() + +} // class Help
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/MatrixFrame.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,441 @@ +package at.ofai.music.match; + +import java.awt.Color; +import java.awt.FontMetrics; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.event.KeyEvent; +import java.awt.event.KeyListener; +import java.awt.event.MouseEvent; +import java.awt.event.MouseListener; +import java.awt.event.MouseMotionListener; +import java.util.ListIterator; + +import javax.swing.JFrame; +import javax.swing.JPanel; + +import at.ofai.music.util.Event; +import at.ofai.music.util.WormEvent; +import at.ofai.music.util.PSPrinter; + +class MatrixFrame extends JFrame implements MouseListener, + MouseMotionListener, + KeyListener { + + ScrollingMatrix sm; + MatrixPanel p; + ListIterator<Event> eventIterator1, eventIterator2; + AudioFile map; // only for mapping time with the path + int wd, ht; // size of whole Frame + int originX, originY; // for dragging window + final int defaultMargin = 80; + final int matchFileMargin = 90; + int top; // size of margin + int bottom; // size of margin + int left; // size of margin + int right; // size of margin + public final Color BACKGROUND = Color.black; + public final Color FOREGROUND = Color.green; + public final int[] flagsToLength = { 0, 5,10,10,15,15,15,15, + 20,20,20,20,20,20,20,20}; + static final double ln2 = Math.log(2.0); + static final long serialVersionUID = 0; + + public MatrixFrame(ScrollingMatrix s, boolean makeVisible) { + super(GUI.title); + sm = s; + bottom = defaultMargin; + left = defaultMargin; + if (sm.pm2.hasMatchFile) + top = matchFileMargin; + else + top = defaultMargin; + if (sm.pm1.hasMatchFile) + right = matchFileMargin; + else + right = defaultMargin; + wd = sm.xSize + left + right; + ht = sm.ySize + top + bottom; + originX = 0; + originY = 0; + sm.setBounds(left, top, wd - left - right, ht - top - bottom); + sm.parent = this; + if (sm.pm1.events != null) { + eventIterator1 = sm.pm1.events.listIterator(); + if (sm.pm1.hasWormFile) + generateLabels(eventIterator1); + if (sm.pm1.hasMatchFile) + correctTiming(eventIterator1, sm.pm1.matchFileOffset); + } else + eventIterator1 = null; + if (sm.pm2.events != null) { + eventIterator2 = sm.pm2.events.listIterator(); + if (sm.pm2.hasWormFile) + generateLabels(eventIterator2); + if (sm.pm2.hasMatchFile) + correctTiming(eventIterator2, sm.pm2.matchFileOffset); + } else + eventIterator2 = null; + map = new AudioFile(); + p = new MatrixPanel(); + p.add(sm); + p.setLayout(null); + p.setBounds(0, 0, wd, ht); + add(p); + getContentPane().setBackground(Color.white); + setUndecorated(true); + setLayout(null); + setSize(wd, ht); + setLocation(250,10); + setResizable(false); + addMouseListener(this); + addMouseMotionListener(this); + addKeyListener(this); + if (makeVisible) { + setVisible(true); + requestFocusInWindow(); + } + } // constructor + + protected void generateLabels(ListIterator<Event> it) { + int bar = 0; + while (it.hasNext()) { + WormEvent w = (WormEvent) it.next(); + if ((w.flags & 4) != 0) { + bar++; + if ((w.flags & 8) != 0) + w.label = "*" + bar; + else + w.label = "" + bar; + if ((w.flags & 16) != 0) + w.label += "*"; + } + } + } // generateLabels() + + protected void correctTiming(ListIterator<Event> it, double offset) { + if (it.hasNext()) { + offset -= it.next().keyDown; + it.previous(); + } + while (it.hasNext()) { + Event e = it.next(); + e.keyDown += offset; + e.keyUp += offset; + e.pedalUp += offset; + } + } // correctTiming() + + + class MatrixPanel extends JPanel { + + FontMetrics fm; + static final long serialVersionUID = 0; + + protected MatrixPanel() { + fm = null; + } // constructor + + /** Converts time in seconds to time in minutes and seconds: [-]mm:ss */ + protected String timeString(int t) { + String s = ""; + if (t < 0) { + s = "-"; + t = -t; + } + s += (t / 60) + ":"; + if (t % 60 < 10) + s += "0"; + s += t % 60; + return s; + } // timeString() + + protected void addHTick(Graphics g, String s, int x) { + g.drawLine(x, ht - bottom, x, ht - bottom + 8); + x -= fm.stringWidth(s) / 2; + int y = ht - bottom + fm.getHeight() + 8; + g.drawString(s, x, y); + } // addHTick() + + protected void addHMarker(Graphics g, String s, int x, int l) { + g.drawLine(x, top - l, x, top); + if (s != null) { + x -= fm.stringWidth(s) / 2; + int y = top - l - 4; + g.drawString(s, x, y); + } + } // addHMarker() + + /** Displays bar and beat markers from a WormFile. */ + protected void addHWorm(Graphics g) { + while (eventIterator2.hasPrevious()) { + Event e = eventIterator2.previous(); + if (e.keyDown / sm.hop2 + sm.x0 < 0) + break; + } + while (eventIterator2.hasNext()) { + WormEvent e = (WormEvent) eventIterator2.next(); + int x = (int) Math.round(e.keyDown / sm.hop2) + sm.x0; + if (x < 0) + continue; + else if (x > wd - left - right) + break; + addHMarker(g, e.label, x + left, flagsToLength[e.flags&15]); + } + } // addHWorm() + + /** Displays tempo curve based on the OTHER file's WormFile. */ + protected void addHTempoCurve(Graphics g) { + map.setMatch(sm.sPathY, sm.hop1, sm.sPathX, sm.hop2, + sm.sPathLength); // x is reference + double t1 = 0; + while (eventIterator1.hasPrevious()) { + Event e = eventIterator1.previous(); + t1 = map.toReferenceTimeD(e.keyDown); + if (t1 / sm.hop1 + sm.x0 < 0) + break; + } + int x1 = (int) Math.round(t1 / sm.hop1) + sm.x0; + int y1 = -1; + double stop = 0; + if (sm.sPathLength > 0) + stop = sm.sPathY[sm.sPathLength - 1] * sm.hop1; + while (eventIterator1.hasNext()) { + WormEvent e = (WormEvent) eventIterator1.next(); + double t2 = map.toReferenceTimeD(e.keyDown); + int x2 = (int) Math.round(t2 / sm.hop1) + sm.x0; + int y2 = y1; + if (t2 > t1) { + y2 = 30 + (int) Math.round(10 * Math.log(t2 - t1) / ln2); + if (y2 < 0) + y2 = 0; + else if (y2 > 50) + y2 = 50; + } + if (x2 < 0) + continue; + else if (e.keyDown > stop) + break; + addHMarker(g, e.label, x2 + left, flagsToLength[e.flags&15]); + if (y1 >= 0) + g.drawLine(x1 + left, y1, x2 + left, y2); + t1 = t2; + x1 = x2; + y1 = y2; + } + } // addHTempoCurve() + + /** Displays piano roll notation from a match file. */ + protected void addHMatch(Graphics g) { + while (eventIterator2.hasPrevious()) { + Event e = eventIterator2.previous(); + if (e.pedalUp / sm.hop2 + sm.x0 < left + right - wd) + break; + } + while (eventIterator2.hasNext()) { + Event e = (Event) eventIterator2.next(); + int x1 = (int) Math.round(e.keyDown / sm.hop2) + sm.x0; + int x2 = (int) Math.round(e.pedalUp / sm.hop2) + sm.x0; + if (x2 < 0) + continue; + else if (x1 > wd - left - right) + break; + if (x1 < 0) + x1 = 0; + if (x2 > wd - left - right) + x2 = wd - left - right; + int y = top + 20 - e.midiPitch; + g.drawLine(x1 + left, y, x2 + left, y); + } + } // addHMatch() + + // add tick marks and labels on the vertical axis without rotation + protected void addVTickH(Graphics g, String s, int y) { + g.drawLine(left - 8, y, left, y); + if (s != null) { + int x = left - 10 - fm.stringWidth(s); + y += fm.getHeight() / 2; + g.drawString(s, x, y); + } + } // addVTickH() + + protected void addVTick(Graphics g, String s, int y) { + g.drawLine(-y, left - 8, -y, left); + int x = -y - fm.stringWidth(s) / 2; + y = left - 12; + g.drawString(s, x, y); + } // addVTick() + + // add markers on the right hand border without rotation + protected void addVMarkerH(Graphics g, String s, int y, int l) { + g.drawLine(wd - right, y, wd - right + l, y); + if (s != null) { + int x = wd - right + l + 4; + y += fm.getHeight() / 2; + g.drawString(s, x, y); + } + } // addVMarkerH() + + protected void addVMarker(Graphics g, String s, int y, int l) { + g.drawLine(-y, wd - right, -y, wd - right + l); + if (s != null) { + int x = -y - fm.stringWidth(s) / 2; + y = wd - right + l + 4 + fm.getHeight(); + g.drawString(s, x, y); + } + } // addVMarker() + + protected void addVWorm(Graphics g) { + while (eventIterator1.hasPrevious()) { + Event e = eventIterator1.previous(); + if (sm.y0 - e.keyDown / sm.hop1 > ht - top - bottom) + break; + } + while (eventIterator1.hasNext()) { + WormEvent e = (WormEvent) eventIterator1.next(); + int y = sm.y0 - (int) Math.round(e.keyDown / sm.hop1); + if (y > ht - top - bottom) + continue; + else if (y < 0) + break; + addVMarker(g, e.label, y + top, flagsToLength[e.flags&15]); + } + } // addVWorm() + + protected void addVMatch(Graphics g) { + while (eventIterator1.hasPrevious()) { + Event e = eventIterator1.previous(); + if (sm.y0 - e.pedalUp / sm.hop1 > 2 * (ht - top - bottom)) + break; + } + while (eventIterator1.hasNext()) { + Event e = eventIterator1.next(); + int y1 = sm.y0 - (int) Math.round(e.keyDown / sm.hop1); + int y2 = sm.y0 - (int) Math.round(e.pedalUp / sm.hop1); + if (y2 > ht - top - bottom) + continue; + else if (y1 < 0) + break; + if (y1 > ht - top - bottom) + y1 = ht - top - bottom; + if (y2 < 0) + y2 = 0; + int x = wd + 20 - e.midiPitch; + g.drawLine(-y1-top, x, -y2-top, x); // axes have been rotated + } + } // addVMatch() + + protected void addHLabel(Graphics g, String s) { + int x = (wd - fm.stringWidth(s)) / 2; + int y = ht - bottom + 2 * fm.getHeight() + 12; + g.drawString(s, x, y); + } // addHLabel() + + protected void addVLabel(Graphics2D g, String s) { + int y = left - fm.getHeight() - 16; + int x = -(ht + fm.stringWidth(s)) / 2; + g.drawString(s, x, y); + } // addVLabel() + + public void paintComponent(Graphics g) { + if (fm == null) + fm = g.getFontMetrics(); + g.setColor(BACKGROUND); + g.fillRect(0, 0, wd, top); + g.fillRect(0, ht - bottom, wd, bottom); + g.fillRect(0, top, left, ht - top - bottom); + g.fillRect(wd - right, top, right, ht - top - bottom); + g.setColor(FOREGROUND); + g.drawRect(left-1, top-1, wd-left-right+1, ht-top-bottom+1); + if (sm.pm2.audioFileName == null) + addHLabel(g, "Live Input"); + else + addHLabel(g, sm.pm2.audioFileName); + int tickx = (int) Math.ceil((wd - left - right) * sm.hop2 / 10); + int tx = (int) Math.ceil(-sm.x0 * sm.hop2 / tickx) * tickx; + int x = (int) Math.round(tx / sm.hop2) + sm.x0; + int dx = (int) Math.round(tickx / sm.hop2); + while (x < wd - left - right) { + addHTick(g, timeString(tx), x + left); + tx += tickx; + x += dx; + } + if (eventIterator2 != null) { + if (sm.pm2.hasWormFile) + addHWorm(g); + if (sm.pm2.hasMatchFile) + addHMatch(g); + } else if ((eventIterator1 != null) && sm.pm1.hasWormFile) { + // TODO: implement addHTempoCurve and equiv for V + // adds tempo curve based on worm + addHTempoCurve(g); + } + int ticky = (int) Math.ceil((ht - top - bottom) * sm.hop1 / 10); + int ty = (int) Math.floor(sm.y0 * sm.hop1 / ticky) * ticky; + int y = (int) Math.round(sm.y0 - ty / sm.hop1); + int dy = (int) Math.round(ticky / sm.hop1); + Graphics2D g2 = (Graphics2D)g; + g2.rotate(-Math.PI / 2); + if (sm.pm1.audioFileName == null) + addVLabel(g2, "Live Input"); + else + addVLabel(g2, sm.pm1.audioFileName); + while (y < ht - top - bottom) { + addVTick(g2, timeString(ty), y + top); + ty -= ticky; + y += dy; + } + if (eventIterator1 != null) { + if (sm.pm1.hasWormFile) + addVWorm(g2); + if (sm.pm1.hasMatchFile) + addVMatch(g2); + } + g2.rotate(Math.PI / 2); + } // paintComponent() + + } // class MatrixPanel + + // interface MouseMotionListener + public void mouseDragged(MouseEvent e) { // move GUI + setLocation(getX() + e.getX() - originX, getY() + e.getY() - originY); + } // mouseDragged() + + public void mouseMoved(MouseEvent e) {} // mouseMoved() + + // interface MouseListener + public void mouseEntered(MouseEvent e) { + requestFocusInWindow(); // for KeyEvents + } // mouseEntered() + + public void mouseExited(MouseEvent e) {} + + public void mouseClicked(MouseEvent e) {} + + public void mouseReleased(MouseEvent e) { + originX = 0; + originY = 0; + } // mouseReleased() + + public void mousePressed(MouseEvent e) { + originX = e.getX(); + originY = e.getY(); + } // mouseClicked() + + // interface KeyListener + public void keyPressed(KeyEvent e) { + switch(e.getKeyCode()) { + case KeyEvent.VK_P: + PSPrinter.print(this); + break; + case KeyEvent.VK_Q: + case KeyEvent.VK_ESCAPE: + setVisible(false); + break; + } + } // keyPressed() + + public void keyTyped(KeyEvent e) {} // ignore + public void keyReleased(KeyEvent e) {} // ignore + +} // class MatrixFrame
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/Path.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,75 @@ +package at.ofai.music.match; + +class Path { + + public static final int MAX_RUN_LENGTH = 50; + protected static int[] val = new int[10000]; + protected static int[] len = new int[10000]; + + /** Smooths an alignment path.<BR> + * Consider the path as a sequence of horizontal (H), vertical (V) and + * diagonal (D) steps. The smoothing consists of 2 rewrite rules:<BR> + * HnDmVn / Dm+n (where m is less than MAX_RUN_LENGTH)<BR> + * VnDmHn / Dm+n (where m is less than MAX_RUN_LENGTH)<BR> + * The new path is written over the old path. Note that the end points of + * each application of a rewrite rule do not change. + * @return the length of the new path + */ + public static int smooth(int[] x, int[] y, int length) { + if (length == 0) + return 0; + if (val.length < length) { + val = new int[length]; + len = new int[length]; + } + int p = 0; + val[0] = len[0] = 0; + for (int i = 1; i < length; i++) { // H = 1; V = 2; D = 3 + int current = x[i] - x[i-1] + 2 * (y[i] - y[i-1]); + if (current == val[p]) { + len[p]++; + } else if ((current == 3) || (val[p] == 0)) { + val[++p] = current; + len[p] = 1; + } else if (val[p] + current == 3) { // 1 + 2 + if (--len[p] == 0) + p--; + if (val[p] == 3) + len[p]++; + else { + val[++p] = 3; + len[p] = 1; + } + } else { // val[p] == 3 && current != 3 + if ((val[p-1] == current) || + (val[p-1] == 0) || + (len[p] > MAX_RUN_LENGTH)) { + val[++p] = current; + len[p] = 1; + } else { + if (--len[p-1] == 0) { + val[p-1] = val[p]; + len[p-1] = len[p]; + p--; + if (val[p-1] == 3) { + len[p-1] += len[p]; + p--; + } + } + len[p]++; + } + } + } + int i = 1; + for (int pp = 1; pp <= p; pp++) { + int dx = val[pp] & 1; + int dy = val[pp] >> 1; + for (int j = len[pp]; j > 0; j--, i++) { + x[i] = x[i-1] + dx; + y[i] = y[i-1] + dy; + } + } + return i; + } // smooth() + +} // class Path
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/PerformanceMatcher.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,1273 @@ +package at.ofai.music.match; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.ListIterator; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.SourceDataLine; +import javax.sound.sampled.TargetDataLine; + +import at.ofai.music.audio.FFT; +import at.ofai.music.util.Format; +import at.ofai.music.util.Event; +import at.ofai.music.util.EventList; +import at.ofai.music.util.Profile; + +/** Represents an audio stream that can be matched to another audio stream of + * the same piece of music. The matching algorithm uses dynamic time warping. + * The distance metric is a Euclidean metric on the FFT data with the higher + * frequencies mapped onto a linear scale. + */ +public class PerformanceMatcher { + + /** Points to the other performance with which this one is being compared. + * The data for the distance metric and the dynamic time warping is shared + * between the two matchers. In the original version, only one of the two + * performance matchers contained the distance metric. (See + * <code>first</code>) */ + protected PerformanceMatcher otherMatcher; + + /** Indicates which performance is considered primary (the score). This is + * the performance shown on the vertical axis, and referred to as "this" in + * the codes for the direction of DTW steps. */ + protected boolean firstPM; + + /** Input data for this performance (possibly in compressed format) */ + protected AudioInputStream rawInputStream; + + /** Uncompressed version of <code>rawInputStream</code>. + * In the (normal) case where the input is already PCM data, + * <code>rawInputStream == pcmInputStream</code> */ + protected AudioInputStream pcmInputStream; + + /** Line for audio output (only one PerformanceMatcher should set this!) */ + protected SourceDataLine audioOut; + + /** Flag to select audio output from this PerformanceMatcher */ + protected boolean audioOutputRequested; + + /** Format of the audio data in <code>pcmInputStream</code> */ + protected AudioFormat audioFormat; + + /** Number of channels of audio in <code>audioFormat</code> */ + protected int channels; + + /** Sample rate of audio in <code>audioFormat</code> */ + protected float sampleRate; + + /** Source of input data. + * Later to be extended to include live input from the sound card. */ + protected String audioFileName; + + /** The AudioFile object from the AligningAudioPlayer, or null if the + * command line version is being used. + */ + protected AudioFile audioFile; + + /** For assessing the matching algorithm, match files are used, which give + * the times and velocities of all notes (as recorded by the Boesendorfer + * SE290). */ + protected String matchFileName; + protected boolean hasWormFile; + protected boolean hasMatchFile; + protected EventList events; + protected WormHandler wormHandler; + protected boolean liveWorm; + + /** Onset time of the first note in the audio file, in order to establish + * synchronisation between the match file and the audio data. */ + protected double matchFileOffset; + + /** Flag (command line options <b>-n1</b> and <b>-N1</b>) indicating whether + * or not each frame of audio should be normalised to have a sum of 1. + * (Default = false). */ + protected boolean normalise1; + + /** Flag (command line options <b>-n2</b> and <b>-N2</b>) indicating whether + * or not the distance metric for pairs of audio frames should be + * normalised by the sum of the two frames. + * (Default = false). */ + protected boolean normalise2; + + /** Flag (command line options <b>-n3</b> and <b>-N3</b>) indicating whether + * or not each frame of audio should be normalised by the long term average + * of the summed energy. + * (Default = false; assumes normalise1 == false). */ + protected boolean normalise3; + + /** Flag (command line options <b>-n4</b> and <b>-N4</b>) indicating whether + * or not the distance metric for pairs of audio frames should be + * normalised by the log of the sum of the frames. + * (Default = false; assumes normalise2 == false). */ + protected boolean normalise4; + + /** Flag (command line options <b>-d</b> and <b>-D</b>) indicating whether + * or not the half-wave rectified spectral difference should be used in + * calculating the distance metric for pairs of audio frames, instead of + * the straight spectrum values. (Default = true). */ + protected boolean useSpectralDifference; + + protected boolean useChromaFrequencyMap; + + /** Scaling factor for distance metric; must guarantee that the final value + * fits in the data type used, that is, (unsigned) byte. (Default = 16). + */ + protected double scale; + + /** Spacing of audio frames (determines the amount of overlap or skip + * between frames). This value is expressed in seconds and can be set by + * the command line option <b>-h hopTime</b>. (Default = 0.020s) */ + protected double hopTime; + + /** The size of an FFT frame in seconds, as set by the command line option + * <b>-f FFTTime</b>. (Default = 0.04644s). Note that the value is not + * taken to be precise; it is adjusted so that <code>fftSize</code> is + * always power of 2. */ + protected double fftTime; + + /** The width of the search band (error margin) around the current match + * position, measured in seconds. Strictly speaking the width is measured + * backwards from the current point, since the algorithm has to work + * causally. + */ + protected double blockTime; + + /** Spacing of audio frames in samples (see <code>hopTime</code>) */ + protected int hopSize; + + /** The size of an FFT frame in samples (see <code>fftTime</code>) */ + protected int fftSize; // in samples + + /** Width of the search band in FFT frames (see <code>blockTime</code>) */ + protected int blockSize; // in frames + + /** The number of frames of audio data which have been read. */ + protected int frameCount; + + /** RMS amplitude of the current frame. */ + protected double frameRMS; + + /** Long term average frame energy (in frequency domain representation). */ + protected double ltAverage; + + /** The number of frames sequentially processed by this matcher, without a + * frame of the other matcher being processed. + */ + protected int runCount; + + /** Interactive control of the matching process allows pausing computation + * of the cost matrices in one direction. + */ + protected boolean paused; + + /** The total number of frames of audio data to be read. */ + protected int maxFrames; + + /** Audio data is initially read in PCM format into this buffer. */ + protected byte[] inputBuffer; + + /** Audio data is scaled to the range [0,1] and averaged to one channel and + * stored in a circular buffer for reuse (if hopTime < fftTime). */ + protected double[] circBuffer; + + /** The index of the next position to write in the circular buffer. */ + protected int cbIndex; + + /** The window function for the STFT, currently a Hamming window. */ + protected double[] window; + + /** The real part of the data for the in-place FFT computation. + * Since input data is real, this initially contains the input data. */ + protected double[] reBuffer; + + /** The imaginary part of the data for the in-place FFT computation. + * Since input data is real, this initially contains zeros. */ + protected double[] imBuffer; + + /** A mapping function for mapping FFT bins to final frequency bins. + * The mapping is linear (1-1) until the resolution reaches 2 points per + * semitone, then logarithmic with a semitone resolution. e.g. for + * 44.1kHz sampling rate and fftSize of 2048 (46ms), bin spacing is + * 21.5Hz, which is mapped linearly for bins 0-34 (0 to 732Hz), and + * logarithmically for the remaining bins (midi notes 79 to 127, bins 35 to + * 83), where all energy above note 127 is mapped into the final bin. */ + protected int[] freqMap; + + /** The number of entries in <code>freqMap</code>. Note that the length of + * the array is greater, because its size is not known at creation time. */ + protected int freqMapSize; + + /** The most recent frame; used for calculating the frame to frame + * spectral difference. */ + protected double[] prevFrame; + protected double[] newFrame; + + /** A block of previously seen frames are stored in this structure for + * calculation of the distance matrix as the new frames are read in. + * One can think of the structure of the array as a circular buffer of + * vectors. The last element of each vector stores the total energy. */ + protected double[][] frames; + + /** The best path cost matrix. */ + protected int[][] bestPathCost; + + /** The distance matrix. */ + protected byte[][] distance; + + /** The bounds of each row of data in the distance and path cost matrices.*/ + protected int[] first, last; + + /** GUI component which shows progress of alignment. */ + protected GUI.FileNameSelection progressCallback; + + /** Total number of audio frames, or -1 for live or compressed input. */ + protected long fileLength; + + /** Disable or enable debugging output */ + protected static boolean silent = true; + + public static boolean batchMode = false; + public static boolean matrixVisible = false; + public static boolean guiVisible = true; + public static boolean stop = false; + + public static final int liveInputBufferSize = 32768; /* ~195ms buffer @CD */ + public static final int outputBufferSize = 32768; /* ~195ms buffer @CD */ + + /** Encoding of minimum-cost steps performed in DTW algorithm. */ + protected static final int ADVANCE_THIS = 1; + protected static final int ADVANCE_OTHER = 2; + protected static final int ADVANCE_BOTH = ADVANCE_THIS | ADVANCE_OTHER; + protected static final int MASK = 0xFC; + protected static final double decay = 0.99; + protected static final double silenceThreshold = 0.0004; + protected static final int MAX_RUN_COUNT = 3; + protected static final int MAX_LENGTH = 3600; // seconds, i.e. 1 hour + + /** Constructor for PerformanceMatcher. + * @param p The PerformanceMatcher representing the performance with which + * this one is going to be matched. Some information is shared between the + * two matchers (currently one possesses the distance matrix and optimal + * path matrix). + */ + public PerformanceMatcher(PerformanceMatcher p) { + otherMatcher = p; // the first matcher will need this to be set later + firstPM = (p == null); + matchFileOffset = 0; + cbIndex = 0; + frameRMS = 0; + ltAverage = 0; + frameCount = 0; + runCount = 0; + paused = false; + hopSize = 0; + fftSize = 0; + blockSize = 0; + hopTime = 0.020; // DEFAULT, overridden with -h + fftTime = 0.04644; // DEFAULT, overridden with -f + blockTime = 10.0; // DEFAULT, overridden with -c + normalise1 = true; + normalise2 = false; + normalise3 = false; + normalise4 = true; + useSpectralDifference = true; + useChromaFrequencyMap = false; + audioOutputRequested = false; + scale = 90; + maxFrames = 0; // stop at EOF + progressCallback = null; + hasMatchFile = false; + hasWormFile = false; + liveWorm = false; + matchFileName = null; + events = null; + } // default constructor + + /** For debugging, outputs information about the PerformanceMatcher to + * standard error. + */ + public void print() { + System.err.println(this); + } // print() + + /** Gives some basic `header' information about the PerformanceMatcher. */ + public String toString() { + return "PerformanceMatcher\n\tAudio file: " + audioFileName + + " (" + Format.d(sampleRate/1000,1).substring(1) + "kHz, " + + channels + " channels)" + + "\n\tHop size: " + hopSize + + "\n\tFFT size: " + fftSize + + "\n\tBlock size: " + blockSize; + } // toString() + + /** Adds a link to the PerformanceMatcher object representing the + * performance which is going to be matched to this one. + * @param p the PerformanceMatcher representing the other performance + */ + public void setOtherMatcher(PerformanceMatcher p) { + otherMatcher = p; + } // setOtherMatcher() + + /** Adds a link to the GUI component which shows the progress of matching. + * @param c the PerformanceMatcher representing the other performance + */ + public void setProgressCallback(GUI.FileNameSelection c) { + progressCallback = c; + } // setProgressCallback() + + /** Sets the match file for automatic evaluation of the PerformanceMatcher. + * @param fileName The path name of the match file + * @param tStart The offset of the audio recording, that is, the time of + * the first note onset relative to the beginning of the audio file. This + * is required for precise synchronisation of the audio and match files. + * @param isWorm Indicates whether the match file is in Worm file format. + */ + public void setMatchFile(String fileName, double tStart, boolean isWorm) { + matchFileName = fileName; + matchFileOffset = tStart; + hasWormFile = isWorm; + hasMatchFile = !isWorm; + try { + if (isWorm) { + setInputFile(EventList.getAudioFileFromWormFile(matchFileName)); + events = EventList.readWormFile(matchFileName); + if (otherMatcher.wormHandler != null) + otherMatcher.wormHandler.init(); + } else + events = EventList.readMatchFile(matchFileName); + } catch (Exception e) { + System.err.println("Error reading matchFile: " + fileName + "\n"+e); + events = null; + } + } // setMatchFile() + + public void setMatchFile(String fileName, double tStart) { + setMatchFile(fileName, tStart, false); + } // setMatchFile() + + /** Sets up the streams and buffers for live audio input (CD quality). + * If any Exception is thrown within this method, it is caught, and any + * opened streams are closed, and <code>pcmInputStream</code> is set to + * <code>null</code>, indicating that the method did not complete + * successfully. + */ + public void setLiveInput() { + try { + channels = 2; + sampleRate = 44100; + AudioFormat desiredFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, sampleRate, 16, + channels, channels * 2, sampleRate, false); + TargetDataLine tdl = AudioSystem.getTargetDataLine(desiredFormat); + tdl.open(desiredFormat, liveInputBufferSize); + pcmInputStream = new AudioInputStream(tdl); + audioFormat = pcmInputStream.getFormat(); + init(); + tdl.start(); + } catch (Exception e) { + e.printStackTrace(); + closeStreams(); // make sure it exits in a consistent state + } + } // setLiveInput() + + public void setInputFile(AudioFile f) { + audioFile = f; + setInputFile(f.path); + } // setInputFile() + + /** Sets up the streams and buffers for audio file input. + * If any Exception is thrown within this method, it is caught, and any + * opened streams are closed, and <code>pcmInputStream</code> is set to + * <code>null</code>, indicating that the method did not complete + * successfully. + * @param fileName The path name of the input audio file. + */ + public void setInputFile(String fileName) { + closeStreams(); // release previously allocated resources + audioFileName = fileName; + try { + if (audioFileName == null) + throw new Exception("No input file specified"); + File audioFile = new File(audioFileName); + if (!audioFile.isFile()) + throw new FileNotFoundException( + "Requested file does not exist: " + audioFileName); + rawInputStream = AudioSystem.getAudioInputStream(audioFile); + audioFormat = rawInputStream.getFormat(); + channels = audioFormat.getChannels(); + sampleRate = audioFormat.getSampleRate(); + pcmInputStream = rawInputStream; + if ((audioFormat.getEncoding()!=AudioFormat.Encoding.PCM_SIGNED) || + (audioFormat.getFrameSize() != channels * 2) || + audioFormat.isBigEndian()) { + AudioFormat desiredFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, sampleRate, 16, + channels, channels * 2, sampleRate, false); + pcmInputStream = AudioSystem.getAudioInputStream(desiredFormat, + rawInputStream); + audioFormat = desiredFormat; + } + init(); + } catch (Exception e) { + e.printStackTrace(); + closeStreams(); // make sure it exits in a consistent state + } + } // setInputFile() + + protected void init() { + hopSize = (int) Math.round(sampleRate * hopTime); + fftSize = (int) Math.round(Math.pow(2, + Math.round( Math.log(fftTime * sampleRate) / Math.log(2)))); + blockSize = (int) Math.round(blockTime / hopTime); + makeFreqMap(fftSize, sampleRate); + int buffSize = hopSize * channels * 2; + if ((inputBuffer == null) || (inputBuffer.length != buffSize)) + inputBuffer = new byte[buffSize]; + if ((circBuffer == null) || (circBuffer.length != fftSize)) { + circBuffer = new double[fftSize]; + reBuffer = new double[fftSize]; + imBuffer = new double[fftSize]; + window = FFT.makeWindow(FFT.HAMMING, fftSize, fftSize); + for (int i=0; i < fftSize; i++) + window[i] *= Math.sqrt(fftSize); + } + if ((prevFrame == null) || (prevFrame.length != freqMapSize)) { + prevFrame = new double[freqMapSize]; + newFrame = new double[freqMapSize]; + frames = new double[blockSize][freqMapSize+1]; + } else if (frames.length != blockSize) + frames = new double[blockSize][freqMapSize+1]; + int len = (int) (MAX_LENGTH / hopTime); + distance = new byte[len][]; + bestPathCost = new int[len][]; + first = new int[len]; + last = new int[len]; + for (int i = 0; i < blockSize; i++) { + distance[i] = new byte[(MAX_RUN_COUNT+1) * blockSize]; + bestPathCost[i] = new int[(MAX_RUN_COUNT+1) * blockSize]; + } + frameCount = 0; + runCount = 0; + cbIndex = 0; + frameRMS = 0; + ltAverage = 0; + paused = false; + progressCallback = null; + // hasMatchFile = false; // For consistency, it would be good to + // hasWormFile = false; // clear these, but they have to be set + // matchFileName = null; // before init() is called, so we rely on + // events = null; // the user to maintain consistency. + if (pcmInputStream == rawInputStream) + fileLength = pcmInputStream.getFrameLength() / hopSize; + else + fileLength = -1; + if (!silent) + print(); + try { + if (audioOutputRequested) { + audioOut = AudioSystem.getSourceDataLine(audioFormat); + audioOut.open(audioFormat, outputBufferSize); + audioOut.start(); + } + } catch (Exception e) { + e.printStackTrace(); + audioOut = null; + } + } // init() + + /** Closes the input stream(s) associated with this object. */ + public void closeStreams() { + if (pcmInputStream != null) { + try { + pcmInputStream.close(); + if (pcmInputStream != rawInputStream) + rawInputStream.close(); + if (audioOut != null) { + audioOut.drain(); + audioOut.close(); + } + } catch (Exception e) {} + pcmInputStream = null; + audioOut = null; + } + } // closeStreams() + + protected void makeFreqMap(int fftSize, float sampleRate) { + freqMap = new int[fftSize/2+1]; + if (useChromaFrequencyMap) + makeChromaFrequencyMap(fftSize, sampleRate); + else + makeStandardFrequencyMap(fftSize, sampleRate); + } // makeFreqMap() + + /** Creates a map of FFT frequency bins to comparison bins. + * Where the spacing of FFT bins is less than 0.5 semitones, the mapping is + * one to one. Where the spacing is greater than 0.5 semitones, the FFT + * energy is mapped into semitone-wide bins. No scaling is performed; that + * is the energy is summed into the comparison bins. See also + * processFrame() + */ + protected void makeStandardFrequencyMap(int fftSize, float sampleRate) { + double binWidth = sampleRate / fftSize; + int crossoverBin = (int)(2 / (Math.pow(2, 1/12.0) - 1)); + int crossoverMidi = (int)Math.round(Math.log(crossoverBin*binWidth/440)/ + Math.log(2) * 12 + 69); + // freq = 440 * Math.pow(2, (midi-69)/12.0) / binWidth; + int i = 0; + while (i <= crossoverBin) + freqMap[i++] = i; + while (i <= fftSize/2) { + double midi = Math.log(i*binWidth/440) / Math.log(2) * 12 + 69; + if (midi > 127) + midi = 127; + freqMap[i++] = crossoverBin + (int)Math.round(midi) - crossoverMidi; + } + freqMapSize = freqMap[i-1] + 1; + // System.err.println("Map size: " + freqMapSize + + // "; Crossover at: " + crossoverBin); + } // makeStandardFrequencyMap() + + // Test whether chroma is better or worse + protected void makeChromaFrequencyMap(int fftSize, float sampleRate) { + double binWidth = sampleRate / fftSize; + int crossoverBin = (int)(1 / (Math.pow(2, 1/12.0) - 1)); + int crossoverMidi = (int)Math.round(Math.log(crossoverBin*binWidth/440)/ + Math.log(2) * 12 + 69); + // freq = 440 * Math.pow(2, (midi-69)/12.0) / binWidth; + int i = 0; + while (i <= crossoverBin) + freqMap[i++] = 0; + while (i <= fftSize/2) { + double midi = Math.log(i*binWidth/440) / Math.log(2) * 12 + 69; + freqMap[i++] = ((int)Math.round(midi)) % 12 + 1; + } + freqMapSize = 13; + // System.err.println("Map size: " + freqMapSize + + // "; Crossover at: " + crossoverBin); + // for (i = 0; i < fftSize / 2; i++) + // System.err.println("freqMap[" + i + "] = " + freqMap[i]); + } // makeChromaFrequencyMap() + + /** Reads a frame of input data, averages the channels to mono, scales + * to a maximum possible absolute value of 1, and stores the audio data + * in a circular input buffer. + * @return true if a frame (or part of a frame, if it is the final frame) + * is read. If a complete frame cannot be read, the InputStream is set + * to null. + */ + public boolean getFrame() { + if (pcmInputStream == null) + return false; + try { + int bytesRead = (int) pcmInputStream.read(inputBuffer); + if ((audioOut != null) && (bytesRead > 0)) + if (audioOut.write(inputBuffer, 0, bytesRead) != bytesRead) + System.err.println("Error writing to audio device"); + if (bytesRead < inputBuffer.length) { + if (!silent) + System.err.println("End of input: " + audioFileName); + closeStreams(); + return false; + } + } catch (IOException e) { + e.printStackTrace(); + closeStreams(); + return false; + } + frameRMS = 0; + double sample; + switch(channels) { + case 1: + for (int i = 0; i < inputBuffer.length; i += 2) { + sample = ((inputBuffer[i+1]<<8) | + (inputBuffer[i]&0xff)) / 32768.0; + frameRMS += sample * sample; + circBuffer[cbIndex++] = sample; + if (cbIndex == fftSize) + cbIndex = 0; + } + break; + case 2: // saves ~0.1% of RT (total input overhead ~0.4%) :) + for (int i = 0; i < inputBuffer.length; i += 4) { + sample = (((inputBuffer[i+1]<<8) | (inputBuffer[i]&0xff)) + + ((inputBuffer[i+3]<<8) | (inputBuffer[i+2]&0xff))) + / 65536.0; + frameRMS += sample * sample; + circBuffer[cbIndex++] = sample; + if (cbIndex == fftSize) + cbIndex = 0; + } + break; + default: + for (int i = 0; i < inputBuffer.length; ) { + sample = 0; + for (int j = 0; j < channels; j++, i+=2) + sample += (inputBuffer[i+1]<<8) | (inputBuffer[i]&0xff); + sample /= 32768.0 * channels; + frameRMS += sample * sample; + circBuffer[cbIndex++] = sample; + if (cbIndex == fftSize) + cbIndex = 0; + } + } + frameRMS = Math.sqrt(frameRMS / inputBuffer.length); + return true; + } // getFrame() + + /** Processes a frame of audio data by first computing the STFT with a + * Hamming window, then mapping the frequency bins into a part-linear + * part-logarithmic array, then (optionally) computing the half-wave + * rectified spectral difference from the previous frame, then (optionally) + * normalising to a sum of 1, then calculating the distance to all frames + * stored in the otherMatcher and storing them in the distance matrix, and + * finally updating the optimal path matrix using the dynamic time warping + * algorithm. + */ + protected void processFrame() { + if (getFrame()) { + for (int i = 0; i < fftSize; i++) { + reBuffer[i] = window[i] * circBuffer[cbIndex]; + if (++cbIndex == fftSize) + cbIndex = 0; + } + Arrays.fill(imBuffer, 0); + FFT.fft(reBuffer, imBuffer, FFT.FORWARD); + Arrays.fill(newFrame, 0); + for (int i = 0; i <= fftSize/2; i++) { + newFrame[freqMap[i]] += reBuffer[i] * reBuffer[i] + + imBuffer[i] * imBuffer[i]; + } + int frameIndex = frameCount % blockSize; + if (firstPM && (frameCount >= blockSize)) { + int len = last[frameCount - blockSize] - + first[frameCount - blockSize]; + byte[] dOld = distance[frameCount - blockSize]; + byte[] dNew = new byte[len]; + int[] bpcOld = bestPathCost[frameCount - blockSize]; + int[] bpcNew = new int[len]; + for (int i = 0; i < len; i++) { + dNew[i] = dOld[i]; + bpcNew[i] = bpcOld[i]; + } + distance[frameCount] = dOld; + distance[frameCount - blockSize] = dNew; + bestPathCost[frameCount] = bpcOld; + bestPathCost[frameCount - blockSize] = bpcNew; + } + double totalEnergy = 0; + if (useSpectralDifference) { + for (int i = 0; i < freqMapSize; i++) { + totalEnergy += newFrame[i]; + if (newFrame[i] > prevFrame[i]) { + frames[frameIndex][i] = newFrame[i] - prevFrame[i]; +// totalEnergy += frames[frameIndex][i]; + } else + frames[frameIndex][i] = 0; + } + } else { + for (int i = 0; i < freqMapSize; i++) { + frames[frameIndex][i] = newFrame[i]; + totalEnergy += frames[frameIndex][i]; + } + } + frames[frameIndex][freqMapSize] = totalEnergy; + if (wormHandler != null) + wormHandler.addPoint(totalEnergy); +// System.err.print(Format.d(max(newFrame),3) + " " + +// Format.d(totalEnergy,3)); + double decay = frameCount >= 200? 0.99: + (frameCount < 100? 0: (frameCount - 100) / 100.0); + if (ltAverage == 0) + ltAverage = totalEnergy; + else + ltAverage = ltAverage * decay + totalEnergy * (1.0 - decay); +// System.err.println(Format.d(ltAverage,4) + " " + +// Format.d(totalEnergy) + " " + +// Format.d(frameRMS)); + if (frameRMS <= silenceThreshold) + for (int i = 0; i < freqMapSize; i++) + frames[frameIndex][i] = 0; + else if (normalise1) + for (int i = 0; i < freqMapSize; i++) + frames[frameIndex][i] /= totalEnergy; + else if (normalise3) + for (int i = 0; i < freqMapSize; i++) + frames[frameIndex][i] /= ltAverage; + int stop = otherMatcher.frameCount; + int index = stop - blockSize; + if (index < 0) + index = 0; + first[frameCount] = index; + last[frameCount] = stop; + boolean overflow = false; + int mn=-1; + int mx=-1; + for ( ; index < stop; index++) { + int dMN = calcDistance(frames[frameIndex], + otherMatcher.frames[index % blockSize]); + if (mx<0) + mx = mn = dMN; + else if (dMN > mx) + mx = dMN; + else if (dMN < mn) + mn = dMN; + if (dMN >= 255) { + overflow = true; + dMN = 255; + } + if ((frameCount == 0) && (index == 0)) // first element + setValue(0, 0, 0, 0, dMN); + else if (frameCount == 0) // first row + setValue(0, index, ADVANCE_OTHER, + getValue(0, index-1, true), dMN); + else if (index == 0) // first column + setValue(frameCount, index, ADVANCE_THIS, + getValue(frameCount - 1, 0, true), dMN); + else if (index == otherMatcher.frameCount - blockSize) { + // missing value(s) due to cutoff + // - no previous value in current row (resp. column) + // - no diagonal value if prev. dir. == curr. dirn + int min2 = getValue(frameCount - 1, index, true); + // if ((firstPM && (first[frameCount - 1] == index)) || + // (!firstPM && (last[index-1] < frameCount))) + if (first[frameCount - 1] == index) + setValue(frameCount, index, ADVANCE_THIS, min2, dMN); + else { + int min1 = getValue(frameCount - 1, index - 1, true); + if (min1 + dMN <= min2) + setValue(frameCount, index, ADVANCE_BOTH, min1,dMN); + else + setValue(frameCount, index, ADVANCE_THIS, min2,dMN); + } + } else { + int min1 = getValue(frameCount, index-1, true); + int min2 = getValue(frameCount - 1, index, true); + int min3 = getValue(frameCount - 1, index-1, true); + if (min1 <= min2) { + if (min3 + dMN <= min1) + setValue(frameCount, index, ADVANCE_BOTH, min3,dMN); + else + setValue(frameCount, index, ADVANCE_OTHER,min1,dMN); + } else { + if (min3 + dMN <= min2) + setValue(frameCount, index, ADVANCE_BOTH, min3,dMN); + else + setValue(frameCount, index, ADVANCE_THIS, min2,dMN); + } + } + otherMatcher.last[index]++; + } // loop for row (resp. column) + double[] tmp = prevFrame; + prevFrame = newFrame; + newFrame = tmp; + frameCount++; + runCount++; + otherMatcher.runCount = 0; + if (overflow && !silent) + System.err.println("WARNING: overflow in distance metric: " + + "frame " + frameCount + ", val = " + mx); + // if (debug) + // System.err.println("Frame " + frameCount + ", d = " + (mx-mn)); + if ((frameCount % 100) == 0) { + if (!silent) { + System.err.println("Progress:" + frameCount + " " + + Format.d(ltAverage, 3)); + Profile.report(); + } + if ((progressCallback != null) && (fileLength > 0)) + progressCallback.setFraction((double)frameCount/fileLength); + } + if (frameCount == maxFrames) + closeStreams(); + } + } // processFrame() + + // protected double mins = 100, maxs = -100; + + /** Calculates the Manhattan distance between two vectors, with an optional + * normalisation by the combined values in the vectors. Since the + * vectors contain energy, this could be considered as a squared Euclidean + * distance metric. Note that normalisation assumes the values are all + * non-negative. + * @param f1 one of the vectors involved in the distance calculation + * @param f2 one of the vectors involved in the distance calculation + * @return the distance, scaled and truncated to an integer + */ + protected int calcDistance(double[] f1, double[] f2) { + double d = 0; + double sum = 0; + for (int i=0; i < freqMapSize; i++) { + d += Math.abs(f1[i] - f2[i]); + sum += f1[i] + f2[i]; + } + // System.err.print(" " + Format.d(d,3)); + if (sum == 0) + return 0; + if (normalise2) + return (int)(scale * d / sum); // 0 <= d/sum <= 2 + if (!normalise4) + return (int)(scale * d); + // double weight = (5 + Math.log(f1[freqMapSize] + f2[freqMapSize]))/10.0; + double weight = (8 + Math.log(sum)) / 10.0; + // if (weight < mins) { + // mins = weight; + // System.err.println(Format.d(mins,3) + " " + Format.d(maxs)); + // } + // if (weight > maxs) { + // maxs = weight; + // System.err.println(Format.d(mins,3) + " " + Format.d(maxs)); + // } + if (weight < 0) + weight = 0; + else if (weight > 1) + weight = 1; + return (int)(scale * d / sum * weight); + } // calcDistance() + + /** Retrieves values from the minimum cost matrix. + * @param i the frame number of this PerformanceMatcher + * @param j the frame number of the other PerformanceMatcher + * @return the cost of the minimum cost path to this location + */ + protected int getValue(int i, int j, boolean firstAttempt) { + if (firstPM) + return bestPathCost[i][j - first[i]]; + else + return otherMatcher.bestPathCost[j][i - otherMatcher.first[j]]; + } // getValue() + + /** Stores entries in the distance matrix and the optimal path matrix. + * @param i the frame number of this PerformanceMatcher + * @param j the frame number of the other PerformanceMatcher + * @param dir the direction from which this position is reached with + * minimum cost + * @param value the cost of the minimum path except the current step + * @param dMN the distance cost between the two frames + */ + protected void setValue(int i, int j, int dir, int value, int dMN) { + if (firstPM) { + distance[i][j - first[i]] = (byte)((dMN & MASK) | dir); + bestPathCost[i][j - first[i]] = + (value + (dir==ADVANCE_BOTH? dMN*2: dMN)); + } else { + if (dir == ADVANCE_THIS) + dir = ADVANCE_OTHER; + else if (dir == ADVANCE_OTHER) + dir = ADVANCE_THIS; + int idx = i - otherMatcher.first[j]; + if (idx == otherMatcher.distance[j].length) { + // This should never happen, but if we allow arbitrary pauses + // in either direction, and arbitrary lengths at end, it is + // better than an IndexOutOfBoundsException + int[] tmp1 = new int[idx*2]; + byte[] tmp2 = new byte[idx*2]; + for (int k = 0; k < idx; k++) { + tmp1[k] = otherMatcher.bestPathCost[j][k]; + tmp2[k] = otherMatcher.distance[j][k]; + } + otherMatcher.bestPathCost[j] = tmp1; + otherMatcher.distance[j] = tmp2; + } + otherMatcher.distance[j][idx] = (byte)((dMN & MASK) | dir); + otherMatcher.bestPathCost[j][idx] = + (value + (dir==ADVANCE_BOTH? dMN*2: dMN)); + } + } // setValue() + + /** Inner class for representing the corresponding note onset times in two + * audio files. Each Onset object represents a score position, with a + * beat number (numbered from 0 at the beginning of the first full bar), + * and the time (in seconds) of the average onset time of the notes at this + * score position in the respective audio files. + */ + class Onset { + double beat, time1, time2; + /** Constructor for Onset. + * @param beat the score position in beats, numbered from 0 at the + * beginning of the first complete bar + * @param time1 the average onset time of notes at score position b in + * the audio file corresponding to PerformanceMatcher pm1 + * @param time2 the average onset time of notes at score position b in + * the audio file corresponding to PerformanceMatcher pm2 + */ + public Onset(double beat, double time1, double time2) { + this.beat = beat; + this.time1 = time1; + this.time2 = time2; + } + } // class Onset + + /** Matches two match files, creating a composite list of corresponding + * onset times, consisting of the mean onset times of the notes in each + * notated score position. This is the basis for evaluating the alignment + * of the two performances. The asynchrony of notationally simultaneous + * notes limits the accuracy achievable by the PerformanceMatcher. + */ + public LinkedList<Onset> evaluateMatch(PerformanceMatcher pm) { + if ((events == null) || (pm.events == null)) + return null; + Event current; + double prevBeat = Double.NaN; + int count = 0; + double sum = 0; + double correction = 0; + LinkedList<Onset> l = new LinkedList<Onset>(); + for (Iterator<Event> i = events.iterator(); i.hasNext(); ) { + current = i.next(); + if (count == 0) { + sum = current.keyDown; + if (hasMatchFile) + correction = matchFileOffset - current.keyDown; + count = 1; + prevBeat = current.scoreBeat; + } else if (current.scoreBeat == prevBeat) { + sum += current.keyDown; + count++; + } else { + l.add(new Onset(prevBeat, sum/count + correction, -1)); + sum = current.keyDown; + count = 1; + prevBeat = current.scoreBeat; + } + } + if (count != 0) + l.add(new Onset(prevBeat, sum/count + correction, -1)); + if (l.size() == 0) + return null; + count = 0; + sum = 0; + correction = 0; + ListIterator<Onset> li = l.listIterator(); + Onset onset = li.next(); + prevBeat = Double.NaN; + for (Iterator<Event> i = pm.events.iterator(); i.hasNext(); ) { + current = i.next(); + if (count == 0) { + sum = current.keyDown; + if (pm.hasMatchFile) + correction = pm.matchFileOffset - current.keyDown; + count = 1; + prevBeat = current.scoreBeat; + } else if (current.scoreBeat == prevBeat) { + sum += current.keyDown; + count++; + } else { + while ((onset.beat < prevBeat) && li.hasNext()) + onset = li.next(); + while ((onset.beat > prevBeat) && li.hasPrevious()) + onset = li.previous(); + if (onset.beat == prevBeat) { + onset.time2 = sum/count + correction; + } else { + li.add(new Onset(prevBeat, -1, sum/count + correction)); + } + sum = current.keyDown; + count = 1; + prevBeat = current.scoreBeat; + } + } + if (count != 0) { + while ((onset.beat < prevBeat) && li.hasNext()) + onset = li.next(); + while ((onset.beat > prevBeat) && li.hasPrevious()) + onset = li.previous(); + if (onset.beat == prevBeat) { + onset.time2 = sum/count + correction; + } else { + li.add(new Onset(prevBeat, -1, sum/count + correction)); + } + } + li = l.listIterator(); + while (li.hasNext()) { + onset = li.next(); + String s = String.format("%8.3f %8.3f %8.3f", + onset.beat, onset.time1, onset.time2); + if ((onset.time1 < 0) || (onset.time2 < 0)) { + System.err.println("Match Error: " + s); + li.remove(); // notes must exist in both performances + } + } + return l; + } // evaluateMatch() + + /** Tracks a performance by choosing a likely band for the optimal path. */ + public static void doMatch(PerformanceMatcher pm1, PerformanceMatcher pm2, + ScrollingMatrix s) { + Finder f = new Finder(pm1, pm2); + while ((pm1.pcmInputStream != null) || (pm2.pcmInputStream != null)) { + // Profile.start(0); + if (pm1.frameCount < pm1.blockSize) { // fill initial block + pm1.processFrame(); + pm2.processFrame(); + } else if (pm1.pcmInputStream == null) { // stream 1 at end + // int index = pm1.first[pm1.frameCount - pm1.blockSize]; + // if (pm2.frameCount < index + (MAX_RUN_COUNT+1) * pm2.blockSize) + pm2.processFrame(); // see setValue() for alternative fix + // else { + // if (!silent) + // System.err.println("Closing streams early"); + // pm2.closeStreams(); + // } + } else if (pm2.pcmInputStream == null) // stream 2 at end + pm1.processFrame(); + else if (pm1.paused) { + if (pm2.paused) + try { + if (stop) + break; + Thread.sleep(100); + continue; // no update + } catch (InterruptedException e) {} + else + pm2.processFrame(); + } else if (pm2.paused) + pm1.processFrame(); + else if (pm1.runCount >= MAX_RUN_COUNT) // slope constraints + pm2.processFrame(); + else if (pm2.runCount >= MAX_RUN_COUNT) + pm1.processFrame(); + else + switch(f.getExpandDirection(pm1.frameCount-1,pm2.frameCount-1)){ + case ADVANCE_THIS: + pm1.processFrame(); + break; + case ADVANCE_OTHER: + pm2.processFrame(); + break; + case ADVANCE_BOTH: + pm1.processFrame(); + pm2.processFrame(); + break; + } + // Profile.log(0); + if (Thread.currentThread().isInterrupted()) { + System.err.println("info: INTERRUPTED in doMatch()"); + return; + } + if (!batchMode) + s.updateMatrix(true); + } + if (pm2.progressCallback != null) + pm2.progressCallback.setFraction(1.0); + if (!batchMode) { + s.updatePaths(false); // calculate complete paths + s.repaint(); + } + } // doMatch() + + /** Processes command line arguments. + * @param pm1 The first PerformanceMatcher + * @param pm2 The second PerformanceMatcher + * @param args Command line arguments<br> + * Usage: java PerformanceMatcher [optional-args] inputFile1 inputFile2<br> + * Optional args are:<ul> + * <li><b>-h hopTime</b> spacing of audio frames (in seconds, default 0.01) + * <li><b>-f frameTime</b> size of FFT (in seconds, default 0.01161) + * <li><b>-x maxFrames</b> stop after maxFrames frames have been processed + * <li><b>-m1 matchFile1 offset1</b> matchFile + start time for inputFile1 + * <li><b>-m2 matchFile2 offset2</b> matchFile + start time for inputFile2 + * <li><b>-w1 wormFile1</b> wormFile for inputFile1 + * <li><b>-w2 wormFile2</b> wormFile for inputFile2 + * <li><b>-n1</b> normalise FFT frames before comparison + * <li><b>-N1</b> do not normalise FFT frames before comparison (default) + * <li><b>-n2</b> normalise distance metric + * <li><b>-N2</b> do not normalise distance metric (default) + * <li><b>-n3</b> normalise ......... + * <li><b>-N3</b> do not normalise (default) + * <li><b>-d</b> use half-wave rectified spectral difference (default) + * <li><b>-D</b> do not use half-wave rectified spectral difference + * <li><b>-s scale</b> set scaling factor for distance metric + * <li><b>-b</b> set batch mode + * <li><b>-B</b> unset batch mode (default) + * <li><b>-smooth length</b> set smoothing window size to length + * <li><b>-a</b> audio out (file 1) + * <li><b>-A</b> audio out (file 2) + * <li><b>-l</b> live input (file 1) + * <li><b>-L</b> live input (file 2) + * <li><b>-w</b> live worm output (file 1) + * <li><b>-W</b> live worm output (file 2) + * <li><b>-z fileName</b> worm output (map file 1 to 2) + * <li><b>-Z fileName</b> worm output (map file 2 to 1) + * </ul> + */ + public static int processArgs(PerformanceMatcher pm1, + PerformanceMatcher pm2, String[] args) { + for (int i=0; i<args.length; i++) { + if (!silent) + System.err.println("args["+i+"] = "+args[i]); + if (args[i].equals("-h")) { + try { + pm1.hopTime = pm2.hopTime = Double.parseDouble(args[++i]); + } catch (RuntimeException e) { // NumberFormat/ArrayOutOfBounds + System.err.println(e); + } + } else if (args[i].equals("-f")) { + try { + pm1.fftTime = pm2.fftTime = Double.parseDouble(args[++i]); + } catch (RuntimeException e) { + System.err.println(e); + } + } else if (args[i].equals("-c")) { + try { + pm1.blockTime = pm2.blockTime=Double.parseDouble(args[++i]); + } catch (RuntimeException e) { + System.err.println(e); + } + } else if (args[i].equals("-s")) { + try { + pm1.scale = pm2.scale = Double.parseDouble(args[++i]); + } catch (RuntimeException e) { + System.err.println(e); + } + } else if (args[i].equals("-x")) { + try { + pm1.maxFrames = pm2.maxFrames = Integer.parseInt(args[++i]); + } catch (RuntimeException e) { + System.err.println(e); + } + } else if (args[i].equals("-m1")) { + try { + pm1.setMatchFile(args[++i], Double.parseDouble(args[++i])); + } catch (RuntimeException e) { + System.err.println(e); + } + } else if (args[i].equals("-m2")) { + try { + pm2.setMatchFile(args[++i], Double.parseDouble(args[++i])); + } catch (RuntimeException e) { + System.err.println(e); + } + } else if (args[i].equals("-w1")) { + pm1.setMatchFile(args[++i], 0, true); + } else if (args[i].equals("-w2")) { + pm2.setMatchFile(args[++i], 0, true); + } else if (args[i].equals("-d")) { + pm1.useSpectralDifference = pm2.useSpectralDifference = true; + } else if (args[i].equals("-D")) { + pm1.useSpectralDifference = pm2.useSpectralDifference = false; + } else if (args[i].equals("-n1")) { + pm1.normalise1 = pm2.normalise1 = true; + } else if (args[i].equals("-N1")) { + pm1.normalise1 = pm2.normalise1 = false; + } else if (args[i].equals("-n2")) { + pm1.normalise2 = pm2.normalise2 = true; + } else if (args[i].equals("-N2")) { + pm1.normalise2 = pm2.normalise2 = false; + } else if (args[i].equals("-n3")) { + pm1.normalise3 = pm2.normalise3 = true; + } else if (args[i].equals("-N3")) { + pm1.normalise3 = pm2.normalise3 = false; + } else if (args[i].equals("-n4")) { + pm1.normalise4 = pm2.normalise4 = true; + } else if (args[i].equals("-N4")) { + pm1.normalise4 = pm2.normalise4 = false; + } else if (args[i].equals("--use-chroma-map")) { + pm1.useChromaFrequencyMap = pm2.useChromaFrequencyMap = true; + } else if (args[i].equals("-b")) { + batchMode = true; + matrixVisible = false; + guiVisible = false; + } else if (args[i].equals("-B")) { + batchMode = false; + } else if (args[i].equals("-v")) { + matrixVisible = true; + batchMode = false; + } else if (args[i].equals("-V")) { + matrixVisible = false; + } else if (args[i].equals("-g")) { + guiVisible = true; + batchMode = false; + } else if (args[i].equals("-G")) { + guiVisible = false; + } else if (args[i].equals("-q")) { + silent = true; + } else if (args[i].equals("-Q")) { + silent = false; + } else if (args[i].equals("-a")) { + pm1.audioOutputRequested = true; + pm2.audioOutputRequested = false; + } else if (args[i].equals("-A")) { + pm2.audioOutputRequested = true; + pm1.audioOutputRequested = false; + } else if (args[i].equals("-r")) { + guiVisible = true; + batchMode = false; + GUI.loadFile = args[++i]; + } else if (args[i].equals("-l")) { + pm1.audioOutputRequested = true; + pm2.audioOutputRequested = false; + pm1.setLiveInput(); + } else if (args[i].equals("-L")) { + pm2.audioOutputRequested = true; + pm1.audioOutputRequested = false; + pm2.setLiveInput(); + } else if (args[i].equals("-w")) { + pm1.wormHandler = new WormHandler(pm1); + pm1.liveWorm = true; + } else if (args[i].equals("-W")) { + pm2.wormHandler = new WormHandler(pm2); + pm2.liveWorm = true; + } else if (args[i].equals("-z")) { + batchMode = true; + pm1.wormHandler = new WormHandler(pm1); + pm1.matchFileName = args[++i]; + } else if (args[i].equals("-Z")) { + batchMode = true; + pm2.wormHandler = new WormHandler(pm2); + pm2.matchFileName = args[++i]; + } else + return i; + // System.err.println("WARNING: Ignoring argument: " + args[i]); + } + return args.length; + } // processArgs() + + /** Entry point for command line version of performance matcher. + * @see #processArgs(PerformanceMatcher,PerformanceMatcher,String[]) + * processArgs(PerformanceMatcher,PerformanceMatcher,String[]) for + * documentation of command-line arguments. + */ + public static void main(String[] args) { + PerformanceMatcher pm1 = new PerformanceMatcher(null); + PerformanceMatcher pm2 = new PerformanceMatcher(pm1); + pm1.setOtherMatcher(pm2); + int argc = processArgs(pm1, pm2, args); + ScrollingMatrix s = new ScrollingMatrix(pm1, pm2); // pm1 vertical + MatrixFrame m = new MatrixFrame(s, matrixVisible); + GUI g = new GUI(pm1, pm2, s, guiVisible); + if (guiVisible && (argc != args.length)) + g.addFiles(args, argc); + else { + if ((argc < args.length) && (pm1.pcmInputStream == null)) + pm1.setInputFile(args[argc++]); + if ((argc < args.length) && (pm2.pcmInputStream == null)) + pm2.setInputFile(args[argc++]); + if (pm2.pcmInputStream != null) { + doMatch(pm1, pm2, s); + s.updatePaths(false); + if ((pm1.hasMatchFile && pm2.hasMatchFile) || + (pm1.hasWormFile && pm2.hasWormFile)) + s.evaluatePaths(); + else if (pm1.hasWormFile && !pm2.hasWormFile) { + if ((pm2.matchFileName != null) && !pm2.hasMatchFile) + s.wormHandler.write(new File(pm2.matchFileName), false); + else + g.saveWormFile(); + } + } + if (!silent) + System.err.println("Processed " + pm1.frameCount + " and " + + pm2.frameCount + " frames of "+pm1.fftSize+" samples"); + } + if (batchMode) // ScrollThread is running + System.exit(0); + } // main() + +} // class PerformanceMatcher
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/ScrollingMatrix.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,1048 @@ +package at.ofai.music.match; + +import java.awt.Canvas; +import java.awt.Color; +import java.awt.Dimension; +import java.awt.Graphics; +import java.awt.Rectangle; +import java.awt.event.InputEvent; +import java.awt.event.KeyEvent; +import java.awt.event.KeyListener; +import java.awt.event.MouseEvent; +import java.awt.event.MouseListener; +import java.awt.event.MouseMotionListener; +import java.awt.image.BufferedImage; +import java.util.LinkedList; +import java.util.ListIterator; + +import javax.swing.JFrame; + +//import at.ofai.music.util.Profile; +import at.ofai.music.util.PSPrinter; + +/** Displays a 2D cost matrix in greyscale, and shows the minimum cost paths + * calculated forwards and backwards in real time, scrolling as necessary. + * The backward path algorithm uses DTW; the forward paths use OLTW and OLTW + * with smoothing. The latter are (causal) real-time estimates of the optimal + * path. + */ +class ScrollingMatrix extends Canvas implements KeyListener, + MouseListener, + MouseMotionListener { + + protected JFrame parent; + protected BufferedImage img; + protected Graphics gImage, gThis; + protected Dimension sz; + protected Rectangle clipRect; + protected LinkedList<PerformanceMatcher.Onset> correct; + protected ListIterator<PerformanceMatcher.Onset> listIterator; + protected double hop1, hop2; + protected int x0prev, y0prev; + protected int showHorizontal, showVertical; + protected int x0, y0; // origin + protected int xscroll, yscroll; + protected int pathStartX, pathStartY; // , pathEndX, pathEndY; + protected int[] fPathX, fPathY, bPathX, bPathY, sPathX, sPathY; + protected int fPathLength, bPathLength, sPathLength; + protected PerformanceMatcher pm1, pm2; + protected Finder finder; + protected boolean showForwardPath, showBackwardPath, showSmoothedPath; + protected boolean showCorrect; + protected boolean showCentreCrossbar; + private transient int[] range = new int[2]; + protected FixedPoint lastPoint; + protected int currentX, currentY; + protected int xSize, ySize; + protected WormHandler wormHandler; + protected boolean liveWorm; + protected int paintCount; + + static final int white = 0x00FFFFFF; + static final int red = 0x00FF0000; + static final int green = 0x0000FF00; + static final int blue = 0x000000FF; + static final int yellow = 0x00FFFF00; + static final int cyan = 0x0000FFFF; + static final int magenta = 0x00FF00FF; + static final int black = 0x00000000; + static final int scrollFactor = 10; + static final int DEFAULT_HEIGHT = 600; + static final int DEFAULT_WIDTH = 600; + static final long serialVersionUID = 0; + + /** Creates a window containing the similarity matrix (distance matrix) for + * the two PerformanceMatchers showing the minimum cost paths. + * By pressing button 1, the minimum cost function at the current mouse + * position can be shown. + * Note that if <code>PerformanceMatcher.batchMode</code> is + * <code>true</code> then no display + * will be opened, but the optimal path will be calculated and printed to + * standard output, including the evaluation based on the match files. + * @param pm1 the PerformanceMatcher corresponding to the vertical axis of + * the display + * @param pm2 the PerformanceMatcher corresponding to the horizontal axis + * of the display + */ +// public static ScrollingMatrix showInFrame(PerformanceMatcher pm1, +// PerformanceMatcher pm2) { +// ScrollingMatrix sm = new ScrollingMatrix(pm1, pm2); +// if (!PerformanceMatcher.batchMode) +// new MatrixFrame(sm); +// return sm; +// } // showInFrame() + + public ScrollingMatrix(PerformanceMatcher pm1, PerformanceMatcher pm2) { + this(pm1, pm2, DEFAULT_HEIGHT, DEFAULT_WIDTH, pm1.evaluateMatch(pm2)); + } // constructor + + /** Constructor, usually called by showInFrame(). + * The two performance matchers contain m, the minimum path cost matrix, + * and d, the distance matrix (bits 2-7) and the direction of the minimum + * cost path (bits 0-1), which are accessed by finder. The first dimension + * of the matrices corresponds to the first performance matcher, which is + * indexed by row number (i.e. shown on the vertical axis). + * @param pm1 the first PerformanceMatcher + * @param pm2 the second PerformanceMatcher + * @param sz1 the vertical size of the display window + * @param sz2 the horizontal size of the display window + * @param l the list of paired onset times from the match files of the two + * performances + */ + public ScrollingMatrix(PerformanceMatcher pm1, PerformanceMatcher pm2, + int sz1, int sz2, LinkedList<PerformanceMatcher.Onset> l) { + parent = null; + this.pm1 = pm1; + this.pm2 = pm2; + liveWorm = pm1.liveWorm || pm2.liveWorm; + wormHandler = pm1.wormHandler; + if (wormHandler == null) + wormHandler = pm2.wormHandler; + if (wormHandler != null) + wormHandler.setScrollingMatrix(this); + xSize = sz2; + ySize = sz1; + finder = new Finder(pm1, pm2); + clipRect = new Rectangle(0, 0, xSize, ySize); + correct = l; + if (correct != null) + listIterator = correct.listIterator(); + else + listIterator = null; + sz = new Dimension(xSize, ySize); + showForwardPath = false; + showBackwardPath = true; + showSmoothedPath = false; + showCorrect = true; + new Thread(new ScrollThread(this)).start(); + addKeyListener(this); + addMouseListener(this); + addMouseMotionListener(this); + init(); + } // constructor + + public void init() { + hop1 = pm1.hopTime; + hop2 = pm2.hopTime; + showHorizontal = showVertical = -1; + pathStartX = pathStartY = -1; + fPathLength = 0; + bPathLength = 0; + sPathLength = 0; + fPathX = null; // forces creation of arrays in checkArray + paintCount = 0; + x0 = xSize; + y0 = -1; + x0prev = x0; + y0prev = y0; + showCentreCrossbar = false; + lastPoint = null; + currentX = -1; + currentY = -1; + clearImage(); + } // init() + + class ScrollThread implements Runnable { + + ScrollingMatrix sm; + + ScrollThread(ScrollingMatrix s) { + sm = s; + } + + public void run() { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + while (true) { + try { + synchronized (sm) { + sm.wait(); + } + sm.scroll(sm.xscroll, sm.yscroll); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } // inner class ScrollThread + + protected void setTime(double t1, AudioFile current) { + double t2; + showCentreCrossbar = true; + if (pm1.audioFile == current) { + t2 = pm2.audioFile.fromReferenceTimeD(current.toReferenceTimeD(t1)); + } else if (pm2.audioFile == current) { + t2 = t1; + t1 = pm1.audioFile.fromReferenceTimeD(current.toReferenceTimeD(t1)); + } else + return; + x0 = (int)(-t2 / hop2 + xSize / 2); + y0 = (int)(t1 / hop1 + ySize / 2); + redrawImage(); + repaint(); + } // setTime() + + protected void scroll(int x, int y) { + if ((x != 0) || (y != 0)) { + x0 -= x; + y0 -= y; + redrawImage(); + repaint(); + } + } // scroll() + + protected void updateMatrix(boolean playing) { + // Profile.start(1); + if (playing) { // scroll to end + if (liveWorm && (wormHandler != null)) + wormHandler.update(); + if (++paintCount >= 4) { + x0 = sz.width - pm2.frameCount; + y0 = pm1.frameCount - 1; + redrawImage(); + paintCount = 0; + updatePaths(playing); + repaint(); // hack because it is too slow + } + } else { + updatePaths(playing); + setTime(0, pm1.audioFile); + } + // Profile.log(1); + } // updateMatrix() + + protected void clearImage() { + if (gImage != null) { + gImage.setColor(Color.white); + gImage.fillRect(0, 0, sz.width, sz.height); + } + } // clearImage() + + protected void redrawImage() { + if (img == null) { + img = getGraphicsConfiguration().createCompatibleImage(sz.width, + sz.height); + gImage = img.getGraphics(); + clearImage(); + } + int dx = x0 - x0prev; + int dy = y0 - y0prev; + if ((dx != 0) || (dy != 0)) { + /* if (gThis == null) + gThis = getGraphics(); + try { + gThis.copyArea(0, 0, sz.width, sz.height, dx, dy);//much faster + } catch (Exception e) { + if (!PerformanceMatcher.silent) + System.err.print(e); + repaint(); + } + */ gImage.copyArea(0, 0, sz.width, sz.height, dx, dy); + } + int start, stop, colour; + if (dx != 0) { + if (dx < 0) { + start = Math.max(0, sz.width + dx); + stop = sz.width; + } else { + start = 0; + stop = Math.min(dx, sz.width); + } + for (int x = start; x < stop; x++) + for (int y = 0; y < sz.height; y++) { + if (finder.find(y0 - y, x - x0)) { + colour = finder.getDistance() & PerformanceMatcher.MASK; + colour |= (colour << 8) | (colour << 16); + img.setRGB(x,y,colour); + } else + img.setRGB(x,y,white); + } + } + if (dy != 0) { + if (dy < 0) { + start = Math.max(0, sz.height + dy); + stop = sz.height; + } else { + start = 0; + stop = Math.min(dy, sz.height); + } + for (int y = start; y < stop; y++) + for (int x = 0; x < sz.width; x++) { + if (finder.find(y0 - y, x - x0)) { + colour = finder.getDistance() & PerformanceMatcher.MASK; + colour |= (colour << 8) | (colour << 16); + img.setRGB(x,y,colour); + } else + img.setRGB(x,y,white); + } + } + /* + if (dx > 0) + repaint(0, 0, dx, sz.height); + else if (dx < 0) + repaint(sz.width+dx, 0, sz.width, sz.height); + if (dy > 0) + repaint(0, 0, sz.width, dy); + else if (dy < 0) + repaint(0, sz.height+dy, sz.width, sz.height); +/* */ + x0prev = x0; + y0prev = y0; +// Do we want this? Certainly not while playing. Would need to set slider too. +// if (gui != null) { +// if (gui.audioPlayer.currentFile == pm1.audioFile) +// gui.setTimer((y0 - ySize / 2) * hop1, null); +// else if (gui.audioPlayer.currentFile == pm2.audioFile) +// gui.setTimer((xSize / 2 - x0) * hop2, null); +// } + } // redrawImage() + + protected static int[] replace(int[] old, int oldSize, int newSize) { + return replace(old, oldSize, newSize, false); + } // replace()/3 + + protected static int[] replace(int[] old,int oldSz,int newSz,boolean rev) { + if ((oldSz != newSz) || rev) { + int[] tmp = new int[newSz]; + if (rev) + for (int i = 0; i < oldSz; i++) + tmp[i] = old[oldSz-1-i]; + else + for (int i = 0; i < oldSz; i++) + tmp[i] = old[i]; + return tmp; + } else if (rev) { // not used + for (int i = 0; i < oldSz / 2; i++) { + int tmp = old[i]; + old[i] = old[oldSz-1-i]; + old[oldSz-1-i] = tmp; + } + } + return old; + } // replace()/4 + + protected void checkPathArrays(boolean playing) { + int len = 2 * (pm1.frameCount + pm2.frameCount + 1); + if ((fPathX == null) && (len < 2 * (sz.width + sz.height))) { + len = 2 * (sz.width + sz.height); // initial size + playing = false; // so the arrays are created below + } + if ((fPathX == null)||(fPathX.length < pm1.frameCount +pm2.frameCount)){ + fPathX = replace(fPathX, fPathLength, len); + fPathY = replace(fPathY, fPathLength, len); + } + if (!playing || (bPathX.length < pm1.frameCount + pm2.frameCount)) { + bPathX = replace(bPathX, bPathLength, len); + bPathY = replace(bPathY, bPathLength, len); + sPathX = replace(sPathX, sPathLength, len); + sPathY = replace(sPathY, sPathLength, len); + } + } // checkPathArrays() + + public void adjustCurrentPoint(int x, int y) { + currentX = x; + currentY = y; + } // adjustCurrentPoint() + + public void finaliseCurrentPoint(int x, int y) { + if (finder.find(y,x)) { + FixedPoint p = lastPoint.insert(x,y); + if (p != null) { + finder.recalculatePathCostMatrix(p.y, p.x, p.next.y, p.next.x); + updateBackwardPath(false); + updateSmoothedPath(false); + } + } + } // finaliseCurrentPoint() + + public void updatePaths(boolean playing) { + // Profile.start(2); + int tmpx = finder.bestCol; + int tmpy = finder.bestRow; + updateForwardPath(playing); + if (playing) { + pathStartX = tmpx; + pathStartY = tmpy; + } + updateBackwardPath(playing); + updateSmoothedPath(playing); + // Profile.log(2); + } // updatePaths() + + /** Recalculates the OLTW path. + * Note that the forward path calculation is very slow; it takes ~1.5 s + * for a 90 second piece (Chopin Etude). + */ + public void updateForwardPath(boolean playing) { + checkPathArrays(playing); + int x, y; + if (fPathLength == 0) { + if (pathStartX > 0) + x = pathStartX; + else + x = 0; + if (pathStartY > 0) + y = pathStartY; + else + y = 0; + } else { + x = fPathX[fPathLength - 1]; + y = fPathY[fPathLength - 1]; + } + if (!finder.find(y,x)) + return; + while ((x < pm2.frameCount) && (y < pm1.frameCount)) { + int to = finder.getExpandDirection(y,x,true); + switch (to) { + case PerformanceMatcher.ADVANCE_THIS: y++; break; + case PerformanceMatcher.ADVANCE_OTHER: x++; break; + case PerformanceMatcher.ADVANCE_BOTH: y++; x++; break; + } + if (finder.find(y,x)) { + fPathX[fPathLength] = x; + fPathY[fPathLength] = y; + fPathLength++; + } else if ((x < pm2.frameCount) && (y < pm1.frameCount)) + System.err.println("updateForwardPath(): path error at (" + + y + ", " + x + ")"); + else + break; + } + } // updateForwardPath() + + /** Recalculates the DTW path, given a list of constraints, that is, points + * that the path should pass through. Note that the forward path + * recalculation had to be removed from this method because it doesn't + * need to change and it takes 1.5 seconds to calculate for a 90 second + * piece (Chopin Etude). + */ + public void updateBackwardPathOld(boolean playing) { + checkPathArrays(playing); + int x, y, xStop, yStop; + if ((pathStartX >= 0) && (pathStartY >= 0)) { + x = pathStartX; + y = pathStartY; + } else { + x = pm2.frameCount - 1; + y = pm1.frameCount - 1; + } + if (playing && !(pm1.paused && pm2.paused)) { + xStop = x - sz.width - 1; + if (xStop < 0) + xStop = 0; + yStop = y - sz.height - 1; + if (yStop < 0) + yStop = 0; + } else + xStop = yStop = 0; + if ((bPathX[0] != x) || (bPathY[0] != y) || (bPathLength == 0) || + (bPathX[bPathLength-1] > xStop) || + (bPathY[bPathLength-1] > yStop)) { + bPathLength = 0; + while (finder.find(y,x) && ((x > xStop) || (y > yStop))) { + bPathX[bPathLength] = x; + bPathY[bPathLength] = y; + bPathLength++; + switch (finder.getDistance() & PerformanceMatcher.ADVANCE_BOTH){ + case PerformanceMatcher.ADVANCE_THIS: y--; break; + case PerformanceMatcher.ADVANCE_OTHER: x--; break; + case PerformanceMatcher.ADVANCE_BOTH: x--; y--; break; + } + } + } + } // updateBackwardPathOld() + + public void updateBackwardPath(boolean playing) { + if (lastPoint == null) { + updateBackwardPathOld(playing); + return; + } + checkPathArrays(playing); + FixedPoint p = lastPoint.prev; + while (p.prev != null) + p = p.prev; + bPathLength = 0; +// long startTime = System.nanoTime(); +// long currentTime; +// while (p.next != null) { // restrict matrix to paths through all p's +// System.err.print("Recalc: " + p.x + "," + p.y + " to " + +// p.next.x + "," + p.next.y); +// finder.recalculatePathCostMatrix(p.y, p.x, p.next.y, p.next.x); +// currentTime = System.nanoTime(); +// System.err.println(" Time: " + ((currentTime - startTime) / 1e6)); +// startTime = currentTime; +// p = p.next; +// } + p = lastPoint; + while (p.prev != null) { // calculate subpaths + int x = p.x; + int y = p.y; + p = p.prev; +// System.err.print("Path from: " +x+ "," +y+ " to " +p.x+ "," + p.y); + loop: while (finder.find(y,x) && ((x > p.x) || (y > p.y))) { + bPathX[bPathLength] = x; + bPathY[bPathLength] = y; + bPathLength++; + switch (finder.getDistance() & PerformanceMatcher.ADVANCE_BOTH){ + case PerformanceMatcher.ADVANCE_THIS: y--; break; + case PerformanceMatcher.ADVANCE_OTHER: x--; break; + case PerformanceMatcher.ADVANCE_BOTH: x--; y--; break; + case 0: System.err.println("\n" + x + "," + y); break loop; + } + } +// currentTime = System.nanoTime(); +// System.err.println(" Time: " + ((currentTime - startTime) / 1e6)); +// startTime = currentTime; + } + } // updateBackwardPath() + + public void updateSmoothedPath(boolean playing) { + for (int i = 0; i < bPathLength; i++) { + sPathX[i] = bPathX[bPathLength-1-i]; + sPathY[i] = bPathY[bPathLength-1-i]; + } + sPathLength = Path.smooth(sPathX, sPathY, bPathLength); + if ((pm1.audioFile == null) || (pm2.audioFile == null)) + return; + if (pm1.audioFile.isReference) { + pm2.audioFile.setMatch(sPathX, pm2.hopTime, sPathY, pm1.hopTime, + sPathLength); + pm2.audioFile.setFixedPoints(lastPoint, true); + } else if (pm2.audioFile.isReference) { + pm1.audioFile.setMatch(sPathY, pm1.hopTime, sPathX, pm2.hopTime, + sPathLength); + pm2.audioFile.setFixedPoints(lastPoint, false); + System.err.println("Warning: pm1 is not reference file"); + } + } // updateSmoothedPath() + + public int[] forwardPathX() { + return replace(fPathX, fPathLength, fPathLength); + } // forwardPathX() + + public int[] forwardPathY() { + return replace(fPathY, fPathLength, fPathLength); + } // forwardPathY() + + public int[] backwardPathX() { + return replace(bPathX, bPathLength, bPathLength, true); + } // backwardPathX() + + public int[] backwardPathY() { + return replace(bPathY, bPathLength, bPathLength, true); + } // backwardPathY() + + public int[] smoothedPathX() { + return replace(sPathX, sPathLength, sPathLength); + } // smoothedPathX() + + public int[] smoothedPathY() { + return replace(sPathY, sPathLength, sPathLength); + } // smoothedPathY() + + public void evaluatePaths() { + if ((correct != null) && (correct.size() > 0)) { + System.out.println("Evaluation of forward (OLTW) path"); + evaluateForwards(fPathX, fPathY, fPathLength); + System.out.println("Evaluation of smoothed DTW path"); + evaluateForwards(sPathX, sPathY, sPathLength); + System.out.println("Evaluation of backward (DTW) path"); + evaluateBackwards(bPathX, bPathY, bPathLength); + } else + System.err.println("evaluation error: match file empty or missing"); + } // evaluatePaths() + + protected void evaluateForwards(int[] pathx, int[] pathy, int pathLength) { + listIterator = correct.listIterator(); + PerformanceMatcher.Onset onset; + int xc, yc, dist, best, besti; + int count = 0; + int sumDist = 0; + int i = 0; + System.out.println("note err cum.err av(s) beat y yc x xc"); + while (listIterator.hasNext()) { // evaluate best path + onset = listIterator.next(); + yc = (int) Math.round(onset.time1 / hop1); + xc = (int) Math.round(onset.time2 / hop2); + while (((pathx[i] >= xc) || (pathy[i] >= yc)) && (i > 0)) + i--; + best = Math.abs(pathy[i]-yc) + Math.abs(pathx[i]-xc); // Manhattan + besti = i; + while (((pathx[i] <= xc) || (pathy[i] <= yc)) && (i < pathLength)) { + i++; + dist = Math.abs(pathy[i]-yc) + Math.abs(pathx[i]-xc); + if (dist < best) { + best = dist; + besti = i; + } + } + sumDist += best; + count++; + System.out.println(String.format( + "%4d %4d %6d %5.3f %7.2f %4d %4d %4d %4d", + count, best, sumDist, hop1*sumDist/count, + onset.beat, pathy[besti], yc, pathx[besti], xc)); + } + System.out.println("Summary[" + count + "] " + sumDist); + } // evaluateForwards() + + protected void evaluateBackwards(int[] pathx, int[] pathy, int pathLength) { + listIterator = correct.listIterator(); // was listIter(correct.size()) + PerformanceMatcher.Onset onset; + int xc, yc, dist, best, besti; + int count = 0; + int sumDist = 0; + int i = pathLength - 1; + System.out.println("note err cum.err av(s) beat y yc x xc"); + while (listIterator.hasNext()) { // evaluate best path + onset = listIterator.next(); + yc = (int) Math.round(onset.time1 / hop1); + xc = (int) Math.round(onset.time2 / hop2); + while (((pathx[i] >= xc) || (pathy[i] >= yc)) && (i < pathLength-1)) + i++; + best = Math.abs(pathy[i]-yc) + Math.abs(pathx[i]-xc); + besti = i; + while (((pathx[i] <= xc) || (pathy[i] <= yc)) && (i > 0)) { + i--; + dist = Math.abs(pathy[i]-yc) + Math.abs(pathx[i]-xc); + if (dist < best) { + best = dist; + besti = i; + } + } + sumDist += best; + count++; + System.out.println(String.format( + "%4d %4d %6d %5.3f %7.2f %4d %4d %4d %4d", + count, best, sumDist, hop1*sumDist/count, + onset.beat, pathy[besti], yc, pathx[besti], xc)); + } + System.out.println("Summary[" + count + "] " + sumDist); + } // evaluateBackwards() + +/* + protected void paintPixel(int x, int y, int colour) { + x = x0 + x; + y = y0 - y; + if ((x >= 0) && (y >= 0) && (x < sz.width) && (y < sz.height)) + img.setRGB(x,y,colour); + } // paintPixel() + + public void paintPathForwards(int x, int y) { + while (finder.find(y,x)) { + paintPixel(x, y, red); + switch (finder.getExpandDirection(y,x)) { + case PerformanceMatcher.ADVANCE_THIS: y++; break; + case PerformanceMatcher.ADVANCE_OTHER: x++; break; + case PerformanceMatcher.ADVANCE_BOTH: x++; y++; break; + } + } + } // paintPathForwards() + + public void paintPathBackwards(int x, int y) { + while (finder.find(y,x) && (x+y > 0)) { // trace best path + paintPixel(x, y, magenta); + switch (finder.getDistance() & PerformanceMatcher.ADVANCE_BOTH) { + case PerformanceMatcher.ADVANCE_THIS: y--; break; + case PerformanceMatcher.ADVANCE_OTHER: x--; break; + case PerformanceMatcher.ADVANCE_BOTH: x--; y--; break; + } + } + } // paintPathBackwards() +*/ + + /** Overrides Component.update() to stop flashing due to clearing bkgnd. + * Only called for non-Swing Components, in response to a repaint(). + */ + public void update(Graphics g) { + paint(g); + } // update() + + public void paint(Graphics g) { + // Profile.start(3); + if (img != null) + g.drawImage(img, 0, 0, null); + paintCorrect(g); + paintPaths(g); + paintPathCostFunctions(g); + paintFixedPoints(g); + if (parent != null) + parent.repaint(); + synchronized (this) { // Scroller needs to know + notify(); + } + // Profile.log(3); + } // paint() + + /** Paints the minimum cost paths forwards, smoothed and backwards. + * @param g the Graphics context for painting + */ + protected void paintPaths(Graphics g) { + if (g.getClipBounds(clipRect) == null) + clipRect.setRect(0, 0, sz.width, sz.height); + if (showForwardPath) + paintPath(g, fPathX, fPathY, fPathLength, Color.blue, false); + if (showSmoothedPath) + paintPath(g, sPathX, sPathY, sPathLength, Color.yellow, false); + if (showBackwardPath) + paintPath(g, bPathX, bPathY, bPathLength, Color.green, true); + } // paintPaths() + + /** Paints a sequence of points. + * @param g the Graphics context for painting + * @param xp the x-coordinates of the points + * @param yp the y-coordinates of the points + * @param len the number of points to paint + * @param c the colour to paint the points + */ + protected void paintPath(Graphics g, int[] xp, int[] yp, + int len, Color c, boolean reverse) { + g.setColor(c); + for (int i = 0; i < len; i++) { + int x = x0 + xp[i]; + int y = y0 - yp[i]; + if (reverse) { + if ((x < clipRect.x) && (y >= clipRect.y + clipRect.height)) + break; + if ((x < clipRect.x + clipRect.width) || (y >= clipRect.y)) + g.drawLine(x, y, x, y); // why is there no drawPixel()??! + } else { + if ((x >= clipRect.x + clipRect.width) || (y < clipRect.y)) + break; + if ((x >= clipRect.x) && (y < clipRect.y + clipRect.height)) + g.drawLine(x, y, x, y); // why is there no drawPixel()??! + } + } + } // paintPath() + + /** Paints the onsets of corresponding chords as red '+'s. + * @param g the Graphics object for painting this object + */ + protected void paintCorrect(Graphics g) { + g.setColor(Color.red); + if (showCentreCrossbar) { + g.drawLine(xSize/2, 0, xSize/2, ySize); + g.drawLine(0, ySize/2, xSize, ySize/2); + } + if (showCorrect && (listIterator != null)) { + if (g.getClipBounds(clipRect) == null) + clipRect.setRect(0, 0, sz.width, sz.height); + while (listIterator.hasPrevious()) { + PerformanceMatcher.Onset onset = listIterator.previous(); + int y = y0 - (int) (onset.time1 / hop1); + int x = x0 + (int) (onset.time2 / hop2); + if ((x < clipRect.x) || (y >= clipRect.y + clipRect.height)) + break; + if ((x < clipRect.x + clipRect.width) && (y >= clipRect.y)) { + g.drawLine(x-5,y,x+5,y); + g.drawLine(x,y-5,x,y+5); + } + } + while (listIterator.hasNext()) { + PerformanceMatcher.Onset onset = listIterator.next(); + int y = y0 - (int) (onset.time1 / hop1); + int x = x0 + (int) (onset.time2 / hop2); + if ((x >= clipRect.x + clipRect.width) || (y < clipRect.y)) + break; + if ((x >= clipRect.x) && (y < clipRect.y + clipRect.height)) { + g.drawLine(x-5,y,x+5,y); + g.drawLine(x,y-5,x,y+5); + } + } + } + } // paintCorrect() + + protected void paintFixedPoints(Graphics g) { + int rad = 5; + if (currentX >= 0) { + int x = x0 + currentX; + int y = y0 - currentY; + g.setColor(Color.red); + g.drawOval(x - rad, y - rad, 2 * rad, 2 * rad); + } + if (lastPoint != null) { + g.setColor(Color.blue); + for (FixedPoint p = lastPoint.prev; p.prev != null; p = p.prev) { + int x = x0 + p.x; + int y = y0 - p.y; + g.drawOval(x - rad, y - rad, 2 * rad, 2 * rad); + } + } + } // paintFixedPoints() + + /** Shows the matrix values along a vertical and horizontal axis. + * Note: The matrix point (x,y) is mapped on the screen to (x+x0, -y+y0) + */ + protected void paintPathCostFunctions(Graphics g) { + int scale = 1; + int x = showVertical - x0; + if ((showVertical >= 0) && (showVertical < sz.width) && + (x >= 0) && (x < pm2.frameCount)) { + g.setColor(Color.cyan); + g.drawLine(showVertical, 0, showVertical, sz.height); + finder.getRowRange(x, range); + int min = finder.getPathCost(range[0], x); + for (int y = range[0] + 1; y < range[1]; y++) { + int tmp = finder.getPathCost(y, x); + if ((tmp > 0) && (tmp < min)) + min = tmp; + } + int prev = (finder.getPathCost(range[0], x) - min) / scale; + for (int y = range[0] + 1; y < range[1]; y++) { + int curr = (finder.getPathCost(y, x) - min) / scale; + int y1 = y0 - y; + g.drawLine(showVertical + prev, y1, showVertical + curr, y1-1); + prev = curr; + } + } + int y = y0 - showHorizontal; + if ((showHorizontal >= 0) && (showHorizontal < sz.height) && + (y >= 0) && (y < pm1.frameCount)) { + g.setColor(Color.magenta); + g.drawLine(0, showHorizontal, sz.width, showHorizontal); + finder.getColRange(y, range); + int min = finder.getPathCost(y, range[0]); + for (int xx = range[0] + 1; xx < range[1]; xx++) { + int tmp = finder.getPathCost(y, xx); + if ((tmp > 0) && (tmp < min)) + min = tmp; + } + int prev = (finder.getPathCost(y, range[0]) - min) / scale; + for (int xx = range[0] + 1; xx < range[1]; xx++) { + int curr = (finder.getPathCost(y, xx) - min) / scale; + int x1 = xx + x0; + g.drawLine(x1-1, showHorizontal-prev, x1, showHorizontal-curr); + prev = curr; + } + } + } // paintPathCostFunctions() + + public void setVisible(boolean v) { + if (parent != null) + parent.setVisible(v); + } // setVisible() + + // interface MouseListener ************************************************* + + public void mouseEntered(MouseEvent e) { requestFocusInWindow(); } + public void mouseExited(MouseEvent e) {} + public void mouseClicked(MouseEvent e) {} + public void mousePressed(MouseEvent e) { mouseDragged(e); } + public void mouseReleased(MouseEvent e) { + if (e.getButton() == MouseEvent.BUTTON1) { + showVertical = -1; + showHorizontal = -1; + } else if (e.getButton() == MouseEvent.BUTTON2) { + if (lastPoint == null) + lastPoint = FixedPoint.newList(0, 0, pm2.frameCount - 1, + pm1.frameCount - 1); + if ((e.getModifiersEx() & InputEvent.SHIFT_DOWN_MASK) != 0) { + for (FixedPoint p = lastPoint.prev; p.prev != null; p= p.prev) { + if ((Math.abs(e.getX() - x0 - p.x) < 5) && + (Math.abs(y0 - e.getY() - p.y) < 5) && + (Math.abs(pathStartX - p.x) < 5) && + (Math.abs(pathStartY - p.y) < 5)) { + p.remove(); + finder.recalculatePathCostMatrix(p.prev.y, p.prev.x, + p.next.y, p.next.x); + updateBackwardPath(false); + updateSmoothedPath(false); + break; + } + } + } else + finaliseCurrentPoint(e.getX() - x0, y0 - e.getY()); + currentX = -1; + } else if (e.getButton() == MouseEvent.BUTTON3) { + // // fPathLength = 0; + // pathStartX = -1; + // pathStartY = -1; + // updateBackwardPath(); + // updateSmoothedPath(); + } + repaint(); + } // mouseReleased() + + // interface MouseMotionListener ******************************************* + + public void mouseMoved(MouseEvent e) {} // mouseMoved() + + /** Button 1: shows best path cost curves for the current line and column; + * Button 2: shows the optimal paths through the current point; + * Button 3: not used + */ + public void mouseDragged(MouseEvent e) { + if ((e.getModifiersEx() & InputEvent.BUTTON1_DOWN_MASK) != 0) { + showHorizontal = e.getY(); + showVertical = e.getX(); + } else if ((e.getModifiersEx() & InputEvent.BUTTON2_DOWN_MASK) != 0) { + // move the current fixed point + if ((e.getModifiersEx() & InputEvent.SHIFT_DOWN_MASK) != 0) { + pathStartX = e.getX() - x0; + pathStartY = y0 - e.getY(); + } + adjustCurrentPoint(e.getX() - x0, y0 - e.getY()); + } else if ((e.getModifiersEx() & InputEvent.BUTTON3_DOWN_MASK) != 0) { + // // fPathLength = 0; + // pathStartX = e.getX() - x0; + // pathStartY = y0 - e.getY(); + // updateBackwardPath(); + // updateSmoothedPath(); + } + repaint(); + } // mouseDragged() + + // interface KeyListener *************************************************** + + public void keyPressed(KeyEvent e) { + processKey(e, true); + } // keyPressed() + + public void keyReleased(KeyEvent e) { + processKey(e, false); + } // keyReleased() + + public void keyTyped(KeyEvent e) {} + + public void processKey(KeyEvent e, boolean down) { + int sign = down? 1:-1; + switch (e.getKeyCode()) { + case KeyEvent.VK_P: + if (down) + PSPrinter.print(this); + break; + case KeyEvent.VK_Q: + setVisible(false); + return; + case KeyEvent.VK_H: + pm1.paused = true; + pm2.paused = false; + break; + case KeyEvent.VK_W: + pm1.paused = true; + pm2.paused = true; + break; + case KeyEvent.VK_V: + pm1.paused = false; + pm2.paused = true; + break; + case KeyEvent.VK_SPACE: + pm1.paused = false; + pm2.paused = false; + break; + case KeyEvent.VK_X: + pm1.paused = true; + pm2.paused = true; + PerformanceMatcher.stop = true; + break; + case KeyEvent.VK_C: + if (down) + showCorrect = !showCorrect; + repaint(); + break; + case KeyEvent.VK_B: + if (down) + showBackwardPath = !showBackwardPath; + repaint(); + break; + case KeyEvent.VK_F: + if (down) + showForwardPath = !showForwardPath; + repaint(); + break; + case KeyEvent.VK_S: + if (down) + showSmoothedPath = !showSmoothedPath; + repaint(); + break; + case KeyEvent.VK_UP: + case KeyEvent.VK_KP_UP: + case KeyEvent.VK_NUMPAD8: + yscroll += -scrollFactor * sign; + break; + case KeyEvent.VK_DOWN: + case KeyEvent.VK_KP_DOWN: + case KeyEvent.VK_NUMPAD2: + yscroll += scrollFactor * sign; + break; + case KeyEvent.VK_LEFT: + case KeyEvent.VK_KP_LEFT: + case KeyEvent.VK_NUMPAD4: + xscroll += -scrollFactor * sign; + break; + case KeyEvent.VK_RIGHT: + case KeyEvent.VK_KP_RIGHT: + case KeyEvent.VK_NUMPAD6: + xscroll += scrollFactor * sign; + break; + case KeyEvent.VK_NUMPAD1: + xscroll += -scrollFactor * sign; + yscroll += scrollFactor * sign; + break; + case KeyEvent.VK_NUMPAD3: + xscroll += scrollFactor * sign; + yscroll += scrollFactor * sign; + break; + case KeyEvent.VK_NUMPAD7: + xscroll += -scrollFactor * sign; + yscroll += -scrollFactor * sign; + break; + case KeyEvent.VK_NUMPAD9: + xscroll += scrollFactor * sign; + yscroll += -scrollFactor * sign; + break; + case KeyEvent.VK_END: + if (down) { + x0 = sz.width - pm2.frameCount; + y0 = pm1.frameCount - 1; + redrawImage(); + } + repaint(); + return; + case KeyEvent.VK_HOME: + if (down) { + x0 = 0; + y0 = sz.height - 1; + redrawImage(); + } + repaint(); + return; + case KeyEvent.VK_PAGE_UP: + xscroll += sz.width / 2 * sign; + yscroll += -sz.height / 2 * sign; + break; + case KeyEvent.VK_PAGE_DOWN: + xscroll += -sz.width / 2 * sign; + yscroll += sz.height / 2 * sign; + break; + } + synchronized (this) { + notify(); + } + } // processKey() + +} // class ScrollingMatrix
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/at/ofai/music/match/WormHandler.java Fri Oct 08 16:02:41 2010 +0100 @@ -0,0 +1,246 @@ +package at.ofai.music.match; + +import java.util.ListIterator; +import java.io.File; +import java.io.BufferedReader; +import java.io.PrintStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.FileNotFoundException; + +import at.ofai.music.worm.Worm; +import at.ofai.music.worm.WormParameters; +import at.ofai.music.worm.WormFile; +import at.ofai.music.util.Event; +import at.ofai.music.util.WormEvent; + +class WormHandler { + + PerformanceMatcher parent, reference; + Worm w; + ScrollingMatrix sm; + double framePeriod; + int hopsPerFrame; + int hopCount, frameCount, prevFrameCount, nextIndex, nextRefIndex; + double energy; + double[] loudness, tempo, refEventTime, eventTime; + double refMaxTime, prevRefMaxTime, prevMaxTime, maxTime; + int[] refFlag, flag; + String[] strFlag, refStrFlag; + AudioFile audioFile; + WormParameters param; + double trackLevel; + static int recalc = 20; // number of frames to recalculate + static final double frameTime = 0.1; // 10 FPS (or nearest mult of hop) + static final int MAX_LENGTH = 36000; // 1 hour + + /** Assumes live input against a reference worm file */ + public WormHandler(PerformanceMatcher pm) { + parent = pm; + reference = pm.otherMatcher; + w = Worm.createInFrame(new String[0]); + sm = null; + hopsPerFrame = (int) Math.round(frameTime / pm.hopTime); + framePeriod = hopsPerFrame * pm.hopTime; + w.setDelay(0); + w.setFramePeriod(framePeriod); + audioFile = new AudioFile(); + loudness = new double[MAX_LENGTH]; + tempo = new double[MAX_LENGTH]; + flag = new int[MAX_LENGTH]; + strFlag = new String[MAX_LENGTH]; + } // constructor + + public void init() { // can't be run in constructor - null pointers + hopCount = 0; + frameCount = 0; + prevFrameCount = 0; + nextRefIndex = 0; + prevRefMaxTime = 0; + prevMaxTime = 0; + energy = 0; + param = new WormParameters(null); + try { + param.read(new BufferedReader(new FileReader( + new File(reference.matchFileName)))); + trackLevel = param.getTrackLevel(); + } catch (FileNotFoundException e) { + System.err.println("No reference file in WormHandler.init()"); + trackLevel = 1; + } catch (IOException e) { + trackLevel = 1; + } + refEventTime = new double[reference.events.size()]; + eventTime = new double[reference.events.size()]; + refFlag = new int[reference.events.size()]; + refStrFlag = new String[reference.events.size()]; + ListIterator<Event> iterator = reference.events.listIterator(); + int i = 0; + int bar = 0, beat = 0; + while (iterator.hasNext()) { + WormEvent e = (WormEvent) iterator.next(); + if (e.flags == 0) + continue; + refEventTime[i] = e.keyDown; + if ((e.flags & WormFile.BAR) != 0) + bar++; + if ((e.flags & WormFile.BEAT) != 0) + beat++; + refStrFlag[i] = bar + ":" + beat + ":0:"; // ignore track level + refFlag[i++] = e.flags; + } + } // init() + + public void setScrollingMatrix(ScrollingMatrix s) { + sm = s; + } // setScrollingMatrix() + + public void addPoint(double hopEnergy) { + energy += hopEnergy; + if (++hopCount >= hopsPerFrame) { + loudness[frameCount] = 80 +10 * Math.log10(energy / hopsPerFrame); + if (loudness[frameCount] < 0) + loudness[frameCount] = 0; + frameCount++; + energy = 0; + hopCount = 0; + } + } // addPoint() + + private int findFirstGE(double t) { + int i = nextRefIndex; + while ((i < eventTime.length) && (refEventTime[i] < t)) + i++; + while ((i > 0) && (refEventTime[i-1] >= t)) + i--; + return i; + } // findFirstGE() + + public void update() { + if (prevFrameCount != frameCount) { + int recalcPoint = prevFrameCount - recalc; + if (recalcPoint < 0) + recalcPoint = 0; + prevMaxTime = recalcPoint * framePeriod; + prevRefMaxTime = audioFile.toReferenceTimeD(prevMaxTime); + if (parent == sm.pm1) + audioFile.setMatch(sm.sPathY, sm.hop1, + sm.sPathX, sm.hop2, sm.sPathLength); + else + audioFile.setMatch(sm.sPathX, sm.hop2, + sm.sPathY, sm.hop1, sm.sPathLength); + maxTime = frameCount * framePeriod; + refMaxTime = audioFile.toReferenceTimeD(maxTime); + double refPrevMaxTime = audioFile.toReferenceTimeD(prevMaxTime); + if (prevRefMaxTime < refPrevMaxTime) { // insert missing events + int start = findFirstGE(prevRefMaxTime); + int stop = findFirstGE(refPrevMaxTime); + int dest = recalcPoint; + nextRefIndex = stop; + if (start < stop) { + System.out.println("Insert: " + start + " " + stop + " " + + prevRefMaxTime + " " + refPrevMaxTime + " " + dest); + } + while ((start < stop--) && (--dest >= 0)) { + if (flag[dest] != 0) + start--; + else { + flag[dest] = refFlag[stop]; + strFlag[dest] = refStrFlag[stop]; + } + System.out.println(start + " " + stop + " " + dest + " " + + flag[dest] + " " + strFlag[dest]); + } + } else if (prevRefMaxTime > refPrevMaxTime) { // delete doubles + int start = findFirstGE(refPrevMaxTime); + int stop = findFirstGE(prevRefMaxTime); + int dest = recalcPoint; + nextRefIndex = start; + if (start < stop) { + System.out.println("Delete: " + start + " " + stop + " " + + refPrevMaxTime + " " + prevRefMaxTime + " " + dest); + } + while ((start < stop) && (--dest >= 0)) { + if (flag[dest] != 0) { + flag[dest] = 0; + stop--; + } + } + String current = "0:0:0:"; + if (dest > 0) + current = strFlag[dest-1]; + if (dest < 0) + dest = 0; + while (dest < recalcPoint) + strFlag[dest++] = current; + } else + nextRefIndex = findFirstGE(refMaxTime); + if (nextRefIndex < eventTime.length) { + eventTime[nextRefIndex] = + audioFile.fromReferenceTimeD(refEventTime[nextRefIndex]); + nextIndex =(int)Math.round(eventTime[nextRefIndex]/framePeriod); + } else + nextIndex = frameCount; + double currentTempo = 0; + String currentLabel = "0:0:0:"; + if (recalcPoint != 0) { + currentTempo = tempo[recalcPoint-1]; + currentLabel = strFlag[recalcPoint-1]; + } + for (int i = recalcPoint; i < frameCount; i++) { + if (i == nextIndex) { + flag[i] = refFlag[nextRefIndex]; + currentLabel = refStrFlag[nextRefIndex]; + double tDiff = 0; + int count = 1; + while ((tDiff == 0) && (nextRefIndex >= count)) + tDiff = eventTime[nextRefIndex] - + eventTime[nextRefIndex - count++]; + if (count > 1) + tDiff /= count - 1; + if (tDiff == 0) + currentTempo = 0; + else + currentTempo = 60 * trackLevel / tDiff; + if (++nextRefIndex < eventTime.length) { + eventTime[nextRefIndex] = audioFile.fromReferenceTimeD( + refEventTime[nextRefIndex]); + nextIndex = (int) Math.round(eventTime[nextRefIndex] / + framePeriod); + if (i >= nextIndex) // multiple flags on a frame + nextIndex = i+1; + } else + nextIndex = frameCount; + } else { + flag[i] = 0; + } + tempo[i] = currentTempo; + strFlag[i] = currentLabel; + } + System.out.println(recalcPoint + " " + frameCount + " " + + nextIndex + " " + nextRefIndex + " " + + maxTime + " " + refMaxTime + " " + + currentTempo + " " + currentLabel); + w.setPoints(tempo, loudness, strFlag, recalcPoint, frameCount); + prevFrameCount = frameCount; + } + } // update() + + public void write(File f, boolean doEdit) { + update(); + param.print(); + param.editParameters(doEdit); + param.print(); + try { + PrintStream out = new PrintStream(f); + param.write(out, frameCount, framePeriod); + for (int i = 0; i < frameCount; i++) + out.printf("%5.3f\t%5.3f\t%5.3f\t%1d\n", + i * framePeriod, tempo[i], loudness[i], flag[i]); + out.close(); + } catch (IOException e) { + System.err.println("Unable to write worm file: " + e); + } + } // write() + +} // class WormHandler