Mercurial > hg > silvet
changeset 185:78212f764251 noteagent
Merge from default branch
| author | Chris Cannam |
|---|---|
| date | Wed, 28 May 2014 14:56:01 +0100 |
| parents | 9b9cdfccbd14 (diff) 59e3cca75b8d (current diff) |
| children | 9d70d687e4eb |
| files | src/Silvet.cpp |
| diffstat | 10 files changed, 1193 insertions(+), 183 deletions(-) [+] |
line wrap: on
line diff
--- a/Makefile.inc Fri May 23 18:17:59 2014 +0100 +++ b/Makefile.inc Wed May 28 14:56:01 2014 +0100 @@ -19,11 +19,30 @@ PLUGIN := silvet$(PLUGIN_EXT) -PLUGIN_HEADERS := $(SRC_DIR)/Silvet.h $(SRC_DIR)/EM.h $(SRC_DIR)/Instruments.h -PLUGIN_SOURCES := $(SRC_DIR)/Silvet.cpp $(SRC_DIR)/EM.cpp $(SRC_DIR)/Instruments.cpp $(SRC_DIR)/libmain.cpp +PLUGIN_HEADERS := \ + $(SRC_DIR)/Silvet.h \ + $(SRC_DIR)/EM.h \ + $(SRC_DIR)/Instruments.h \ + $(SRC_DIR)/AgentFeeder.h \ + $(SRC_DIR)/AgentFeederMono.h \ + $(SRC_DIR)/AgentFeederPoly.h \ + $(SRC_DIR)/AgentHypothesis.h \ + $(SRC_DIR)/NoteHypothesis.h -BQVEC_HEADERS := $(BQVEC_DIR)/Allocators.h $(BQVEC_DIR)/Restrict.h $(BQVEC_DIR)/VectorOps.h -BQVEC_SOURCES := $(BQVEC_DIR)/Allocators.cpp +PLUGIN_SOURCES := \ + $(SRC_DIR)/Silvet.cpp \ + $(SRC_DIR)/EM.cpp \ + $(SRC_DIR)/Instruments.cpp \ + $(SRC_DIR)/NoteHypothesis.cpp \ + $(SRC_DIR)/libmain.cpp + +BQVEC_HEADERS := \ + $(BQVEC_DIR)/Allocators.h \ + $(BQVEC_DIR)/Restrict.h \ + $(BQVEC_DIR)/VectorOps.h + +BQVEC_SOURCES := \ + $(BQVEC_DIR)/Allocators.cpp HEADERS := $(PLUGIN_HEADERS) $(BQVEC_HEADERS) SOURCES := $(PLUGIN_SOURCES) $(BQVEC_SOURCES) @@ -48,7 +67,9 @@ # DO NOT DELETE -src/Silvet.o: src/Silvet.h src/MedianFilter.h src/Instruments.h src/EM.h +src/Silvet.o: src/Silvet.h src/MedianFilter.h src/Instruments.h +src/Silvet.o: src/NoteHypothesis.h src/AgentHypothesis.h src/EM.h +src/Silvet.o: src/AgentFeederPoly.h src/AgentFeeder.h src/Silvet.o: constant-q-cpp/src/dsp/Resampler.h src/EM.o: src/EM.h src/Instruments.h src/Instruments.o: src/Instruments.h data/include/templates.h @@ -58,9 +79,17 @@ src/Instruments.o: data/include/oboe.h data/include/tenorsax.h src/Instruments.o: data/include/violin.h data/include/piano1.h src/Instruments.o: data/include/piano2.h data/include/piano3.h +src/NoteHypothesis.o: src/NoteHypothesis.h src/AgentHypothesis.h +src/NoteHypothesis.o: src/AgentFeederPoly.h src/AgentFeeder.h src/libmain.o: src/Silvet.h src/MedianFilter.h src/Instruments.h +src/libmain.o: src/NoteHypothesis.h src/AgentHypothesis.h bqvec/src/Allocators.o: bqvec/src/Allocators.h bqvec/src/VectorOps.h bqvec/src/Allocators.o: bqvec/src/Restrict.h -src/Silvet.o: src/MedianFilter.h src/Instruments.h +src/Silvet.o: src/MedianFilter.h src/Instruments.h src/NoteHypothesis.h +src/Silvet.o: src/AgentHypothesis.h +src/AgentFeeder.o: src/AgentHypothesis.h +src/AgentFeederMono.o: src/AgentFeeder.h src/AgentHypothesis.h +src/AgentFeederPoly.o: src/AgentFeeder.h src/AgentHypothesis.h +src/NoteHypothesis.o: src/AgentHypothesis.h bqvec/src/Allocators.o: bqvec/src/VectorOps.h bqvec/src/Restrict.h bqvec/src/VectorOps.o: bqvec/src/Restrict.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/AgentFeeder.h Wed May 28 14:56:01 2014 +0100 @@ -0,0 +1,31 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Silvet + + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2012 Chris Cannam. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef AGENT_FEEDER_H +#define AGENT_FEEDER_H + +#include "AgentHypothesis.h" + +class AgentFeeder +{ +public: + virtual void feed(AgentHypothesis::Observation) = 0; + virtual void finish() = 0; + + virtual ~AgentFeeder() { } +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/AgentFeederMono.h Wed May 28 14:56:01 2014 +0100 @@ -0,0 +1,161 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Silvet + + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2012 Chris Cannam. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef AGENT_FEEDER_MONO_H +#define AGENT_FEEDER_MONO_H + +#include "AgentFeeder.h" + +//#define DEBUG_FEEDER 1 + +/** + * Take a series of observations or estimates (one at a time) and feed + * them to a set of agent hypotheses, creating a new candidate agent + * for each observation and also testing the observation against the + * existing set of hypotheses. + * + * One satisfied hypothesis is considered to be "accepted" at any + * moment (that is, the earliest contemporary hypothesis to have + * become satisfied). The series of accepted and completed hypotheses + * from construction to the present time can be queried through + * getAcceptedHypotheses(). + * + * Call feed() to provide a new observation. Call finish() when all + * observations have been provided. The set of hypotheses returned by + * getAcceptedHypotheses() will not be complete unless finish() has + * been called. + */ +template <typename Hypothesis> +class AgentFeederMono : public AgentFeeder +{ +public: + AgentFeederMono() { } + + typedef std::set<Hypothesis> Hypotheses; + + virtual void feed(AgentHypothesis::Observation o) { + +#ifdef DEBUG_FEEDER + std::cerr << "feed: have observation [value = " << o.value << ", time = " << o.time << "]" << std::endl; +#endif + + if (!m_current.accept(o)) { + + if (m_current.getState() == Hypothesis::Expired) { + m_accepted.insert(m_current); +#ifdef DEBUG_FEEDER + std::cerr << "current has expired, pushing to accepted" << std::endl; +#endif + } + + bool swallowed = false; + +#ifdef DEBUG_FEEDER + std::cerr << "not swallowed by current" << std::endl; +#endif + + Hypotheses newCandidates; + + for (typename Hypotheses::iterator i = m_candidates.begin(); + i != m_candidates.end(); ++i) { + + Hypothesis h = *i; + + if (swallowed) { + + // don't offer: each observation can only belong to one + // satisfied hypothesis + newCandidates.insert(h); + + } else { + + if (h.accept(o)) { +#ifdef DEBUG_FEEDER + std::cerr << "accepted, state is " << h.getState() << std::endl; +#endif + if (h.getState() == Hypothesis::Satisfied) { + + swallowed = true; + + if (m_current.getState() == Hypothesis::Expired || + m_current.getState() == Hypothesis::Rejected) { +#ifdef DEBUG_FEEDER + std::cerr << "current has ended, updating from candidate" << std::endl; +#endif + m_current = h; + } else { + newCandidates.insert(h); + } + + } else { + newCandidates.insert(h); + } + } + } + } + + if (!swallowed) { + Hypothesis h; + h.accept(o); // must succeed, as h is new + newCandidates.insert(h); +#ifdef DEBUG_FEEDER + std::cerr << "not swallowed, creating new hypothesis" << std::endl; +#endif + } + + // reap rejected/expired hypotheses from candidates list, + // and assign back to m_candidates + + m_candidates.clear(); + + for (typename Hypotheses::const_iterator i = newCandidates.begin(); + i != newCandidates.end(); ++i) { + Hypothesis h = *i; + if (h.getState() != Hypothesis::Rejected && + h.getState() != Hypothesis::Expired) { + m_candidates.insert(h); + } else { +#ifdef DEBUG_FEEDER + std::cerr << "reaping a candidate" << std::endl; +#endif + } + } + } +#ifdef DEBUG_FEEDER + std::cerr << "have " << m_candidates.size() << " candidates" << std::endl; +#endif + } + + virtual void finish() { + if (m_current.getState() == Hypothesis::Satisfied) { +#ifdef DEBUG_FEEDER + std::cerr << "finish: current is satisfied, pushing to accepted" << std::endl; +#endif + m_accepted.insert(m_current); + } + } + + Hypotheses getAcceptedHypotheses() const { + return m_accepted; + } + +private: + Hypotheses m_candidates; + Hypothesis m_current; + Hypotheses m_accepted; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/AgentFeederPoly.h Wed May 28 14:56:01 2014 +0100 @@ -0,0 +1,284 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Silvet + + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2012 Chris Cannam. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef AGENT_FEEDER_POLY_H +#define AGENT_FEEDER_POLY_H + +#include "AgentFeeder.h" + +#include <cassert> +#include <stdexcept> + +#define DEBUG_FEEDER 1 + +/** + * Take a series of observations or estimates (one at a time) and feed + * them to a set of agent hypotheses, creating a new candidate agent + * for each observation and also testing the observation against the + * existing set of hypotheses. + * + *!!! -- todo: document poly-ness of it + * + * Call feed() to provide a new observation. Call finish() when all + * observations have been provided. The set of hypotheses returned by + * getAcceptedHypotheses() will not be complete unless finish() has + * been called. + */ +template <typename Hypothesis> +class AgentFeederPoly : public AgentFeeder +{ +private: + typedef std::vector<Hypothesis> Hypotheses; + + struct State { + Hypotheses provisional; + Hypotheses satisfied; + Hypotheses completed; + }; + State m_state; + +public: + AgentFeederPoly() { } + + virtual void feed(AgentHypothesis::Observation o) { +#ifdef DEBUG_FEEDER + std::cerr << "\nfeed: have observation [value = " << o.value << ", time = " << o.time << "]" << std::endl; +#endif + + m_state = update(m_state, o); + } + + virtual void finish() { +#ifdef DEBUG_FEEDER + std::cerr << "finish: satisfied count == " << m_state.satisfied.size() << std::endl; +#endif + for (typename Hypotheses::const_iterator i = m_state.satisfied.begin(); + i != m_state.satisfied.end(); ++i) { + m_state.completed.push_back(*i); + } + } + + std::set<Hypothesis> getAcceptedHypotheses() const { + std::set<Hypothesis> hs; + for (typename Hypotheses::const_iterator i = m_state.completed.begin(); + i != m_state.completed.end(); ++i) { + hs.insert(*i); + } + return hs; + } + +private: + State update(State s, AgentHypothesis::Observation o) { + + /* + An observation can "belong" to any number of provisional + hypotheses, but only to one satisfied hypothesis. + + A new observation is first offered to the hypotheses that + have already been satisfied. If one of these accepts it, it + gets to keep it and no other hypothesis can have it. + + Any observation not accepted by a hypothesis in satisfied + state is then offered to the provisional hypotheses; any + number of these may accept it. Also, every observation that + belongs to no satisfied hypothesis is used as the first + observation in its own new hypothesis (regardless of how + many other provisional hypotheses have accepted it). + + When a hypothesis subsequently becomes satisfied, all other + provisional hypotheses containing any of its observations + must be discarded. + */ + + State newState; + + // We only ever add to the completed hypotheses, never remove + // anything from them. But we may remove from provisional (if + // rejected or transferred to satisfied) and satisfied (when + // completed). + + newState.completed = s.completed; + + bool swallowed = false; + + for (typename Hypotheses::iterator i = s.satisfied.begin(); + i != s.satisfied.end(); ++i) { + + Hypothesis h = *i; + + assert(h.getState() == Hypothesis::Satisfied); + + if (swallowed) { + + // An observation that has already been accepted by a + // hypothesis cannot be offered to any other, because + // it can only belong to one satisfied hypothesis. Any + // subsequent satisfied hypotheses are retained + // unchanged in our updated state. We can't test them + // for expiry, because the state is only updated when + // accept() is called. + //!!! That looks like a limitation in the Hypothesis API + newState.satisfied.push_back(h); + + } else { // !swallowed + + if (h.accept(o)) { +#ifdef DEBUG_FEEDER + std::cerr << "accepted by satisfied hypothesis " << &(*i) << ", state is " << h.getState() << std::endl; +#endif + swallowed = true; + newState.satisfied.push_back(h); + } else if (h.getState() == Hypothesis::Expired) { + newState.completed.push_back(h); + } else { + newState.satisfied.push_back(h); + } + } + } + + if (swallowed) { + +#ifdef DEBUG_FEEDER + std::cerr << "was swallowed by satisfied hypothesis" << std::endl; +#endif + // no provisional hypotheses have become satisfied, no new + // ones have been introduced + newState.provisional = s.provisional; + + } else { + +#ifdef DEBUG_FEEDER + std::cerr << "remained unswallowed by " << newState.satisfied.size() << " satisfied hypotheses" << std::endl; +#endif + + // not swallowed by any satisfied hypothesis + + Hypothesis promoted; + + for (typename Hypotheses::iterator i = s.provisional.begin(); + i != s.provisional.end(); ++i) { + + Hypothesis h = *i; + + assert(h.getState() == Hypothesis::Provisional); + + // can only have one satisfied hypothesis for each + // observation, so try this only if promoted has not been + // set to something else yet + if (promoted == Hypothesis() && + h.accept(o) && + h.getState() == Hypothesis::Satisfied) { + newState.satisfied.push_back(h); +#ifdef DEBUG_FEEDER + std::cerr << "promoting a hypothesis to satisfied, have " << newState.satisfied.size() << " satisfied now" << std::endl; +#endif + promoted = h; + } else if (h.getState() != Hypothesis::Rejected) { + newState.provisional.push_back(h); + } + } + + if (promoted == Hypothesis()) { + + // No provisional hypothesis has become satisfied + + Hypothesis h; + h.accept(o); + + if (h.getState() == Hypothesis::Provisional) { + newState.provisional.push_back(h); + } else if (h.getState() == Hypothesis::Satisfied) { + newState.satisfied.push_back(h); + } + +#ifdef DEBUG_FEEDER + std::cerr << "update: new hypothesis of state " << h.getState() << ", provisional count -> " << newState.provisional.size() << std::endl; +#endif + } else { + +#ifdef DEBUG_FEEDER + std::cerr << "a hypothesis became satisfied, reaping its observations" << std::endl; +#endif + newState = reap(newState); + } + } + + return newState; + } + + State reap(State s) { + + // "When a hypothesis subsequently becomes satisfied, all + // other provisional hypotheses containing any of its + // observations must be discarded." + + if (s.provisional.empty()) return s; + + int reaped = 0; + + Hypotheses prior = s.provisional; + s.provisional = Hypotheses(); + + for (typename Hypotheses::const_iterator hi = prior.begin(); + hi != prior.end(); ++hi) { + + const AgentHypothesis::Observations obs = + hi->getAcceptedObservations(); + + bool keep = true; + + for (AgentHypothesis::Observations::const_iterator oi = obs.begin(); + oi != obs.end(); ++oi) { + + for (typename Hypotheses::const_iterator si = s.satisfied.end(); + si != s.satisfied.begin(); ) { + + --si; + + const AgentHypothesis::Observations sobs = + si->getAcceptedObservations(); + + if (sobs.find(*oi) != sobs.end()) { + keep = false; + break; + } + } + + if (!keep) { + break; + } + } + + if (keep) { + s.provisional.push_back(*hi); + } else { + ++reaped; + } + } + +#ifdef DEBUG_FEEDER + std::cerr << "reap: have " + << s.satisfied.size() << " satisfied, " + << s.provisional.size() << " provisional, " + << s.completed.size() << " completed, reaped " + << reaped << std::endl; +#endif + + return s; + } +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/AgentHypothesis.h Wed May 28 14:56:01 2014 +0100 @@ -0,0 +1,146 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Silvet + + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2012 Chris Cannam. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef AGENT_HYPOTHESIS_H +#define AGENT_HYPOTHESIS_H + +#include "vamp-sdk/RealTime.h" + +#include <set> +#include <map> + +/** + * An agent used to test an incoming series of timed observations or + * estimates to see whether they fit a consistent single-object + * relationship. + * + * A freshly constructed hypothesis object should be in New state and + * should accept any observation. + */ + +class AgentHypothesis +{ +public: + virtual ~AgentHypothesis() { } + + enum State { + + /// Just constructed, will provisionally accept any observation + New, + + /// Accepted at least one observation, but not enough evidence to satisfy + Provisional, + + /// Could not find enough consistency in offered observations + Rejected, + + /// Have accepted enough consistent observations to satisfy hypothesis + Satisfied, + + /// Have been satisfied, but evidence has now changed: we're done + Expired + }; + + struct Observation { + + Observation() : value(0), time(), confidence(1) { } + + Observation(double _f, Vamp::RealTime _t, double _c) : + value(_f), time(_t), confidence(_c) { } + + bool operator==(const Observation &o) const { + return o.value == value && o.time == time && o.confidence == confidence; + } + bool operator<(const Observation &o) const { + return + (time < o.time || + (time == o.time && value < o.value) || + (time == o.time && value == o.value && confidence < o.confidence)); + } + + double value; + Vamp::RealTime time; + double confidence; + }; + typedef std::set<Observation> Observations; + + /** + * Test the given observation to see whether it is consistent with + * this hypothesis, and adjust the hypothesis' internal state + * accordingly. If the observation is not inconsistent with the + * hypothesis, return true. + *!!! should be called e.g. test? + */ + virtual bool accept(Observation) = 0; + + /** + * Return the current state of this hypothesis. + */ + virtual State getState() const = 0; + + /** + * If the hypothesis has been satisfied (i.e. is in Satisfied or + * Expired state), return the set of observations that it + * accepted. Otherwise return an empty set + */ + virtual Observations getAcceptedObservations() const = 0; + + /** + * Convert the given set of accepted hypotheses (of type + * subclassed from AgentHypothesis) into a flattened set of their + * accepted observations. + * + * That is, only one is included for at any given moment, so in + * the case of overlapping hypotheses, the observations for the + * earlier are taken until the next hypothesis begins and then the + * latter's observations begin instead. + * + * (If there are gaps between hypotheses, the gaps remain in the + * output.) + */ + template <typename HypothesisType> + static Observations flatten(const std::set<HypothesisType> &agents) { + + typedef typename std::set<HypothesisType>::const_iterator Itr; + Observations flattened; + + if (agents.empty()) return flattened; + Observations obs = agents.begin()->getAcceptedObservations(); + + for (Itr i = agents.begin(); i != agents.end(); ++i) { + + Itr j = i; + ++j; + + Observations nextObs; + if (j != agents.end()) nextObs = j->getAcceptedObservations(); + + for (Observations::const_iterator i = obs.begin(); + i != obs.end(); ++i) { + if (!nextObs.empty() && i->time >= nextObs.begin()->time) { + break; + } + flattened.insert(*i); + } + + obs = nextObs; + } + + return flattened; + } +}; + +#endif
--- a/src/MedianFilter.h Fri May 23 18:17:59 2014 +0100 +++ b/src/MedianFilter.h Wed May 28 14:56:01 2014 +0100 @@ -1,16 +1,17 @@ /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ /* - QM DSP Library + Silvet - Centre for Digital Music, Queen Mary, University of London. - This file Copyright 2010 Chris Cannam. + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2010 Chris Cannam. - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. See the file - COPYING included with this distribution for more information. + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. */ #ifndef MEDIAN_FILTER_H
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/NoteHypothesis.cpp Wed May 28 14:56:01 2014 +0100 @@ -0,0 +1,299 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Silvet + + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2012 Chris Cannam. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "NoteHypothesis.h" +#include "AgentFeederPoly.h" + +#include <cmath> +#include <cassert> + +#include <map> +#include <algorithm> + +using Vamp::RealTime; + +using std::cerr; +using std::endl; + +#define DEBUG_NOTE_HYPOTHESIS 1 + +NoteHypothesis::NoteHypothesis() +{ + m_state = New; +} + +NoteHypothesis::~NoteHypothesis() +{ +} + +bool +NoteHypothesis::isWithinTolerance(Observation s) const +{ + if (m_pending.empty()) { + return true; + } + + // check we are within a relatively close tolerance of the last + // candidate + Observations::const_iterator i = m_pending.end(); + --i; + Observation last = *i; + double r = s.value / last.value; + int cents = lrint(1200.0 * (log(r) / log(2.0))); +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "isWithinTolerance: this " << s.value << " is " << cents + << " cents from prior " << last.value << endl; +#endif + if (cents < -60 || cents > 60) return false; + + // and within a slightly bigger tolerance of the current mean + double meanFreq = getMeanFrequency(); + r = s.value / meanFreq; + cents = lrint(1200.0 * (log(r) / log(2.0))); +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "isWithinTolerance: this " << s.value << " is " << cents + << " cents from mean " << meanFreq << endl; +#endif + if (cents < -80 || cents > 80) return false; + + return true; +} + +bool +NoteHypothesis::isOutOfDateFor(Observation s) const +{ + if (m_pending.empty()) return false; + + Observations::const_iterator i = m_pending.end(); + --i; + Observation last = *i; + +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "isOutOfDateFor: this " << s.time << " is " + << (s.time - last.time) << " from last " << last.time + << " (threshold " << RealTime::fromMilliseconds(40) << ")" + << endl; +#endif + + return ((s.time - last.time) > RealTime::fromMilliseconds(40)); +} + +bool +NoteHypothesis::isSatisfied() const +{ + if (m_pending.empty()) return false; + + double meanConfidence = 0.0; + for (Observations::const_iterator i = m_pending.begin(); + i != m_pending.end(); ++i) { + meanConfidence += i->confidence; + } + meanConfidence /= m_pending.size(); + + //!!! surely this depends on the hop size? + int lengthRequired = 100; + if (meanConfidence > 0.0) { + lengthRequired = int(2.0 / meanConfidence + 0.5); + } + //!!! + lengthRequired = lengthRequired / 2; + if (lengthRequired < 1) lengthRequired = 1; + +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "meanConfidence " << meanConfidence << ", lengthRequired " << lengthRequired << endl; +#endif + + return ((int)m_pending.size() > lengthRequired); +} + +static void printState(NoteHypothesis::State s) +{ + switch (s) { + case NoteHypothesis::New: cerr << "New"; break; + case NoteHypothesis::Provisional: cerr << "Provisional"; break; + case NoteHypothesis::Rejected: cerr << "Rejected"; break; + case NoteHypothesis::Satisfied: cerr << "Satisfied"; break; + case NoteHypothesis::Expired: cerr << "Expired"; break; + } +} + +bool +NoteHypothesis::accept(Observation s) +{ + bool accept = false; + +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "NoteHypothesis[" << this << "]::accept (value " << s.value << ", time " << s.time << ", confidence " << s.confidence << "): state "; + printState(m_state); + cerr << "..." << endl; +#endif + + static double negligibleConfidence = 0.0001; + + if (s.confidence < negligibleConfidence) { + // avoid piling up a lengthy sequence of estimates that are + // all acceptable but are in total not enough to cause us to + // be satisfied + if (m_state == New) { + m_state = Rejected; + } + return false; + } + + switch (m_state) { + + case New: + m_state = Provisional; + accept = true; + break; + + case Provisional: + if (isOutOfDateFor(s)) { + m_state = Rejected; + } else if (isWithinTolerance(s)) { + accept = true; + } + break; + + case Satisfied: + if (isOutOfDateFor(s)) { + m_state = Expired; + } else if (isWithinTolerance(s)) { + accept = true; + } + break; + + case Rejected: + break; + + case Expired: + break; + } + + if (accept) { +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "... accepting" << endl; +#endif + m_pending.insert(s); + if (m_state == Provisional && isSatisfied()) { + m_state = Satisfied; + } + } else { +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "... not accepting" << endl; +#endif + } + +#ifdef DEBUG_NOTE_HYPOTHESIS + cerr << "... -> "; + printState(m_state); + cerr << " (pending: " << m_pending.size() << ")" << endl; +#endif + + return accept; +} + +NoteHypothesis::State +NoteHypothesis::getState() const +{ + return m_state; +} + +NoteHypothesis::Observations +NoteHypothesis::getAcceptedObservations() const +{ + if (m_state == Satisfied || m_state == Expired) { + return m_pending; + } else { + return Observations(); + } +} + +double +NoteHypothesis::getMedianFrequency() const +{ + if (m_pending.empty()) return 0.0; + std::vector<double> freqs; + for (Observations::const_iterator i = m_pending.begin(); + i != m_pending.end(); ++i) { + freqs.push_back(i->value); + } + std::sort(freqs.begin(), freqs.end()); + return freqs[freqs.size()/2]; +} + +double +NoteHypothesis::getMeanFrequency() const +{ + double acc = 0.0; + if (m_pending.empty()) return acc; + for (Observations::const_iterator i = m_pending.begin(); + i != m_pending.end(); ++i) { + acc += i->value; + } + acc /= m_pending.size(); + return acc; +} + +double +NoteHypothesis::getMedianConfidence() const +{ + if (m_pending.empty()) return 0.0; + std::vector<double> confs; + for (Observations::const_iterator i = m_pending.begin(); + i != m_pending.end(); ++i) { + confs.push_back(i->confidence); + } + std::sort(confs.begin(), confs.end()); + return confs[confs.size()/2]; +} + +NoteHypothesis::Note +NoteHypothesis::getAveragedNote() const +{ + Note n; + + n.time = getStartTime(); + n.duration = getDuration(); + n.freq = getMedianFrequency(); + n.confidence = getMedianConfidence(); + + return n; +} + +RealTime +NoteHypothesis::getStartTime() const +{ + if (!(m_state == Satisfied || m_state == Expired)) { + return RealTime::zeroTime; + } else { + return m_pending.begin()->time; + } +} + +RealTime +NoteHypothesis::getDuration() const +{ +//!!! test this! it is wrong + if (!(m_state == Satisfied || m_state == Expired)) { + return RealTime::zeroTime; + } else { + RealTime start = m_pending.begin()->time; + Observations::const_iterator i = m_pending.end(); + --i; + return i->time - start; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/NoteHypothesis.h Wed May 28 14:56:01 2014 +0100 @@ -0,0 +1,130 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Silvet + + A Vamp plugin for note transcription. + Centre for Digital Music, Queen Mary University of London. + This file Copyright 2012 Chris Cannam. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef NOTE_HYPOTHESIS_H +#define NOTE_HYPOTHESIS_H + +#include "AgentHypothesis.h" + +#include <set> +#include <vector> + +/** + * An AgentHypothesis which tests a series of instantaneous pitch + * estimates to see whether they fit a single-note relationship. + * Contains rules specific to testing note pitch and timing. + */ + +class NoteHypothesis : public AgentHypothesis +{ +public: + /** + * Construct an empty hypothesis. This will be in New state and + * will provisionally accept any estimate. + */ + NoteHypothesis(); + + /** + * Destroy the hypothesis + */ + ~NoteHypothesis(); + + virtual bool accept(Observation); + virtual State getState() const; + virtual Observations getAcceptedObservations() const; + + struct Note { + Note() : freq(0), time(), duration(), confidence(1.0) { } + Note(double _f, Vamp::RealTime _t, Vamp::RealTime _d, double _i) : + freq(_f), time(_t), duration(_d), confidence(_i) { } + bool operator==(const Note &e) const { + return e.freq == freq && e.time == time && + e.duration == duration && e.confidence == confidence; + } + double freq; + Vamp::RealTime time; + Vamp::RealTime duration; + double confidence; + }; + + /** + * Return the mean frequency of the accepted observations + */ + double getMeanFrequency() const; + + /** + * Return the median frequency of the accepted observations + */ + double getMedianFrequency() const; + + /** + * Return the median confidence of the accepted observations + */ + double getMedianConfidence() const; + + /** + * Return a single note roughly matching this hypothesis + */ + Note getAveragedNote() const; + + /** + * Return the time of the first accepted observation + */ + Vamp::RealTime getStartTime() const; + + /** + * Return the difference between the start time and the end of the + * final accepted observation + */ + Vamp::RealTime getDuration() const; + + //!!! + bool operator==(const NoteHypothesis &other) const { + return m_state == other.m_state && m_pending == other.m_pending; + } + + bool operator<(const NoteHypothesis &other) const { + if (getStartTime() != other.getStartTime()) { + return getStartTime() < other.getStartTime(); + } else if (m_state != other.m_state) { + return m_state < other.m_state; + } else if (m_pending.size() != other.m_pending.size()) { + return m_pending.size() < other.m_pending.size(); + } else { + Observations::const_iterator i = m_pending.begin(); + Observations::const_iterator j = other.m_pending.begin(); + while (i != m_pending.end()) { + if (*i == *j) { + ++i; + ++j; + } else { + return *i < *j; + } + } + return false; + } + } + +private: + bool isWithinTolerance(Observation) const; + bool isOutOfDateFor(Observation) const; + bool isSatisfied() const; + + State m_state; + Observations m_pending; +}; + +#endif
--- a/src/Silvet.cpp Fri May 23 18:17:59 2014 +0100 +++ b/src/Silvet.cpp Wed May 28 14:56:01 2014 +0100 @@ -19,6 +19,9 @@ #include <cq/CQSpectrogram.h> #include "MedianFilter.h" +#include "AgentFeederPoly.h" +#include "NoteHypothesis.h" + #include "constant-q-cpp/src/dsp/Resampler.h" #include <vector> @@ -42,7 +45,8 @@ m_hqMode(true), m_fineTuning(false), m_instrument(0), - m_colsPerSec(50) + m_colsPerSec(50), + m_agentFeeder(0) { } @@ -53,6 +57,7 @@ for (int i = 0; i < (int)m_postFilter.size(); ++i) { delete m_postFilter[i]; } + delete m_agentFeeder; } string @@ -353,6 +358,7 @@ { delete m_resampler; delete m_cq; + delete m_agentFeeder; if (m_inputSampleRate != processingSampleRate) { m_resampler = new Resampler(m_inputSampleRate, processingSampleRate); @@ -393,15 +399,18 @@ for (int i = 0; i < m_instruments[0].templateNoteCount; ++i) { m_postFilter.push_back(new MedianFilter<double>(3)); } - m_pianoRoll.clear(); - m_columnCount = 0; + + m_columnCountIn = 0; + m_columnCountOut = 0; m_startTime = RealTime::zeroTime; + + m_agentFeeder = new AgentFeederPoly<NoteHypothesis>(); } Silvet::FeatureSet Silvet::process(const float *const *inputBuffers, Vamp::RealTime timestamp) { - if (m_columnCount == 0) { + if (m_columnCountIn == 0) { m_startTime = timestamp; } @@ -423,7 +432,17 @@ Silvet::getRemainingFeatures() { Grid cqout = m_cq->getRemainingOutput(); + FeatureSet fs = transcribe(cqout); + + m_agentFeeder->finish(); + + FeatureList noteFeatures = obtainNotes(); + for (FeatureList::const_iterator fi = noteFeatures.begin(); + fi != noteFeatures.end(); ++fi) { + fs[m_notesOutputNo].push_back(*fi); + } + return fs; } @@ -453,7 +472,7 @@ //!!! pitches or notes? [terminology] Grid localPitches(width, vector<double>(pack.templateNoteCount, 0.0)); - bool wantShifts = m_hqMode && m_fineTuning; + bool wantShifts = m_hqMode; int shiftCount = 1; if (wantShifts) { shiftCount = pack.templateMaxShift * 2 + 1; @@ -515,16 +534,13 @@ for (int j = 0; j < pack.templateNoteCount; ++j) { m_postFilter[j]->push(0.0); } - m_pianoRoll.push_back(map<int, double>()); - if (wantShifts) { - m_pianoRollShifts.push_back(map<int, int>()); - } continue; } - postProcess(localPitches[i], localBestShifts[i], wantShifts); + postProcess(localPitches[i], localBestShifts[i], + wantShifts, shiftCount); - FeatureList noteFeatures = noteTrack(shiftCount); + FeatureList noteFeatures = obtainNotes(); for (FeatureList::const_iterator fi = noteFeatures.begin(); fi != noteFeatures.end(); ++fi) { @@ -558,13 +574,13 @@ for (int i = 0; i < width; ++i) { - if (m_columnCount < latentColumns) { - ++m_columnCount; + if (m_columnCountIn < latentColumns) { + ++m_columnCountIn; continue; } - int prevSampleNo = (m_columnCount - 1) * m_cq->getColumnHop(); - int sampleNo = m_columnCount * m_cq->getColumnHop(); + int prevSampleNo = (m_columnCountIn - 1) * m_cq->getColumnHop(); + int sampleNo = m_columnCountIn * m_cq->getColumnHop(); bool select = (sampleNo / spacing != prevSampleNo / spacing); @@ -613,7 +629,7 @@ out.push_back(outCol); } - ++m_columnCount; + ++m_columnCountIn; } return out; @@ -622,7 +638,8 @@ void Silvet::postProcess(const vector<double> &pitches, const vector<int> &bestShifts, - bool wantShifts) + bool wantShifts, + int shiftCount) { const InstrumentPack &pack = m_instruments[m_instrument]; @@ -633,172 +650,82 @@ filtered.push_back(m_postFilter[j]->get()); } - // Threshold for level and reduce number of candidate pitches + double threshold = 1; //!!! pack.levelThreshold - typedef std::multimap<double, int> ValueIndexMap; - - ValueIndexMap strengths; + double columnDuration = 1.0 / m_colsPerSec; + int postFilterLatency = int(m_postFilter[0]->getSize() / 2); + RealTime t = RealTime::fromSeconds + (columnDuration * (m_columnCountOut - postFilterLatency) + 0.02); for (int j = 0; j < pack.templateNoteCount; ++j) { + double strength = filtered[j]; - if (strength < pack.levelThreshold) continue; - strengths.insert(ValueIndexMap::value_type(strength, j)); + if (strength < threshold) { + continue; + } + + double freq; + if (wantShifts) { + freq = noteFrequency(j, bestShifts[j], shiftCount); + } else { + freq = noteFrequency(j, 0, shiftCount); + } + + double confidence = strength / 50.0; //!!!??? + if (confidence > 1.0) confidence = 1.0; + + AgentHypothesis::Observation obs(freq, t, confidence); + m_agentFeeder->feed(obs); } - ValueIndexMap::const_iterator si = strengths.end(); - - map<int, double> active; - map<int, int> activeShifts; - - while (int(active.size()) < pack.maxPolyphony && si != strengths.begin()) { - - --si; - - double strength = si->first; - int j = si->second; - - active[j] = strength; - - if (wantShifts) { - activeShifts[j] = bestShifts[j]; - } - } - - m_pianoRoll.push_back(active); - - if (wantShifts) { - m_pianoRollShifts.push_back(activeShifts); - } + m_columnCountOut ++; } Vamp::Plugin::FeatureList -Silvet::noteTrack(int shiftCount) +Silvet::obtainNotes() { - // Minimum duration pruning, and conversion to notes. We can only - // report notes that have just ended (i.e. that are absent in the - // latest active set but present in the prior set in the piano - // roll) -- any notes that ended earlier will have been reported - // already, and if they haven't ended, we don't know their - // duration. - - int width = m_pianoRoll.size() - 1; - - const map<int, double> &active = m_pianoRoll[width]; - - double columnDuration = 1.0 / m_colsPerSec; - - // only keep notes >= 100ms or thereabouts - int durationThreshold = floor(0.1 / columnDuration); // columns - if (durationThreshold < 1) durationThreshold = 1; - FeatureList noteFeatures; - if (width < durationThreshold + 1) { + typedef AgentFeederPoly<NoteHypothesis> NoteFeeder; + + NoteFeeder *feeder = dynamic_cast<NoteFeeder *>(m_agentFeeder); + + if (!feeder) { + cerr << "INTERNAL ERROR: Feeder is not a poly-note-hypothesis-feeder!" + << endl; return noteFeatures; } - - //!!! try: repeated note detection? (look for change in first derivative of the pitch matrix) - for (map<int, double>::const_iterator ni = m_pianoRoll[width-1].begin(); - ni != m_pianoRoll[width-1].end(); ++ni) { + std::set<NoteHypothesis> hh = feeder->getAcceptedHypotheses(); - int note = ni->first; - - if (active.find(note) != active.end()) { - // the note is still playing - continue; + //!!! inefficient + for (std::set<NoteHypothesis>::const_iterator hi = hh.begin(); + hi != hh.end(); ++hi) { + + NoteHypothesis h(*hi); + + if (m_emitted.find(h) != m_emitted.end()) { + continue; // already returned this one } - // the note was playing but just ended - int end = width; - int start = end-1; + m_emitted.insert(h); - while (m_pianoRoll[start].find(note) != m_pianoRoll[start].end()) { - --start; - } - ++start; + NoteHypothesis::Note n = h.getAveragedNote(); - if ((end - start) < durationThreshold) { - continue; - } + int velocity = n.confidence * 127; + if (velocity > 127) velocity = 127; - emitNote(start, end, note, shiftCount, noteFeatures); + Feature f; + f.hasTimestamp = true; + f.hasDuration = true; + f.timestamp = n.time; + f.duration = n.duration; + f.values.clear(); + f.values.push_back(n.freq); + f.values.push_back(velocity); +// f.label = noteName(note, partShift, shiftCount); + noteFeatures.push_back(f); } -// cerr << "returning " << noteFeatures.size() << " complete note(s) " << endl; - return noteFeatures; } - -void -Silvet::emitNote(int start, int end, int note, int shiftCount, - FeatureList ¬eFeatures) -{ - int partStart = start; - int partShift = 0; - int partVelocity = 0; - - Feature f; - f.hasTimestamp = true; - f.hasDuration = true; - - double columnDuration = 1.0 / m_colsPerSec; - int postFilterLatency = int(m_postFilter[0]->getSize() / 2); - int partThreshold = floor(0.05 / columnDuration); - - for (int i = start; i != end; ++i) { - - double strength = m_pianoRoll[i][note]; - - int shift = 0; - - if (shiftCount > 1) { - - shift = m_pianoRollShifts[i][note]; - - if (i == partStart) { - partShift = shift; - } - - if (i > partStart + partThreshold && shift != partShift) { - -// cerr << "i = " << i << ", partStart = " << partStart << ", shift = " << shift << ", partShift = " << partShift << endl; - - // pitch has changed, emit an intermediate note - f.timestamp = RealTime::fromSeconds - (columnDuration * (partStart - postFilterLatency) + 0.02); - f.duration = RealTime::fromSeconds - (columnDuration * (i - partStart)); - f.values.clear(); - f.values.push_back - (noteFrequency(note, partShift, shiftCount)); - f.values.push_back(partVelocity); - f.label = noteName(note, partShift, shiftCount); - noteFeatures.push_back(f); - partStart = i; - partShift = shift; - partVelocity = 0; - } - } - - int v = strength * 2; - if (v > 127) v = 127; - - if (v > partVelocity) { - partVelocity = v; - } - } - - if (end >= partStart + partThreshold) { - f.timestamp = RealTime::fromSeconds - (columnDuration * (partStart - postFilterLatency) + 0.02); - f.duration = RealTime::fromSeconds - (columnDuration * (end - partStart)); - f.values.clear(); - f.values.push_back - (noteFrequency(note, partShift, shiftCount)); - f.values.push_back(partVelocity); - f.label = noteName(note, partShift, shiftCount); - noteFeatures.push_back(f); - } -}
--- a/src/Silvet.h Fri May 23 18:17:59 2014 +0100 +++ b/src/Silvet.h Wed May 28 14:56:01 2014 +0100 @@ -24,6 +24,7 @@ #include "MedianFilter.h" #include "Instruments.h" +#include "NoteHypothesis.h" using std::string; using std::vector; @@ -32,6 +33,7 @@ class Resampler; class CQSpectrogram; +class AgentFeeder; class Silvet : public Vamp::Plugin { @@ -84,19 +86,18 @@ typedef vector<vector<double> > Grid; vector<MedianFilter<double> *> m_postFilter; - vector<map<int, double> > m_pianoRoll; - vector<map<int, int> > m_pianoRollShifts; + + AgentFeeder *m_agentFeeder; + std::set<NoteHypothesis> m_emitted; Grid preProcess(const Grid &); void postProcess(const vector<double> &pitches, const vector<int> &bestShifts, - bool wantShifts); // -> piano roll column + bool wantShifts, + int shiftCount); // -> feeder - FeatureList noteTrack(int shiftCount); - - void emitNote(int start, int end, int note, int shiftCount, - FeatureList ¬eFeatures); + FeatureList obtainNotes(); FeatureSet transcribe(const Grid &); @@ -104,7 +105,8 @@ float noteFrequency(int n, int shift, int shiftCount) const; int m_blockSize; - int m_columnCount; + int m_columnCountIn; + int m_columnCountOut; Vamp::RealTime m_startTime; mutable int m_notesOutputNo;