// ---------------------------------------------------------------------------
// selectMagnitudePeaks (private)
// ---------------------------------------------------------------------------
Peaks
SpectralPeakSelector::selectMagnitudePeaks( ReassignedSpectrum & spectrum,
double minFrequency )
{
using namespace std; // for abs and fabs
const double sampsToHz = mSampleRate / spectrum.size();
const double oneOverSR = 1. / mSampleRate;
const double minFreqSample = minFrequency / sampsToHz;
const double maxCorrectionSamples = mMaxTimeOffset * mSampleRate;
Peaks peaks;
int start_j = 1, end_j = (spectrum.size() / 2) - 2;
double fsample = start_j;
do
{
fsample = spectrum.reassignedFrequency( start_j++ );
} while( fsample < minFreqSample );
for ( int j = start_j; j < end_j; ++j )
{
if ( spectrum.reassignedMagnitude(j) > spectrum.reassignedMagnitude(j-1) &&
spectrum.reassignedMagnitude(j) > spectrum.reassignedMagnitude(j+1) )
{
// skip low-frequency peaks:
double fsample = spectrum.reassignedFrequency( j );
if ( fsample < minFreqSample )
continue;
// skip peaks with large time corrections:
double timeCorrectionSamps = spectrum.reassignedTime( j );
if ( fabs(timeCorrectionSamps) > maxCorrectionSamples )
continue;
double mag = spectrum.reassignedMagnitude( j );
double phase = spectrum.reassignedPhase( j );
// this will be overwritten later in analysis,
// might be ignored altogether, only used if the
// mixed derivative convergence indicator is stored
// as bandwidth in Analyzer:
double bw = spectrum.convergence( j );
// also store the corrected peak time in seconds, won't
// be able to compute it later:
double time = timeCorrectionSamps * oneOverSR;
Breakpoint bp ( fsample * sampsToHz, mag, bw, phase );
peaks.push_back( SpectralPeak( time, bp ) );
} // end if itsa peak
}
/*
debugger << "SpectralPeakSelector::selectMagnitudePeaks: found "
<< peaks.size() << " peaks" << endl;
*/
return peaks;
}
// ---------------------------------------------------------------------------
// FundamentalBuilder::build
// ---------------------------------------------------------------------------
//
void FundamentalBuilder::build( const Peaks & peaks, double frameTime )
{
amplitudes.clear();
frequencies.clear();
for ( Peaks::const_iterator spkpos = peaks.begin(); spkpos != peaks.end(); ++spkpos )
{
if ( spkpos->amplitude() > mAmpThresh &&
spkpos->frequency() < mFreqThresh )
{
amplitudes.push_back( spkpos->amplitude() );
frequencies.push_back( spkpos->frequency() );
}
}
if ( ! amplitudes.empty() )
{
const double fmin = mFminEnv->valueAt( frameTime );
const double fmax = mFmaxEnv->valueAt( frameTime );
// estimate f0
F0Estimate est( amplitudes, frequencies, fmin, fmax, 0.1 );
if ( est.confidence() >= mMinConfidence &&
est.frequency() > fmin && est.frequency() < fmax )
{
// notifier << "f0 is " << est.frequency << endl;
// add breakpoint to fundamental envelope
mEnvelope.insert( frameTime, est.frequency() );
}
}
}
// ---------------------------------------------------------------------------
// fixBandwidth (HELPER)
// ---------------------------------------------------------------------------
// Fix the bandwidth value stored in the specified Peaks.
// This function is invoked if the spectral residue method is
// not used to compute bandwidth (that method overwrites the
// bandwidth already). If the convergence method is used to
// compute bandwidth, the appropriate scaling is applied
// to the stored mixed phase derivative. Otherwise, the
// Peak bandwidth is set to zero.
//
// The convergence value is on the range [0,1], 0 for a sinusoid,
// and 1 for an impulse. If convergence tolerance is specified (as
// a negative value in m_bwAssocParam), it should be positive and
// less than 1, and specifies the convergence value that is to
// correspond to bandwidth equal to 1.0. This is achieved by scaling
// the convergence by the inverse of the tolerance, and saturating
// at 1.0.
void Analyzer::fixBandwidth( Peaks & peaks )
{
if ( m_bwAssocParam < 0 )
{
double scale = 1.0 / (- m_bwAssocParam);
// m_bwAssocParam stores negative tolerance
for ( Peaks::iterator it = peaks.begin(); it != peaks.end(); ++it )
{
SpectralPeak & pk = *it;
pk.setBandwidth( std::min( 1.0, scale * pk.bandwidth() ) );
}
}
else if ( m_bwAssocParam == 0 )
{
for ( Peaks::iterator it = peaks.begin(); it != peaks.end(); ++it )
{
SpectralPeak & pk = *it;
pk.setBandwidth( 0 );
}
}
}
// ---------------------------------------------------------------------------
// buildPartials
// ---------------------------------------------------------------------------
// Append spectral peaks, extracted from a reassigned time-frequency
// spectrum, to eligible Partials, where possible. Peaks that cannot
// be used to extend eliglble Partials spawn new Partials.
//
// This is similar to the basic MQ partial formation strategy, except that
// before matching, all frequencies are normalized by the value of the
// warping envelope at the time of the current frame. This means that
// the frequency envelopes of all the Partials are warped, and need to
// be un-normalized by calling finishBuilding at the end of the building
// process.
//
void
PartialBuilder::buildPartials( Peaks & peaks, double frameTime )
{
mNewlyEligible.clear();
unsigned int matchCount = 0; // for debugging
// frequency-sort the spectral peaks:
// (the eligible partials are always sorted by
// increasing frequency if we always sort the
// peaks this way)
std::sort( peaks.begin(), peaks.end(), SpectralPeak::sort_increasing_freq );
PartialPtrs::iterator eligible = mEligiblePartials.begin();
for ( Peaks::iterator bpIter = peaks.begin(); bpIter != peaks.end(); ++bpIter )
{
//const Breakpoint & bp = bpIter->breakpoint;
const double peakTime = frameTime + bpIter->time();
// find the Partial that is nearest in frequency to the Peak:
PartialPtrs::iterator nextEligible = eligible;
if ( eligible != mEligiblePartials.end() &&
end_frequency( **eligible ) < bpIter->frequency() )
{
++nextEligible;
while ( nextEligible != mEligiblePartials.end() &&
end_frequency( **nextEligible ) < bpIter->frequency() )
{
++nextEligible;
++eligible;
}
if ( nextEligible != mEligiblePartials.end() &&
better_match( **nextEligible, **eligible, *bpIter ) )
{
eligible = nextEligible;
}
}
// INVARIANT:
//
// eligible is the position of the nearest (in frequency)
// eligible Partial (pointer) or it is mEligiblePartials.end().
//
// nextEligible is the eligible Partial with frequency
// greater than bp, or it is mEligiblePartials.end().
#if defined(Debug_Loris) && Debug_Loris
/*
if ( nextEligible != mEligiblePartials.end() )
{
debugger << matchFrequency << "( " << end_frequency( **eligible )
<< ", " << end_frequency( **nextEligible ) << ")" << endl;
}
*/
#endif
// create a new Partial if there is no eligible Partial,
// or the frequency difference to the eligible Partial is
// too great, or the next peak is a better match for the
// eligible Partial, otherwise add this peak to the eligible
// Partial:
Peaks::iterator nextPeak = //Peaks::iterator( bpIter ); ++nextPeak;
++Peaks::iterator( bpIter ); // some compilers choke on this?
// decide whether this match should be made:
// - can only make the match if eligible is not the end of the list
// - the match is only good if it is close enough in frequency
// - even if the match is good, only match if the next one is not better
bool makeMatch = false;
if ( eligible != mEligiblePartials.end() )
{
bool matchIsGood = mFreqDrift >
std::fabs( end_frequency( **eligible ) - bpIter->frequency() );
if ( matchIsGood )
{
bool nextIsBetter = ( nextPeak != peaks.end() &&
better_match( **eligible, *nextPeak, *bpIter ) );
if ( ! nextIsBetter )
{
makeMatch = true;
}
}
}
Breakpoint bp = bpIter->createBreakpoint();
if ( makeMatch )
{
// invariant:
// if makeMatch is true, then eligible is the position of a valid Partial
(*eligible)->insert( peakTime, bp );
mNewlyEligible.push_back( *eligible );
++matchCount;
}
else
{
Partial p;
p.insert( peakTime, bp );
mCollectedPartials.push_back( p );
mNewlyEligible.push_back( & mCollectedPartials.back() );
}
// update eligible, nextEligible is the eligible Partial
// with frequency greater than bp, or it is mEligiblePartials.end():
eligible = nextEligible;
}
mEligiblePartials = mNewlyEligible;
/*
debugger << "PartialBuilder::buildPartials: matched " << matchCount << endl;
debugger << "PartialBuilder::buildPartials: " << mNewlyEligible.size() << " newly eligible partials" << endl;
*/
}
// ---------------------------------------------------------------------------
// selectReassignmentMinima (private)
// ---------------------------------------------------------------------------
Peaks
SpectralPeakSelector::selectReassignmentMinima( ReassignedSpectrum & spectrum,
double minFrequency )
{
using namespace std; // for abs and fabs
const double sampsToHz = mSampleRate / spectrum.size();
const double oneOverSR = 1. / mSampleRate;
const double minFreqSample = minFrequency / sampsToHz;
const double maxCorrectionSamples = mMaxTimeOffset * mSampleRate;
Peaks peaks;
int start_j = 1, end_j = (spectrum.size() / 2) - 2;
double fsample = start_j;
do
{
fsample = spectrum.reassignedFrequency( start_j++ );
} while( fsample < minFreqSample );
for ( int j = start_j; j < end_j; ++j )
{
// look for changes in the frequency reassignment,
// from positive to negative correction, indicating
// a concentration of energy in the spectrum:
double next_fsample = spectrum.reassignedFrequency( j+1 );
if ( fsample > j && next_fsample < j + 1 )
{
// choose the smaller correction of fsample or next_fsample:
// (could also choose the larger magnitude?)
double freq;
int peakidx;
if ( (fsample-j) < (j+1-next_fsample) )
{
freq = fsample * sampsToHz;
peakidx = j;
}
else
{
freq = next_fsample * sampsToHz;
peakidx = j+1;
}
// still possible that the frequency winds up being
// below the specified minimum
if ( freq >= minFrequency )
{
// keep only peaks with small time corrections:
double timeCorrectionSamps = spectrum.reassignedTime( peakidx );
if ( fabs(timeCorrectionSamps) < maxCorrectionSamples )
{
double mag = spectrum.reassignedMagnitude( peakidx );
double phase = spectrum.reassignedPhase( peakidx );
// this will be overwritten later in analysis,
// might be ignored altogether, only used if the
// mixed derivative convergence indicator is stored
// as bandwidth in Analyzer:
double bw = spectrum.convergence( j );
// also store the corrected peak time in seconds, won't
// be able to compute it later:
double time = timeCorrectionSamps * oneOverSR;
Breakpoint bp( freq, mag, bw, phase );
peaks.push_back( SpectralPeak( time, bp ) );
}
}
}
fsample = next_fsample;
}
/*
debugger << "SpectralPeakSelector::selectReassignmentMinima: found "
<< peaks.size() << " peaks" << endl;
*/
return peaks;
}
// ---------------------------------------------------------------------------
// analyze
// ---------------------------------------------------------------------------
//! Analyze a range of (mono) samples at the given sample rate
//! (in Hz) and store the extracted Partials in the Analyzer's
//! PartialList (std::list of Partials). Use the specified envelope
//! as a frequency reference for Partial tracking.
//!
//! \param bufBegin is a pointer to a buffer of floating point samples
//! \param bufEnd is (one-past) the end of a buffer of floating point
//! samples
//! \param srate is the sample rate of the samples in the buffer
//! \param reference is an Envelope having the approximate
//! frequency contour expected of the resulting Partials.
//
void
Analyzer::analyze( const double * bufBegin, const double * bufEnd, double srate,
const Envelope & reference )
{
// configure the reassigned spectral analyzer,
// always use odd-length windows:
// Kaiser window
double winshape = KaiserWindow::computeShape( sidelobeLevel() );
long winlen = KaiserWindow::computeLength( windowWidth() / srate, winshape );
if (! (winlen % 2))
{
++winlen;
}
debugger << "Using Kaiser window of length " << winlen << endl;
std::vector< double > window( winlen );
KaiserWindow::buildWindow( window, winshape );
std::vector< double > windowDeriv( winlen );
KaiserWindow::buildTimeDerivativeWindow( windowDeriv, winshape );
ReassignedSpectrum spectrum( window, windowDeriv );
// configure the peak selection and partial formation policies:
SpectralPeakSelector selector( srate, m_cropTime );
PartialBuilder builder( m_freqDrift, reference );
// configure bw association policy, unless
// bandwidth association is disabled:
std::unique_ptr< AssociateBandwidth > bwAssociator;
if( m_bwAssocParam > 0 )
{
debugger << "Using bandwidth association regions of width "
<< bwRegionWidth() << " Hz" << endl;
bwAssociator.reset( new AssociateBandwidth( bwRegionWidth(), srate ) );
}
else
{
debugger << "Bandwidth association disabled" << endl;
}
// reset envelope builders:
m_ampEnvBuilder->reset();
m_f0Builder->reset();
m_partials.clear();
try
{
const double * winMiddle = bufBegin;
// loop over short-time analysis frames:
while ( winMiddle < bufEnd )
{
// compute the time of this analysis frame:
const double currentFrameTime = long(winMiddle - bufBegin) / srate;
// compute reassigned spectrum:
// sampsBegin is the position of the first sample to be transformed,
// sampsEnd is the position after the last sample to be transformed.
// (these computations work for odd length windows only)
const double * sampsBegin = std::max( winMiddle - (winlen / 2), bufBegin );
const double * sampsEnd = std::min( winMiddle + (winlen / 2) + 1, bufEnd );
spectrum.transform( sampsBegin, winMiddle, sampsEnd );
// extract peaks from the spectrum, and thin
Peaks peaks = selector.selectPeaks( spectrum, m_freqFloor );
Peaks::iterator rejected = thinPeaks( peaks, currentFrameTime );
// fix the stored bandwidth values
// KLUDGE: need to do this before the bandwidth
// associator tries to do its job, because the mixed
// derivative is temporarily stored in the Breakpoint
// bandwidth!!! FIX!!!!
fixBandwidth( peaks );
if ( m_bwAssocParam > 0 )
{
bwAssociator->associateBandwidth( peaks.begin(), rejected, peaks.end() );
}
// remove rejected Breakpoints (needed above to
// compute bandwidth envelopes):
peaks.erase( rejected, peaks.end() );
// estimate the amplitude in this frame:
m_ampEnvBuilder->build( peaks, currentFrameTime );
// collect amplitudes and frequencies and try to
// estimate the fundamental
m_f0Builder->build( peaks, currentFrameTime );
// form Partials from the extracted Breakpoints:
builder.buildPartials( peaks, currentFrameTime );
// slide the analysis window:
winMiddle += long( m_hopTime * srate ); // hop in samples, truncated
} // end of loop over short-time frames
// unwarp the Partial frequency envelopes:
builder.finishBuilding( m_partials );
// fix the frequencies and phases to be consistent.
if ( m_phaseCorrect )
{
fixFrequency( m_partials.begin(), m_partials.end() );
}
// for debugging:
/*
if ( ! m_ampEnv.empty() )
{
LinearEnvelope::iterator peakpos =
std::max_element( m_ampEnv.begin(), m_ampEnv.end(),
compare2nd<LinearEnvelope::iterator::value_type> );
notifier << "Analyzer found amp peak at time : " << peakpos->first
<< " value: " << peakpos->second << endl;
}
*/
}
catch ( Exception & ex )
{
ex.append( "analysis failed." );
throw;
}
}
// ---------------------------------------------------------------------------
// AmpEnvBuilder::build
// ---------------------------------------------------------------------------
//
void AmpEnvBuilder::build( const Peaks & peaks, double frameTime )
{
double x = std::accumulate( peaks.begin(), peaks.end(), 0.0, accumPeakSquaredAmps );
mEnvelope.insert( frameTime, std::sqrt( x ) );
}
// ---------------------------------------------------------------------------
// thinPeaks (HELPER)
// ---------------------------------------------------------------------------
// Reject peaks that are too quiet (low amplitude). Peaks that are retained,
// but are quiet enough to be in the specified fadeRange should be faded.
// Peaks having negative times are also rejected.
//
// This is exactly the same as the basic peak selection strategy, there
// is no tracking here.
//
// Rejected peaks are placed at the end of the peak collection.
// Return the first position in the collection containing a rejected peak,
// or the end of the collection if no peaks are rejected.
//
// This used to be part of SpectralPeakSelector, but it really had no place
// there. It _should_ remove the rejected peaks, but for now, those are needed
// by the bandwidth association strategy.
//
Peaks::iterator
Analyzer::thinPeaks( Peaks & peaks, double frameTime )
{
const double ampFloordB = m_ampFloor;
// fade quiet peaks out over 10 dB:
const double fadeRangedB = 10.0;
// compute absolute magnitude thresholds:
const double threshold = std::pow( 10., 0.05 * ampFloordB );
const double beginFade = std::pow( 10., 0.05 * (ampFloordB+fadeRangedB) );
// louder peaks are preferred, so consider them
// in order of louder magnitude:
std::sort( peaks.begin(), peaks.end(), SpectralPeak::sort_greater_amplitude );
// negative times are not real, but still might represent
// a noisy part of the spectrum...
Peaks::iterator bogusTimes =
std::remove_if( peaks.begin(), peaks.end(), negative_time( frameTime ) );
// ...get rid of them anyway
peaks.erase( bogusTimes, peaks.end() );
bogusTimes = peaks.end();
Peaks::iterator it = peaks.begin();
Peaks::iterator beginRejected = it;
const double freqResolution =
std::max( m_freqResolutionEnv->valueAt( frameTime ), 0.0 );
while ( it != peaks.end() )
{
SpectralPeak & pk = *it;
// keep this peak if it is loud enough and not
// too near in frequency to a louder one:
double lower = pk.frequency() - freqResolution;
double upper = pk.frequency() + freqResolution;
if ( pk.amplitude() > threshold &&
beginRejected == std::find_if( peaks.begin(), beginRejected, can_mask(lower, upper) ) )
{
// this peak is a keeper, fade its
// amplitude if it is too quiet:
if ( pk.amplitude() < beginFade )
{
double alpha = (beginFade - pk.amplitude())/(beginFade - threshold);
pk.setAmplitude( pk.amplitude() * (1. - alpha) );
}
// keep retained peaks at the front of the collection:
if ( it != beginRejected )
{
std::swap( *it, *beginRejected );
}
++beginRejected;
}
++it;
}
// debugger << "thinPeaks retained " << std::distance( peaks.begin(), beginRejected ) << endl;
// remove rejected Breakpoints:
//peaks.erase( beginRejected, peaks.end() );
return beginRejected;
}
