void resize(Mat& img, float fx) {
Mat dst;
int inter = (fx > 1.0) ? CV_INTER_CUBIC : CV_INTER_AREA;
cv::resize(img, dst, Size(), fx, 1.0, inter);
assert(img.rows == dst.rows);
if (img.cols > dst.cols) {
dst.copyTo(img(Range::all(), Range(0, dst.cols)));
img(Range::all(), Range(dst.cols, img.cols)) = cv::Scalar::all(0);
} else {
dst(Range::all(), Range(0, img.cols)).copyTo(img);
}
}
// These can all be static
void specgram::wav_to_specgram(const Mat& wav_col_mat,
const int frame_length_tn,
const int frame_stride_tn,
const int max_time_steps,
const Mat& window,
Mat& specgram)
{
// const Mat& wav_mat = wav.get_data();
// Read as a row vector
Mat wav_mat = wav_col_mat.reshape(1, 1);
// TODO: support more sample formats
if (wav_mat.elemSize1() != 2) {
throw std::runtime_error(
"Unsupported number of bytes per sample: " + std::to_string(wav_mat.elemSize1()));
}
// Go from time domain to strided signal
Mat wav_frames;
{
int num_frames = ((wav_mat.cols - frame_length_tn) / frame_stride_tn) + 1;
num_frames = std::min(num_frames, max_time_steps);
// ensure that there is enough data for at least one frame
assert(num_frames >= 0);
wav_frames.create(num_frames, frame_length_tn, wav_mat.type());
for (int frame = 0; frame < num_frames; frame++) {
int start = frame * frame_stride_tn;
int end = start + frame_length_tn;
wav_mat.colRange(start, end).copyTo(wav_frames.row(frame));
}
}
// Prepare for DFT by converting to float
Mat input;
wav_frames.convertTo(input, CV_32FC1);
// Apply window if it has been created
if (window.cols == frame_length_tn) {
input = input.mul(cv::repeat(window, input.rows, 1));
}
Mat planes[] = {input, Mat::zeros(input.size(), CV_32FC1)};
Mat compx;
cv::merge(planes, 2, compx);
cv::dft(compx, compx, cv::DFT_ROWS);
compx = compx(Range::all(), Range(0, frame_length_tn / 2 + 1));
cv::split(compx, planes);
cv::magnitude(planes[0], planes[1], planes[0]);
// NOTE: at this point we are returning the specgram representation in
// (time_steps, freq_steps) shape order.
specgram = planes[0];
return;
}
int generate(RawMedia* raw, char* buf, int bufSize) {
// TODO: get rid of this assumption
assert(raw->sampleSize() == 2);
assert(_timeSteps * _height == bufSize);
int rows = stridedSignal(raw);
assert(rows <= _timeSteps);
Mat signal(rows, _windowSize, CV_16SC1, (short*) _buf);
Mat input;
signal.convertTo(input, CV_32FC1);
applyWindow(input);
Mat planes[] = {input, Mat::zeros(input.size(), CV_32FC1)};
Mat compx;
cv::merge(planes, 2, compx);
cv::dft(compx, compx, cv::DFT_ROWS);
compx = compx(Range::all(), Range(0, _numFreqs));
cv::split(compx, planes);
cv::magnitude(planes[0], planes[1], planes[0]);
Mat mag;
if (_feature == SPECGRAM) {
mag = planes[0];
} else {
extractFeatures(planes[0], mag);
}
Mat feats;
// Rotate by 90 degrees.
cv::transpose(mag, feats);
cv::flip(feats, feats, 0);
cv::normalize(feats, feats, 0, 255, CV_MINMAX, CV_8UC1);
Mat result(feats.rows, _timeSteps, CV_8UC1, buf);
feats.copyTo(result(Range::all(), Range(0, feats.cols)));
// Pad the rest with zeros.
result(Range::all(), Range(feats.cols, result.cols)) = cv::Scalar::all(0);
randomize(result);
// Return the percentage of valid columns.
return feats.cols * 100 / result.cols;
}
double Calibration::getExtrinsics(const Mat& Tr)
{
// set/create various parameters
Mat rvec;
Mat w(views.world.front());
Mat p(views.pixel.front());
Mat& A = intrinsic_params.A;
Mat& k = intrinsic_params.k;
Mat& R = extrinsic_params.R;
Mat& t = extrinsic_params.t;
// get extrinsic parameters
cv::solvePnP(w, p, A, k, rvec, t, solve_pnp.useExtGuess);
cv::Rodrigues(rvec, R);
// convert default coordinate system to new one
if(!Tr.empty()) {
CV_Assert(Tr.rows == 3 && Tr.cols == 4);
Mat T_Tr = Mat(Tr, Range(0, 3), Range(3, 4));
Mat R_Tr = Mat(Tr, Range(0, 3), Range(0, 3));
t += R*T_Tr;
R *= R_Tr;
}
/*std::cout << "you are printing modified R" << std::endl;
for(int i = 0; i < R.rows; i++) {
for(int j = 0; j < R.cols; j++) {
std::cout << R.at<double>(i,j) << " ";
}
std::cout << std::endl;
}*/
return 0.;
}
void extractFeatures(Mat& spectrogram, Mat& features) {
Mat powspec = spectrogram.mul(spectrogram);
powspec *= 1.0 / _windowSize;
Mat cepsgram = powspec*_fbank;
log(cepsgram, cepsgram);
if (_feature == MFSC) {
features = cepsgram;
return;
}
int pad_cols = cepsgram.cols;
int pad_rows = cepsgram.rows;
if (cepsgram.cols % 2 != 0) {
pad_cols = 1 + cepsgram.cols;
}
if (cepsgram.rows % 2 != 0) {
pad_rows = 1 + cepsgram.rows;
}
Mat padcepsgram = Mat::zeros(pad_rows, pad_cols, CV_32F);
cepsgram.copyTo(padcepsgram(Range(0, cepsgram.rows), Range(0, cepsgram.cols)));
dct(padcepsgram, padcepsgram, cv::DFT_ROWS);
cepsgram = padcepsgram(Range(0, cepsgram.rows), Range(0, cepsgram.cols));
features = cepsgram(Range::all(), Range(0, _numCepstra));
}
void specgram::cepsgram_to_mfcc(const Mat& cepsgram,
const int num_cepstra,
Mat& mfcc)
{
assert(num_cepstra <= cepsgram.cols);
Mat padcepsgram;
if (cepsgram.cols % 2 == 0) {
padcepsgram = cepsgram;
} else {
cv::copyMakeBorder(cepsgram, padcepsgram, 0, 0, 0, 1, cv::BORDER_CONSTANT, cv::Scalar(0));
}
cv::dct(padcepsgram, padcepsgram, cv::DCT_ROWS);
mfcc = padcepsgram(Range::all(), Range(0, num_cepstra));
return;
}
