TEST(ProbWelfordCovarEstimator, sample_covariance) {
const int n = 10;
const int n_learn = 10;
stan::math::welford_covar_estimator estimator(n);
for (int i = 0; i < n_learn; ++i) {
Eigen::VectorXd q = Eigen::VectorXd::Constant(n, i);
estimator.add_sample(q);
}
Eigen::MatrixXd covar(n, n);
estimator.sample_covariance(covar);
for (int i = 0; i < n; ++i)
for (int j = 0; j < n; ++j)
EXPECT_EQ(55.0 / 6.0, covar(i, j));
}
// given a set of parameters, return the velocity and associated error
void compute_answer (int numparams, float *params, measurement *mlist, measurement *****mgrid, float *vx, float *evx, float *vy, float *evy, float *norm, kernel_desc *allkers)
{
int ii, ij, ix, iy, in, ik;
int x0, y0;
float a, b, part;
*vx = 0.0;
*evx = 0.0;
*vy = 0.0;
*evy = 0.0;
*norm = 0.0;
for (ii=0; ii<numparams; ii++)
*norm += params[ii];
for (ii=0; ii<numparams; ii++)
params[ii] /= *norm;
for (ii=0; ii<numparams; ii++)
{
if (params[ii] != 0.0)
{
a = params[ii];
*vx += params[ii] * mlist[ii].vx;
*vy += params[ii] * mlist[ii].vy;
// error w/ covariance
for (ix=-allkers->numoverlap; ix<=allkers->numoverlap; ix++)
{
for (iy=-allkers->numoverlap; iy<=allkers->numoverlap; iy++)
{
x0 = mlist[ii].locx + ix;
y0 = mlist[ii].locy + iy;
if (x0 >= 0 && y0 >= 0 && x0 <= allkers->maxx && y0 <= allkers->maxy)
for (in=0; in<=allkers->maxn; in++)
for (ik=0; ik<=allkers->maxk; ik++)
if (mgrid[x0][y0][in][ik] != NULL)
{
b = params[mgrid[x0][y0][in][ik]->myindex];
part = a * b * covar(mlist+ii, mgrid[x0][y0][in][ik], -1, -1);
// if (part != 0.0) printf("1 %d %f\n", ii, a * b);
*evx += part;
}
}
}
}
}
*evx = sqrt(*evx);
*evy = *evx;
}
Estimator2D<Mahalanobis2D>::Estimator2D(const VECTOR& X, const VECTOR& Y)
: X_(&X), Y_(&Y)
{
setBandwidth(scottBandwidth(X.size(), 2.0, 1.0));
setCovariance(var(X), var(Y), covar(X, Y));
}
/** Executes the algorithm
*
* @throw runtime_error Thrown if algorithm cannot execute
*/
void Fit1D::exec() {
// Custom initialization
prepare();
// check if derivative defined in derived class
bool isDerivDefined = true;
gsl_matrix *M = NULL;
try {
const std::vector<double> inTest(m_parameterNames.size(), 1.0);
std::vector<double> outTest(m_parameterNames.size());
const double xValuesTest = 0;
JacobianImpl J;
M = gsl_matrix_alloc(m_parameterNames.size(), 1);
J.setJ(M);
// note nData set to zero (last argument) hence this should avoid further
// memory problems
functionDeriv(&(inTest.front()), &J, &xValuesTest, 0);
} catch (Exception::NotImplementedError &) {
isDerivDefined = false;
}
gsl_matrix_free(M);
// Try to retrieve optional properties
int histNumber = getProperty("WorkspaceIndex");
const int maxInterations = getProperty("MaxIterations");
// Get the input workspace
MatrixWorkspace_const_sptr localworkspace = getProperty("InputWorkspace");
// number of histogram is equal to the number of spectra
const size_t numberOfSpectra = localworkspace->getNumberHistograms();
// Check that the index given is valid
if (histNumber >= static_cast<int>(numberOfSpectra)) {
g_log.warning("Invalid Workspace index given, using first Workspace");
histNumber = 0;
}
// Retrieve the spectrum into a vector
const MantidVec &XValues = localworkspace->readX(histNumber);
const MantidVec &YValues = localworkspace->readY(histNumber);
const MantidVec &YErrors = localworkspace->readE(histNumber);
// Read in the fitting range data that we were sent
double startX = getProperty("StartX");
double endX = getProperty("EndX");
// check if the values had been set, otherwise use defaults
if (isEmpty(startX)) {
startX = XValues.front();
modifyStartOfRange(startX); // does nothing by default but derived class may
// provide a more intelligent value
}
if (isEmpty(endX)) {
endX = XValues.back();
modifyEndOfRange(endX); // does nothing by default but derived class may
// previde a more intelligent value
}
int m_minX;
int m_maxX;
// Check the validity of startX
if (startX < XValues.front()) {
g_log.warning("StartX out of range! Set to start of frame.");
startX = XValues.front();
}
// Get the corresponding bin boundary that comes before (or coincides with)
// this value
for (m_minX = 0; XValues[m_minX + 1] < startX; ++m_minX) {
}
// Check the validity of endX and get the bin boundary that come after (or
// coincides with) it
if (endX >= XValues.back() || endX < startX) {
g_log.warning("EndX out of range! Set to end of frame");
endX = XValues.back();
m_maxX = static_cast<int>(YValues.size());
} else {
for (m_maxX = m_minX; XValues[m_maxX] < endX; ++m_maxX) {
}
}
afterDataRangedDetermined(m_minX, m_maxX);
// create and populate GSL data container warn user if l_data.n < l_data.p
// since as a rule of thumb this is required as a minimum to obtained
// 'accurate'
// fitting parameter values.
FitData l_data(this, getProperty("Fix"));
l_data.n =
m_maxX -
m_minX; // m_minX and m_maxX are array index markers. I.e. e.g. 0 & 19.
if (l_data.n == 0) {
g_log.error("The data set is empty.");
throw std::runtime_error("The data set is empty.");
}
if (l_data.n < l_data.p) {
g_log.error(
"Number of data points less than number of parameters to be fitted.");
throw std::runtime_error(
"Number of data points less than number of parameters to be fitted.");
}
l_data.X = new double[l_data.n];
l_data.sigmaData = new double[l_data.n];
l_data.forSimplexLSwrap = new double[l_data.n];
l_data.parameters = new double[nParams()];
// check if histogram data in which case use mid points of histogram bins
const bool isHistogram = localworkspace->isHistogramData();
for (unsigned int i = 0; i < l_data.n; ++i) {
if (isHistogram)
l_data.X[i] =
0.5 * (XValues[m_minX + i] +
XValues[m_minX + i + 1]); // take mid-point if histogram bin
else
l_data.X[i] = XValues[m_minX + i];
}
l_data.Y = &YValues[m_minX];
// check that no error is negative or zero
for (unsigned int i = 0; i < l_data.n; ++i) {
if (YErrors[m_minX + i] <= 0.0) {
l_data.sigmaData[i] = 1.0;
} else
l_data.sigmaData[i] = YErrors[m_minX + i];
}
// create array of fitted parameter. Take these to those input by the user.
// However, for doing the
// underlying fitting it might be more efficient to actually perform the
// fitting on some of other
// form of the fitted parameters. For instance, take the Gaussian sigma
// parameter. In practice it
// in fact more efficient to perform the fitting not on sigma but 1/sigma^2.
// The methods
// modifyInitialFittedParameters() and modifyFinalFittedParameters() are used
// to allow for this;
// by default these function do nothing.
m_fittedParameter.clear();
for (size_t i = 0; i < nParams(); i++) {
m_fittedParameter.push_back(getProperty(m_parameterNames[i]));
}
modifyInitialFittedParameters(
m_fittedParameter); // does nothing except if overwritten by derived class
for (size_t i = 0; i < nParams(); i++) {
l_data.parameters[i] = m_fittedParameter[i];
}
// set-up initial guess for fit parameters
gsl_vector *initFuncArg;
initFuncArg = gsl_vector_alloc(l_data.p);
for (size_t i = 0, j = 0; i < nParams(); i++) {
if (l_data.active[i])
gsl_vector_set(initFuncArg, j++, m_fittedParameter[i]);
}
// set-up GSL container to be used with GSL simplex algorithm
gsl_multimin_function gslSimplexContainer;
gslSimplexContainer.n = l_data.p; // n here refers to number of parameters
gslSimplexContainer.f = &gsl_costFunction;
gslSimplexContainer.params = &l_data;
// set-up GSL least squares container
gsl_multifit_function_fdf f;
f.f = &gsl_f;
f.df = &gsl_df;
f.fdf = &gsl_fdf;
f.n = l_data.n;
f.p = l_data.p;
f.params = &l_data;
// set-up remaining GSL machinery for least squared
const gsl_multifit_fdfsolver_type *T = gsl_multifit_fdfsolver_lmsder;
gsl_multifit_fdfsolver *s = NULL;
if (isDerivDefined) {
s = gsl_multifit_fdfsolver_alloc(T, l_data.n, l_data.p);
gsl_multifit_fdfsolver_set(s, &f, initFuncArg);
}
// set-up remaining GSL machinery to use simplex algorithm
const gsl_multimin_fminimizer_type *simplexType =
gsl_multimin_fminimizer_nmsimplex;
gsl_multimin_fminimizer *simplexMinimizer = NULL;
gsl_vector *simplexStepSize = NULL;
if (!isDerivDefined) {
simplexMinimizer = gsl_multimin_fminimizer_alloc(simplexType, l_data.p);
simplexStepSize = gsl_vector_alloc(l_data.p);
gsl_vector_set_all(simplexStepSize,
1.0); // is this always a sensible starting step size?
gsl_multimin_fminimizer_set(simplexMinimizer, &gslSimplexContainer,
initFuncArg, simplexStepSize);
}
// finally do the fitting
int iter = 0;
int status;
double finalCostFuncVal;
double dof = static_cast<double>(
l_data.n - l_data.p); // dof stands for degrees of freedom
// Standard least-squares used if derivative function defined otherwise
// simplex
Progress prog(this, 0.0, 1.0, maxInterations);
if (isDerivDefined) {
do {
iter++;
status = gsl_multifit_fdfsolver_iterate(s);
if (status) // break if error
break;
status = gsl_multifit_test_delta(s->dx, s->x, 1e-4, 1e-4);
prog.report();
} while (status == GSL_CONTINUE && iter < maxInterations);
double chi = gsl_blas_dnrm2(s->f);
finalCostFuncVal = chi * chi / dof;
// put final converged fitting values back into m_fittedParameter
for (size_t i = 0, j = 0; i < nParams(); i++)
if (l_data.active[i])
m_fittedParameter[i] = gsl_vector_get(s->x, j++);
} else {
do {
iter++;
status = gsl_multimin_fminimizer_iterate(simplexMinimizer);
if (status) // break if error
break;
double size = gsl_multimin_fminimizer_size(simplexMinimizer);
status = gsl_multimin_test_size(size, 1e-2);
prog.report();
} while (status == GSL_CONTINUE && iter < maxInterations);
finalCostFuncVal = simplexMinimizer->fval / dof;
// put final converged fitting values back into m_fittedParameter
for (unsigned int i = 0, j = 0; i < m_fittedParameter.size(); i++)
if (l_data.active[i])
m_fittedParameter[i] = gsl_vector_get(simplexMinimizer->x, j++);
}
modifyFinalFittedParameters(
m_fittedParameter); // do nothing except if overwritten by derived class
// Output summary to log file
std::string reportOfFit = gsl_strerror(status);
g_log.information() << "Iteration = " << iter << "\n"
<< "Status = " << reportOfFit << "\n"
<< "Chi^2/DoF = " << finalCostFuncVal << "\n";
for (size_t i = 0; i < m_fittedParameter.size(); i++)
g_log.information() << m_parameterNames[i] << " = " << m_fittedParameter[i]
<< " \n";
// also output summary to properties
setProperty("OutputStatus", reportOfFit);
setProperty("OutputChi2overDoF", finalCostFuncVal);
for (size_t i = 0; i < m_fittedParameter.size(); i++)
setProperty(m_parameterNames[i], m_fittedParameter[i]);
std::string output = getProperty("Output");
if (!output.empty()) {
// calculate covariance matrix if derivatives available
gsl_matrix *covar(NULL);
std::vector<double> standardDeviations;
std::vector<double> sdExtended;
if (isDerivDefined) {
covar = gsl_matrix_alloc(l_data.p, l_data.p);
gsl_multifit_covar(s->J, 0.0, covar);
int iPNotFixed = 0;
for (size_t i = 0; i < nParams(); i++) {
sdExtended.push_back(1.0);
if (l_data.active[i]) {
sdExtended[i] = sqrt(gsl_matrix_get(covar, iPNotFixed, iPNotFixed));
iPNotFixed++;
}
}
modifyFinalFittedParameters(sdExtended);
for (size_t i = 0; i < nParams(); i++)
if (l_data.active[i])
standardDeviations.push_back(sdExtended[i]);
declareProperty(
new WorkspaceProperty<API::ITableWorkspace>(
"OutputNormalisedCovarianceMatrix", "", Direction::Output),
"The name of the TableWorkspace in which to store the final "
"covariance matrix");
setPropertyValue("OutputNormalisedCovarianceMatrix",
output + "_NormalisedCovarianceMatrix");
Mantid::API::ITableWorkspace_sptr m_covariance =
Mantid::API::WorkspaceFactory::Instance().createTable(
"TableWorkspace");
m_covariance->addColumn("str", "Name");
std::vector<std::string>
paramThatAreFitted; // used for populating 1st "name" column
for (size_t i = 0; i < nParams(); i++) {
if (l_data.active[i]) {
m_covariance->addColumn("double", m_parameterNames[i]);
paramThatAreFitted.push_back(m_parameterNames[i]);
}
}
for (size_t i = 0; i < l_data.p; i++) {
Mantid::API::TableRow row = m_covariance->appendRow();
row << paramThatAreFitted[i];
for (size_t j = 0; j < l_data.p; j++) {
if (j == i)
row << 1.0;
else {
row << 100.0 * gsl_matrix_get(covar, i, j) /
sqrt(gsl_matrix_get(covar, i, i) *
gsl_matrix_get(covar, j, j));
}
}
}
setProperty("OutputNormalisedCovarianceMatrix", m_covariance);
}
declareProperty(new WorkspaceProperty<API::ITableWorkspace>(
"OutputParameters", "", Direction::Output),
"The name of the TableWorkspace in which to store the "
"final fit parameters");
declareProperty(
new WorkspaceProperty<MatrixWorkspace>("OutputWorkspace", "",
Direction::Output),
"Name of the output Workspace holding resulting simlated spectrum");
setPropertyValue("OutputParameters", output + "_Parameters");
setPropertyValue("OutputWorkspace", output + "_Workspace");
// Save the final fit parameters in the output table workspace
Mantid::API::ITableWorkspace_sptr m_result =
Mantid::API::WorkspaceFactory::Instance().createTable("TableWorkspace");
m_result->addColumn("str", "Name");
m_result->addColumn("double", "Value");
if (isDerivDefined)
m_result->addColumn("double", "Error");
Mantid::API::TableRow row = m_result->appendRow();
row << "Chi^2/DoF" << finalCostFuncVal;
for (size_t i = 0; i < nParams(); i++) {
Mantid::API::TableRow row = m_result->appendRow();
row << m_parameterNames[i] << m_fittedParameter[i];
if (isDerivDefined && l_data.active[i]) {
// perhaps want to scale standard deviations with sqrt(finalCostFuncVal)
row << sdExtended[i];
}
}
setProperty("OutputParameters", m_result);
// Save the fitted and simulated spectra in the output workspace
MatrixWorkspace_const_sptr inputWorkspace = getProperty("InputWorkspace");
int iSpec = getProperty("WorkspaceIndex");
const MantidVec &inputX = inputWorkspace->readX(iSpec);
const MantidVec &inputY = inputWorkspace->readY(iSpec);
int histN = isHistogram ? 1 : 0;
Mantid::DataObjects::Workspace2D_sptr ws =
boost::dynamic_pointer_cast<Mantid::DataObjects::Workspace2D>(
Mantid::API::WorkspaceFactory::Instance().create(
"Workspace2D", 3, l_data.n + histN, l_data.n));
ws->setTitle("");
ws->getAxis(0)->unit() =
inputWorkspace->getAxis(0)
->unit(); // UnitFactory::Instance().create("TOF");
for (int i = 0; i < 3; i++)
ws->dataX(i)
.assign(inputX.begin() + m_minX, inputX.begin() + m_maxX + histN);
ws->dataY(0).assign(inputY.begin() + m_minX, inputY.begin() + m_maxX);
MantidVec &Y = ws->dataY(1);
MantidVec &E = ws->dataY(2);
double *lOut =
new double[l_data.n]; // to capture output from call to function()
modifyInitialFittedParameters(m_fittedParameter); // does nothing except if
// overwritten by derived
// class
function(&m_fittedParameter[0], lOut, l_data.X, l_data.n);
modifyInitialFittedParameters(m_fittedParameter); // reverse the effect of
// modifyInitialFittedParameters - if any
for (unsigned int i = 0; i < l_data.n; i++) {
Y[i] = lOut[i];
E[i] = l_data.Y[i] - Y[i];
}
delete[] lOut;
setProperty("OutputWorkspace",
boost::dynamic_pointer_cast<MatrixWorkspace>(ws));
if (isDerivDefined)
gsl_matrix_free(covar);
}
// clean up dynamically allocated gsl stuff
if (isDerivDefined)
gsl_multifit_fdfsolver_free(s);
else {
gsl_vector_free(simplexStepSize);
gsl_multimin_fminimizer_free(simplexMinimizer);
}
delete[] l_data.X;
delete[] l_data.sigmaData;
delete[] l_data.forSimplexLSwrap;
delete[] l_data.parameters;
gsl_vector_free(initFuncArg);
return;
}
DistributionGaussian* DistributionGaussian::clone(void) const
{
return new DistributionGaussian(mean(),covar());
}
int main(int argc, char **argv)
{
/* output file pointers */
FILE *rlfp=NULL, *taufp=NULL, *e21fp=NULL;
FILE *e31fp=NULL, *e32fp=NULL, *plnfp=NULL;
FILE *f1fp=NULL, *l1fp=NULL;
FILE *thetafp=NULL, *phifp=NULL, *dirfp=NULL;
FILE *erfp=NULL, *irfp=NULL, *qrfp=NULL;
FILE *headerfp=NULL, *pfiltfp=NULL, *sfiltfp=NULL;
FILE *nfiltfp=NULL, *efiltfp=NULL;
// FILE *pkur=NULL, *skur=NULL;
/* temporary file for trace headers */
/* (one 3C station only) */
char *file=NULL; /* base of output file name(s) */
char *fname=NULL; /* complete output file name */
char *angle=NULL; /* unit used for angles theta and phi */
char *win=NULL; /* shape of used time window */
float fangle=0.0; /* unit conversion factor applied to angles theta and phi */
int iwin=0; /* time window shape identifier */
/* flags (see selfdoc) */
int rl,theta,phi,tau,ellip,pln,f1,l1,dir,amp,verbose,all;
int i,j,icomp; /* indices for components (in loops) */
int it; /* index for time sample in main loop */
int iwl; /* correlation window length in samples */
int nstat; /* number of 3-component datasets */
int nt; /* number of time samples in one trace */
// int kwl; /* kurtosis window length in seconds */
float **data3c; /* three-component data ([1..3][0..nt-1]) */
float **a; /* covariance matrix (a[1..3][1..3]) */
float **v; /* eigenvectors of covariance matrix (v[1..3][1..3]) */
float *d; /* the corresponding eigenvalues (d[1..3]) */
float *w; /* time window weights for correlation window */
float dt; /* sampling interval in seconds */
float rlq; /* contrast factor of rectilinearity */
float wl; /* correlation window length in seconds */
float *data_e21=NULL; /* main ellipticity */
float *data_e31=NULL; /* second ellipticity */
float *data_e32=NULL; /* transverse ellipticity */
float *data_er=NULL; /* eigenresultant */
float *data_f1=NULL; /* flatness coefficient */
float *data_ir=NULL; /* instantaneous resultant */
float *data_l1=NULL; /* linearity coefficient */
float *data_phi=NULL; /* horizontal azimuth phi */
float *data_pln=NULL; /* planarity */
float *data_qr=NULL; /* quadratic resultant */
float *data_rl=NULL; /* rectilinearity factor */
float *data_tau=NULL; /* polarization parameter tau */
float *data_theta=NULL; /* inclination angle theta */
float *data_pfilt=NULL; /* P (vertical) polarization filter */
float *data_sfilt=NULL; /* S (horizontal) polarization filter */
float *data_zfilt=NULL; /* Z Filtered Trace */
float *data_nfilt=NULL; /* N Filtered Trace */
float *data_efilt=NULL; /* E Filtered Trace */
// float *data_kwl=NULL; /* Data for Kurtosis Window */
// float *data_pkur=NULL; /* Kurtosis detector for P */
// float *data_skur=NULL; /* Kurtosis detector for S */
float **data3c_dir=NULL; /* 3 components of direction of polarization ([1..3][0..nt-1]) */
/* initialize */
initargs(argc, argv);
requestdoc(1);
/* get info from first trace */
if(!gettr(&tr)) err("can't get first trace");
nt = tr.ns;
/* get parameters ... */
if (!getparstring("file", &file)) file="polar";
if (!getparstring("angle", &angle)) angle="rad";
if (!getparstring("win", &win)) win="boxcar";
if (!getparfloat("wl", &wl)) wl = 0.1;
if (!getparfloat("dt", &dt)) dt = ((double) tr.dt)/1000000.0;
if (!getparfloat("rlq", &rlq)) rlq = 1.0;
if (!getparint("verbose", &verbose)) verbose = 0;
// if (!getparint("kwl", &kwl)) kwl = 5 * ((int) 1/dt);
/* ... and output flags */
if (!getparint("all", &all)) all = 0;
if (!getparint("rl", &rl)) rl = (all) ? all : 1;
if (!getparint("dir", &dir)) dir = (all) ? 1 : 1;
if (!getparint("theta", &theta)) theta = (all) ? all : 0;
if (!getparint("phi", &phi)) phi = (all) ? all : 0;
if (!getparint("tau", &tau)) tau = (all) ? 1 : 0;
if (!getparint("ellip", &ellip)) ellip = (all) ? 1 : 0;
if (!getparint("pln", &pln)) pln = (all) ? 1 : 0;
if (!getparint("f1", &f1)) f1 = (all) ? 1 : 0;
if (!getparint("l1", &l1)) l1 = (all) ? 1 : 0;
if (!getparint("amp", &)) amp = (all) ? 1 : 0;
checkpars();
/* get time window shape */
if (STREQ(win, "boxcar")) iwin=WBOXCAR;
else if (STREQ(win, "bartlett")) iwin=WBARTLETT;
else if (STREQ(win, "hanning")) iwin=WHANNING;
else if (STREQ(win, "welsh")) iwin=WWELSH;
else err("unknown win=%s", win);
/* get unit conversion factor for angles */
if (STREQ(angle, "rad")) fangle=1.0;
else if (STREQ(angle, "deg")) fangle=180.0/PI;
else if (STREQ(angle, "gon")) fangle=200.0/PI;
else err("unknown angle=%s", angle);
/* convert seconds to samples */
if (!dt) {
dt = 0.004;
warn("dt not set, assuming dt=0.004");
}
iwl = NINT(wl/dt);
/* data validation */
if (iwl<1) err("wl=%g must be positive", wl);
if (iwl>nt) err("wl=%g too long for trace", wl);
if (!strlen(file)) err("file= not set and default overridden");
/* echo some information */
if (verbose && (theta || phi)) warn("computing angles in %s", angle);
if (verbose) warn("%s window length = %d samples\n", win, iwl);
if (rl && theta) warn("computing filtered phase");
/* open temporary file for trace headers */
headerfp = etmpfile();
/* set filenames and open files */
fname = malloc( strlen(file)+7 );
sprintf(fname, "%s.rl", file); if (rl) rlfp = efopen(fname, "w");
sprintf(fname, "%s.theta", file); if (theta) thetafp = efopen(fname, "w");
sprintf(fname, "%s.phi", file); if (phi) phifp = efopen(fname, "w");
sprintf(fname, "%s.tau", file); if (tau) taufp = efopen(fname, "w");
sprintf(fname, "%s.e21", file); if (ellip) e21fp = efopen(fname, "w");
sprintf(fname, "%s.e31", file); if (ellip) e31fp = efopen(fname, "w");
sprintf(fname, "%s.e32", file); if (ellip) e32fp = efopen(fname, "w");
sprintf(fname, "%s.pln", file); if (pln) plnfp = efopen(fname, "w");
sprintf(fname, "%s.f1", file); if (f1) f1fp = efopen(fname, "w");
sprintf(fname, "%s.l1", file); if (l1) l1fp = efopen(fname, "w");
sprintf(fname, "%s.dir", file); if (dir) dirfp = efopen(fname, "w");
sprintf(fname, "%s.er", file); if (amp) erfp = efopen(fname, "w");
sprintf(fname, "%s.ir", file); if (amp) irfp = efopen(fname, "w");
sprintf(fname, "%s.qr", file); if (amp) qrfp = efopen(fname, "w");
sprintf(fname, "%s.pfilt", file); if (rl && theta) pfiltfp = efopen(fname, "w");
sprintf(fname, "%s.sfilt", file); if (rl && theta) sfiltfp = efopen(fname, "w");
sprintf(fname, "%s.nfilt", file); if (rl && theta) nfiltfp = efopen(fname, "w");
sprintf(fname, "%s.efilt", file); if (rl && theta) efiltfp = efopen(fname, "w");
// sprintf(fname, "%s.pkur", file); if (rl && theta) pkur = efopen(fname, "w");
// sprintf(fname, "%s.skur", file); if (rl && theta) skur = efopen(fname, "w");
free(fname);
/* allocate space for input data and analysis matrices */
/* index ranges used here: data3c[1..3][0..nt-1], */
/* a[1..3][1..3], v[1..3][1..3], d[1..3] */
data3c = ealloc2float(nt,3); data3c-=1;
a = ealloc2float(3,3); a[0]-=1; a-=1;
v = ealloc2float(3,3); v[0]-=1; v-=1;
d = ealloc1float(3); d-=1;
/* calculate time window weights */
w = ealloc1float(iwl);
memset((void *) w, 0, iwl*FSIZE);
calc_window(w, iwl, iwin);
/* allocate and zero out space for output data */
if (rl) {
data_rl = ealloc1float(nt);
memset((void *) data_rl, 0, nt*FSIZE);
}
if (theta) {
data_theta = ealloc1float(nt);
memset((void *) data_theta, 0, nt*FSIZE);
}
if (phi) {
data_phi = ealloc1float(nt);
memset((void *) data_phi, 0, nt*FSIZE);
}
if (tau) {
data_tau = ealloc1float(nt);
memset((void *) data_tau, 0, nt*FSIZE);
}
if (ellip) {
data_e21 = ealloc1float(nt);
data_e31 = ealloc1float(nt);
data_e32 = ealloc1float(nt);
memset((void *) data_e21, 0, nt*FSIZE);
memset((void *) data_e31, 0, nt*FSIZE);
memset((void *) data_e32, 0, nt*FSIZE);
}
if (pln) {
data_pln = ealloc1float(nt);
memset((void *) data_pln, 0, nt*FSIZE);
}
if (f1) {
data_f1 = ealloc1float(nt);
memset((void *) data_f1, 0, nt*FSIZE);
}
if (l1) {
data_l1 = ealloc1float(nt);
memset((void *) data_l1, 0, nt*FSIZE);
}
if (amp) {
data_er = ealloc1float(nt);
data_ir = ealloc1float(nt);
data_qr = ealloc1float(nt);
memset((void *) data_er, 0, nt*FSIZE);
memset((void *) data_ir, 0, nt*FSIZE);
memset((void *) data_qr, 0, nt*FSIZE);
}
if (dir) {
data3c_dir = ealloc2float(nt,3); data3c_dir-=1;
for (i=1;i<=3;i++) memset((void *) data3c_dir[i], 0, nt*FSIZE);
}
if (rl && theta) {
data_pfilt = ealloc1float(nt);
memset((void *) data_pfilt, 0, nt*FSIZE);
data_sfilt = ealloc1float(nt);
memset((void *) data_pfilt, 0, nt*FSIZE);
/* data_3Cfilt = ealloc2float(nt,3);
for (i=1;i<=3;i++) memset((void *) data_3Cfilt[i], 0, nt*FSIZE); */
data_zfilt = ealloc1float(nt);
data_nfilt = ealloc1float(nt);
data_efilt = ealloc1float(nt);
memset((void *) data_zfilt, 0, nt*FSIZE);
memset((void *) data_nfilt, 0, nt*FSIZE);
memset((void *) data_efilt, 0, nt*FSIZE);
// data_pkur = ealloc1float(nt);
// data_skur = ealloc1float(nt);
// memset((void *) data_pkur, 0, nt*FSIZE);
// memset((void *) data_skur, 0, nt*FSIZE);
/* Allocate data for kurtosis window arrays */
// data_kwl = ealloc1float(iwl);
// memset((void *) data_kwl, 0, kwl*FSIZE);
}
/* ************************ BEGIN CALCULATION ******************************* */
/* loop over traces */
icomp=0;
nstat=0;
// Need to convert this do while loop into a for loop so as to be easier
// to parallelize
warn("Trace Start Time: %d %d %d %d %d", tr.year, tr.day, tr.hour, tr.minute, tr.sec);
do {
/* store trace header in temporary file and read data */
efwrite(&tr, HDRBYTES, 1, headerfp);
icomp++;
memcpy((void *)data3c[icomp], (const void *) tr.data, nt*FSIZE);
/* process 3-component dataset */
if (icomp==3) {
erewind(headerfp);
icomp = 0;
nstat++;
if (verbose)
fprintf(stderr,"%s: analyzing station %d \r",argv[0], nstat);
/* start loop over samples */
for (it=iwl/2;it<nt-iwl/2;it++) {
//warn("Sample %d", it);
/* covariance matrix */
for (i=1;i<=3;i++)
{
for (j=i;j<=3;j++)
{
a[i][j]=a[j][i]=covar(data3c[i], data3c[j], it-iwl/2, iwl, w);
}
}
/* compute eigenvalues and vectors */
eig_jacobi(a,d,v,3);
sort_eigenvalues(d,v,3);
/* polarization parameters */
if (rl) data_rl[it]=calc_rl(d,rlq,rl);
if (theta) data_theta[it]=calc_theta(v, theta) * fangle;
if (phi) data_phi[it]=calc_phi(v, phi) * fangle;
if (tau) data_tau[it]=calc_tau(d);
if (ellip) {
data_e21[it]=calc_ellip(d,2,1);
data_e31[it]=calc_ellip(d,3,1);
data_e32[it]=calc_ellip(d,3,2);
}
if (pln) data_pln[it]=calc_plan(d);
if (f1) data_f1[it]=calc_f1(d);
if (l1) data_l1[it]=calc_l1(d);
if (amp) data_er[it]=calc_er(d);
if (dir) calc_dir(data3c_dir,v,it);
if (rl && theta)
{
data_zfilt[it] = data3c[1][it] * calc_pfilt(rl, theta);
data_nfilt[it] = data3c[2][it] * calc_sfilt(rl, theta);
data_efilt[it] = data3c[3][it] * calc_sfilt(rl, theta);
data_pfilt[it] = data_zfilt[it];
data_sfilt[it] = (data_nfilt[it] + data_efilt[it]) / 2;
// data_pkur[it] = kurtosiswindow(data_pfilt,data_kwl,it - kwl/2,kwl,nt);
// data_skur[it] = kurtosiswindow(data_sfilt,data_kwl,it - kwl/2,kwl,nt);
}
} /* end loop over samples */
/* compute amplitude parameters */
if (amp) ampparams(data3c, data_ir, data_qr, nt, iwl);
/* *************************** END CALCULATION ****************************** */
/* ***************************** BEGIN WRITE ******************************** */
/* write polarization attributes to files */
if (rl) fputdata(rlfp, headerfp, data_rl, nt);
if (theta) fputdata(thetafp, headerfp, data_theta, nt);
if (phi) fputdata(phifp, headerfp, data_phi, nt);
if (tau) fputdata(taufp, headerfp, data_tau, nt);
if (ellip) {
fputdata(e21fp, headerfp, data_e21, nt);
fputdata(e31fp, headerfp, data_e31, nt);
fputdata(e32fp, headerfp, data_e32, nt);
}
if (pln) fputdata(plnfp, headerfp, data_pln, nt);
if (f1) fputdata(f1fp, headerfp, data_f1, nt);
if (l1) fputdata(l1fp, headerfp, data_l1, nt);
if (amp) {
fputdata(erfp, headerfp, data_er, nt);
fputdata(irfp, headerfp, data_ir, nt);
fputdata(qrfp, headerfp, data_qr, nt);
}
if (dir) fputdata3c(dirfp, headerfp, data3c_dir, nt);
if (rl && theta)
{
fputdata(pfiltfp, headerfp, data_pfilt, nt);
fputdata(sfiltfp, headerfp, data_sfilt, nt);
fputdata(nfiltfp, headerfp, data_nfilt, nt);
fputdata(efiltfp, headerfp, data_efilt, nt);
// fputdata(pkur, headerfp, data_pkur, nt);
// fputdata(skur, headerfp, data_skur, nt);
}
/* ****************************** END WRITE ********************************* */
} /* end of processing three-component dataset */
} while (gettr(&tr)); /* end loop over traces */
if (verbose) {
fprintf(stderr,"\n");
if (icomp) warn("last %d trace(s) skipped", icomp);
}
/* close files */
efclose(headerfp);
if (rl) efclose(rlfp);
if (theta) efclose(thetafp);
if (phi) efclose(phifp);
if (tau) efclose(taufp);
if (ellip) {
efclose(e21fp);
efclose(e31fp);
efclose(e32fp);
}
if (pln) efclose(plnfp);
if (f1) efclose(f1fp);
if (l1) efclose(l1fp);
if (amp) {
efclose(erfp);
efclose(irfp);
efclose(qrfp);
}
if (dir) efclose(dirfp);
if (rl && theta) {
efclose(pfiltfp);
efclose(sfiltfp);
// efclose(pkur);
// efclose(skur);
}
return(CWP_Exit());
}
// Compute mean and covariance from a CSV file of features
Mat AudioAnalyzer::CSVToCovariance(string filename)
{
// CSV Parsing
Mat matrice;
Mat result;
int nframes = 0, nitems = 0;
// Get the CSV file as a stream
ifstream inFile (filename.c_str());
if (inFile.is_open()) {
// Retrieve the number of frames of the file (=number of lines - header)
nframes = count(istreambuf_iterator<char>(inFile),
istreambuf_iterator<char>(), '\n') - 1;
// Go back to the beginning of the file
inFile.seekg(0);
int linenum = 0;
int itemnum = 0;
string line;
while (getline (inFile, line))
{
//cout << "\nLine #" << linenum << ":" << endl;
istringstream linestream(line);
string item;
itemnum = 0;
// forget firts two cols
getline (linestream, item, ';');
getline (linestream, item, ';');
while (getline (linestream, item, ';'))
{
if (linenum > 0) // CHECK WITH NEW CONF FILE !!!!!!!!!
{
matrice.at<float>(linenum-1, itemnum) = (float)atof(item.c_str());
//cout << "Item #" << itemnum << ": " << item << "-"<<matrice.at<float>(linenum-1, itemnum)<< endl;
}
itemnum++;
}
if (linenum == 0)
{
nitems = itemnum;
matrice.create(nframes, nitems, CV_32F);
}
linenum++;
}
inFile.close();
//cout << "matrice.cols : " << matrice.cols << " / matrice.rows " << matrice.rows << endl;
} else {
cout << "Error: can't open CSV file." << endl;
}
// If the matrix has been successfully populated from the CSV file
if (matrice.rows > 0 && matrice.cols > 0) {
// Calculate the mean vector the covariance matrix
Mat covar(0,0,CV_32F);
Mat mean(0,0,CV_32F);
calcCovarMatrix(matrice, covar, mean, CV_COVAR_ROWS | CV_COVAR_NORMAL, CV_32F);
//cout << "CSVToCovariance: covariance ("<<covar.cols<< "," << covar.rows << ") computed for "<<nitems<<" features on "<<nframes<<" frames."<<endl;
// Init the result matrix :
// - number of rows : 1
// - number of columns : nitems (means) + sum(n-i, i=0..n) (variance)
result.create(1, (int)(nitems*(1 + (float)(nitems+1)/2)), CV_32F);
int index = 0;
// Add the mean to the matrix
for(int j=0; j<mean.cols; j++) {
result.at<float>(0,index) = mean.at<float>(0,j);
index++;
}
for(int i=0; i<covar.rows; i++) {
for(int j=i; j<covar.cols; j++) {
result.at<float>(0,index) = covar.at<float>(i,j);
index++;
}
}
} else {
cout << "Error while parsing CSV file" << endl;
}
return result;
}
/** Executes the algorithm
*
* @throw runtime_error Thrown if algorithm cannot execute
*/
void GenericFit::exec()
{
// Try to retrieve optional properties
const int maxInterations = getProperty("MaxIterations");
Progress prog(this,0.0,1.0,maxInterations?maxInterations:1);
std::string funIni = getProperty("Function");
m_function.reset( API::FunctionFactory::Instance().createInitialized(funIni) );
if (m_function.get() == NULL)
{
throw std::runtime_error("Function was not set.");
}
prog.report("Setting workspace");
API::Workspace_sptr ws = getProperty("InputWorkspace");
std::string input = getProperty("Input");
m_function->setWorkspace(ws,input);
prog.report("Setting minimizer");
// force initial parameters to satisfy constraints of function
m_function->setParametersToSatisfyConstraints();
// check if derivative defined in derived class
bool isDerivDefined = true;
try
{
m_function->functionDeriv(NULL);
}
catch (Exception::NotImplementedError&)
{
isDerivDefined = false;
}
// What minimizer to use
std::string methodUsed = getProperty("Minimizer");
if ( !isDerivDefined && methodUsed.compare("Simplex") != 0 )
{
methodUsed = "Simplex";
g_log.information() << "No derivatives available for this fitting function"
<< " therefore Simplex method used for fitting\n";
}
// set-up minimizer
std::string costFunction = getProperty("CostFunction");
IFuncMinimizer* minimizer = FuncMinimizerFactory::Instance().createUnwrapped(methodUsed);
minimizer->initialize(m_function.get(), costFunction);
// create and populate data containers. Warn user if nData < nParam
// since as a rule of thumb this is required as a minimum to obtained 'accurate'
// fitting parameter values.
const size_t nParam = m_function->nActive();
const size_t nData = m_function->dataSize();
if (nParam == 0)
{
g_log.error("There are no active parameters.");
setProperty("OutputChi2overDoF", minimizer->costFunctionVal());
throw std::runtime_error("There are no active parameters.");
}
if (nData == 0)
{
g_log.error("The data set is empty.");
throw std::runtime_error("The data set is empty.");
}
if (nData < nParam)
{
g_log.error("Number of data points less than number of parameters to be fitted.");
throw std::runtime_error("Number of data points less than number of parameters to be fitted.");
}
// finally do the fitting
int iter = 0;
int status = 0;
double finalCostFuncVal = 0.0;
double dof = static_cast<double>(nData - nParam); // dof stands for degrees of freedom
// Standard least-squares used if derivative function defined otherwise simplex
if ( methodUsed.compare("Simplex") != 0 )
{
status = GSL_CONTINUE;
while (status == GSL_CONTINUE && iter < maxInterations)
{
iter++;
status = minimizer->iterate();
if (status != GSL_SUCCESS && minimizer->hasConverged() != GSL_SUCCESS)
{
// From experience it is found that gsl_multifit_fdfsolver_iterate occasionally get
// stock - even after having achieved a sensible fit. This seem in particular to be a
// problem on Linux. For now only fall back to Simplex if iter = 1 or 2, i.e.
// gsl_multifit_fdfsolver_iterate has failed on the first or second hurdle
if (iter < 3)
{
g_log.warning() << "GenericFit algorithm using " << methodUsed << " failed "
<< "reporting the following: " << gsl_strerror(status) << "\n"
<< "Try using Simplex method instead\n";
methodUsed = "Simplex";
delete minimizer;
minimizer = FuncMinimizerFactory::Instance().createUnwrapped(methodUsed);
minimizer->initialize(m_function.get(), costFunction);
iter = 0;
}
break;
}
status = minimizer->hasConverged();
prog.report("Iteration "+boost::lexical_cast<std::string>(iter));
}
finalCostFuncVal = minimizer->costFunctionVal() / dof;
}
if ( methodUsed.compare("Simplex") == 0 )
{
status = GSL_CONTINUE;
while (status == GSL_CONTINUE && iter < maxInterations)
{
iter++;
status = minimizer->iterate();
if (status) // break if error
{
// if failed at first iteration try reducing the initial step size
if (iter == 1)
{
g_log.information() << "Simplex step size reduced to 0.1\n";
delete minimizer;
SimplexMinimizer* sm = new SimplexMinimizer;
sm->initialize(m_function.get(), costFunction);
sm->resetSize(0.1, m_function.get(), costFunction);
minimizer = sm;
status = GSL_CONTINUE;
continue;
}
break;
}
status = minimizer->hasConverged();
prog.report("Iteration "+boost::lexical_cast<std::string>(iter));
}
finalCostFuncVal = minimizer->costFunctionVal() / dof;
}
// Output summary to log file
std::string reportOfFit = gsl_strerror(status);
g_log.information() << "Method used = " << methodUsed << "\n" <<
"Iteration = " << iter << "\n";
Mantid::API::ICostFunction* costfun
= Mantid::API::CostFunctionFactory::Instance().createUnwrapped(costFunction);
if ( reportOfFit == "success" )
g_log.notice() << reportOfFit << " " << costfun->shortName() <<
" (" << costfun->name() << ") = " << finalCostFuncVal << "\n";
else
g_log.warning() << reportOfFit << " " << costfun->shortName() <<
" (" << costfun->name() << ") = " << finalCostFuncVal << "\n";
for (size_t i = 0; i < m_function->nParams(); i++)
{
g_log.debug() << m_function->parameterName(i) << " = " << m_function->getParameter(i) << " \n";
}
// also output summary to properties
setProperty("OutputStatus", reportOfFit);
setProperty("OutputChi2overDoF", finalCostFuncVal);
setProperty("Minimizer", methodUsed);
setPropertyValue("Function",*m_function);
// if Output property is specified output additional workspaces
std::vector<double> standardDeviations;
std::string output = getProperty("Output");
gsl_matrix *covar(NULL);
// only if derivative is defined for fitting function create covariance matrix output workspace
if ( isDerivDefined )
{
// calculate covariance matrix
covar = gsl_matrix_alloc (nParam, nParam);
minimizer->calCovarianceMatrix( 0.0, covar);
// take standard deviations to be the square root of the diagonal elements of
// the covariance matrix
int iPNotFixed = 0;
for(size_t i=0; i < m_function->nParams(); i++)
{
standardDeviations.push_back(1.0);
if (m_function->isActive(i))
{
standardDeviations[i] = sqrt(gsl_matrix_get(covar,iPNotFixed,iPNotFixed));
if (m_function->activeParameter(iPNotFixed) != m_function->getParameter(m_function->indexOfActive(iPNotFixed)))
{// it means the active param is not the same as declared but transformed
standardDeviations[i] *= fabs(transformationDerivative(iPNotFixed));
}
iPNotFixed++;
}
}
}
if (!output.empty())
{
// only if derivative is defined for fitting function create covariance matrix output workspace
if ( isDerivDefined )
{
// Create covariance matrix output workspace
declareProperty(
new WorkspaceProperty<API::ITableWorkspace>("OutputNormalisedCovarianceMatrix","",Direction::Output),
"The name of the TableWorkspace in which to store the final covariance matrix" );
setPropertyValue("OutputNormalisedCovarianceMatrix",output+"_NormalisedCovarianceMatrix");
Mantid::API::ITableWorkspace_sptr m_covariance = Mantid::API::WorkspaceFactory::Instance().createTable("TableWorkspace");
m_covariance->addColumn("str","Name");
std::vector<std::string> paramThatAreFitted; // used for populating 1st "name" column
for(size_t i=0; i < m_function->nParams(); i++)
{
if (m_function->isActive(i))
{
m_covariance->addColumn("double",m_function->parameterName(i));
paramThatAreFitted.push_back(m_function->parameterName(i));
}
}
for(size_t i=0; i<nParam; i++)
{
Mantid::API::TableRow row = m_covariance->appendRow();
row << paramThatAreFitted[i];
for(size_t j=0; j<nParam; j++)
{
if (j == i)
row << 100.0;
else
{
row << 100.0*gsl_matrix_get(covar,i,j)/sqrt(gsl_matrix_get(covar,i,i)*gsl_matrix_get(covar,j,j));
}
}
}
setProperty("OutputNormalisedCovarianceMatrix",m_covariance);
}
// create output parameter table workspace to store final fit parameters
// including error estimates if derivative of fitting function defined
declareProperty(
new WorkspaceProperty<API::ITableWorkspace>("OutputParameters","",Direction::Output),
"The name of the TableWorkspace in which to store the final fit parameters" );
setPropertyValue("OutputParameters",output+"_Parameters");
Mantid::API::ITableWorkspace_sptr m_result = Mantid::API::WorkspaceFactory::Instance().createTable("TableWorkspace");
m_result->addColumn("str","Name");
m_result->addColumn("double","Value");
if ( isDerivDefined )
m_result->addColumn("double","Error");
for(size_t i=0;i<m_function->nParams();i++)
{
Mantid::API::TableRow row = m_result->appendRow();
row << m_function->parameterName(i) << m_function->getParameter(i);
if ( isDerivDefined && m_function->isActive(i))
{
row << standardDeviations[i];
}
}
// Add chi-squared value at the end of parameter table
Mantid::API::TableRow row = m_result->appendRow();
row << "Cost function value" << finalCostFuncVal;
setProperty("OutputParameters",m_result);
if ( isDerivDefined )
gsl_matrix_free(covar);
}
// Add Parameters, Errors and ParameterNames properties to output so they can be queried on the algorithm.
declareProperty(new ArrayProperty<double> ("Parameters",new NullValidator<std::vector<double> >,Direction::Output));
declareProperty(new ArrayProperty<double> ("Errors",new NullValidator<std::vector<double> >,Direction::Output));
declareProperty(new ArrayProperty<std::string> ("ParameterNames",new NullValidator<std::vector<std::string> >,Direction::Output));
std::vector<double> params,errors;
std::vector<std::string> parNames;
for(size_t i=0;i<m_function->nParams();i++)
{
parNames.push_back(m_function->parameterName(i));
params.push_back(m_function->getParameter(i));
if (!standardDeviations.empty())
{
errors.push_back(standardDeviations[i]);
}
else
{
errors.push_back(0.);
}
}
setProperty("Parameters",params);
setProperty("Errors",errors);
setProperty("ParameterNames",parNames);
// minimizer may have dynamically allocated memory hence make sure this memory is freed up
delete minimizer;
return;
}
void PrincipalComponentsAnalysis::compute(DataFrame& df)
{
if (df.getNumFactors() > 2)
{
// see PrincipalComponentsAnalysisTest
cout << "You realize this hasn't been tested, right?" << endl;
}
Matrix dataMat(df.getNumFactors(), df.getNumDataVectors());
Matrix deviates(df.getNumFactors(), df.getNumDataVectors());
SymmetricMatrix covar(df.getNumFactors());
DiagonalMatrix eigenValues(df.getNumFactors());
Matrix eigenVectors;
ColumnVector means(df.getNumFactors());
means = 0.0;
RowVector h(df.getNumDataVectors());
h = 1.0;
for (unsigned int j = 0; j < df.getNumFactors(); j++)
{
if (df.isNominal(j))
{
throw Tgs::Exception("Only numeric values are supported.");
}
}
for(unsigned int i = 0; i < df.getNumDataVectors(); i++)
{
for (unsigned int j = 0; j < df.getNumFactors(); j++)
{
double v = df.getDataElement(i, j);
if (df.isNull(v))
{
throw Tgs::Exception("Only non-null values are supported.");
}
dataMat.element(j, i) = v;
means.element(j) += v / (double)df.getNumDataVectors();
}
}
try
{
deviates = dataMat - (means * h);
covar << (1.0/(float)df.getNumDataVectors()) * (deviates * deviates.t());
Jacobi::jacobi(covar, eigenValues, eigenVectors);
}
catch (const std::exception&)
{
throw;
}
catch (...)
{
throw Tgs::Exception("Unknown error while calculating PCA");
}
_sortEigens(eigenVectors, eigenValues);
_components.resize(df.getNumFactors());
for (unsigned int v = 0; v < df.getNumFactors(); v++)
{
_components[v].resize(df.getNumFactors());
for (unsigned int d = 0; d < df.getNumFactors(); d++)
{
_components[v][d] = eigenVectors.element(d, v);
}
}
}
