void cholesky_readFromCovarianceFunction(double **m, char* fname,double *resx,double *resy,double *rese,int np, int nc){
int ndays = ceil((resx[np-1-nc]-resx[0])+1e-10);
double covarFunc[ndays+1];
double escaleFactor = 1.0;
int i;
FILE* fin;
logmsg("Parsing '%s' as covariance function",fname);
logtchk("reading covariance function from disk, ndays = %d",ndays);
if (!(fin = fopen(fname,"r")))
{
logerr("Unable to open covariance function file: %s",fname);
exit(1);
}
logdbg("ndays = %d",ndays);
fscanf(fin,"%lf",&escaleFactor);
for (i=0;i<=ndays;i++)
{
if (fscanf(fin,"%lf",&covarFunc[i])!=1)
{
logerr("Incorrect number of days in the Cholesky matrix: %s, trying to read %d days",fname,ndays);
exit(1);
}
}
fclose(fin);
if(escaleFactor!=1.0){
logerr("Ignoring 'error scale factor': %g",escaleFactor);
}
logtchk("complete reading covariance function from disk");
cholesky_covarFunc2matrix(m,covarFunc,ndays,resx,resy,rese,np,nc);
}
/**
* Sort ToAs for one pulsar.
*/
void sortToAs(pulsar* psr){
if (!psr->sorted){
logmsg("Sorting ToAs for psr %s",psr->name);
logtchk("call qsort()");
qsort(psr->obsn,psr->nobs,sizeof(observation),compareObs);
psr->sorted=1;
logtchk("exit qsort()");
}
}
void t2Fit(pulsar *psr,unsigned int npsr, const char *covarFuncFile){
// if we have a model for the data covariance function, then use it.
// Otherwise we we will just whiten using the error bars.
bool haveCovar = (covarFuncFile!=NULL && strcmp(covarFuncFile,"NULL"));
/**
* Find out if there are any global parameters and what they are...
*/
FitInfo global_fitinfo;
t2Fit_fillGlobalFitInfo(psr,npsr,global_fitinfo);
logdbg("Nglobal parameters = %d",global_fitinfo.nParams);
// If we had any global parameters (or constraints) then we need to do a global fit
// otherwise we can do a fit for each pulsar individually, which is quicker
// and saves memory.
bool doGlobalFit = (global_fitinfo.nParams > 0) || (global_fitinfo.nConstraints > 0);
unsigned long long totalGlobalData=0; // the number of data points across all pulsars
unsigned int gParams=global_fitinfo.nParams; // the number of global fit parameters
unsigned int gConstraints=global_fitinfo.nConstraints; // the number of global constraints
unsigned long long totalGlobalParams=gParams;
unsigned long long totalGlobalConstraints=gConstraints;
double** gUinvs[MAX_PSR]; // whitening matrix for each pulsar
double* gX[MAX_PSR]; // "x" values for each pulsar
double* gY[MAX_PSR]; // "y" values for each pulsar
double* gW[MAX_PSR]; // whitened "y" values for each pulsar
double** gDM[MAX_PSR]; // design matrix for each pulsar
double** gWDM[MAX_PSR]; // whitened design matrix for each pulsar
double** gCM[MAX_PSR]; // constraints matrix for each pulsar
unsigned int gNdata[MAX_PSR]; // number of data points for each pulsar (size of x and y)
logmsg("NEW fit routine. GlobalFit=%s",doGlobalFit ? "true" : "false");
/**
* However we are going to do the fit, we want to loop over all the pulsars
* to get the input data and design matricies etc.
*/
for (size_t ipsr=0; ipsr < npsr; ipsr++) {
double *psr_x = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_white_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_e = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
int *psr_toaidx = (int*)malloc(sizeof(int)*psr[ipsr].nobs); // mapping from fit data to observation number
double** uinv; // the whitening matrix.
/**
* Working out which data contributes to the fit is done in this routine.
* Basically gets values for all observations within START and FINISH which are
* not deleted.
*
* returns the number of data points.
*/
const unsigned int psr_ndata = t2Fit_getFitData(psr+ipsr,psr_x,psr_y,psr_e,psr_toaidx);
assert(psr_ndata > 0u);
psr[ipsr].nFit = psr_ndata; // pulsar.nFit is the number of data points used in the fit.
/**
* Now we work out which parameters are being fit for, how many parameters,
* and determine the gradient functions for the design matrix and the update functions
* which update the pulsar struct.
*/
t2Fit_fillFitInfo(psr+ipsr,psr[ipsr].fitinfo,global_fitinfo);
/**
* The whitening matrix behaves diferently if we have a covariance matrix.
* If we have a covariance matrix, uinv is an ndata x ndata triangular matrix.
* Otherwise, it only has diagonal elements, so we efficiently store it as
* a 1-d ndata array.
*/
if (haveCovar) {
// ToAs must be sorted for covariance function code
sortToAs(psr+ipsr);
// malloc_uinv does a blas-compatible allocation of a 2-d array.
uinv = malloc_uinv(psr_ndata);
psr[ipsr].fitMode=1; // Note: forcing this to 1 as the Cholesky fit is a weighted fit
logmsg("Doing a FULL COVARIANCE MATRIX fit");
} else {
// Here the whitening matrix is just a diagonal
// weighting matrix. Store diagonal matrix as 1xN
// so that types match later.
uinv=malloc_blas(1,psr_ndata);
if(psr[ipsr].fitMode == 0){
// if we are doing an unweighted fit then we should set the errors to 1.0
// to give uniform weighting.
logdbg("Doing an UNWEIGHTED fit");
for (unsigned int i=0; i < psr_ndata; i++){
psr_e[i]=1.0;
}
} else {
logdbg("Doing a WEIGHTED fit");
}
}
assert(uinv!=NULL);
/**
* Now we form the whitening matrix, uinv.
* Note that getCholeskyMatrix() is clever enough to see that we
* have created a 1 x ndata matrix if we have only diagonal elements.
*/
getCholeskyMatrix(uinv,covarFuncFile,psr+ipsr,
psr_x,psr_y,psr_e,
psr_ndata,0,psr_toaidx);
logtchk("got Uinv");
// define some convinience variables
const unsigned nParams=psr[ipsr].fitinfo.nParams;
const unsigned nConstraints=psr[ipsr].fitinfo.nConstraints;
/**
* The design matrix is the matrix of gradients for the least-squares.
* If the design matrix is M, parameters p, and data d, we are solving
* M.p = d
* It is ndata x nparams in size. We also allocate the whitened DM here.
*/
double** designMatrix = malloc_blas(psr_ndata,nParams);
double** white_designMatrix = malloc_blas(psr_ndata,nParams);
for (unsigned int idata =0; idata < psr_ndata; ++idata){
// t2Fit_buildDesignMatrix is a replacement for the old FITfuncs routine.
// it fills one row of the design matrix.
t2Fit_buildDesignMatrix(psr,ipsr,psr_x[idata], psr_toaidx[idata], designMatrix[idata]);
}
logtchk("made design matrix");
/**
* The constraints matrix is similar to the design matrix, but here we are solving:
* B.p = 0
* Where B is the constraints matrix and p is the parameters. we solve both this
* and the DM equation set simultaniously. TKleastSquares will do this for us.
*
* If there are no constraints we leave it as NULL, which is detected in TKfit as
* no constraints anyway.
*/
double** constraintsMatrix =NULL;
if(psr[ipsr].fitinfo.nConstraints > 0){
computeConstraintWeights(psr+ipsr);
constraintsMatrix = malloc_blas(nConstraints,nParams);
for (unsigned int iconstraint =0; iconstraint < nConstraints; ++iconstraint){
// similar to t2Fit_buildDesignMatrix, t2Fit_buildConstraintsMatrix
// creates one row of the constraints matrix.
t2Fit_buildConstraintsMatrix(psr, ipsr, iconstraint, constraintsMatrix[iconstraint]);
}
}
logtchk("made constraints matrix");
/**
* Now we multiply the design matrix and the data vector by the whitening matrix.
* If we just have variances (uinv is diagonal) then we do it traditionally, otherwise
* we use TKmultMatrix as this is usually backed by LAPACK and so is fast :)
*/
if(haveCovar){
TKmultMatrixVec(uinv,psr_y,psr_ndata,psr_ndata,psr_white_y);
TKmultMatrix_sq(uinv,designMatrix,psr_ndata,nParams,white_designMatrix);
} else {
for(unsigned i=0;i<psr_ndata;++i){
psr_white_y[i]=psr_y[i]*uinv[0][i];
for(unsigned j=0;j<nParams;++j){
white_designMatrix[i][j] = designMatrix[i][j]*uinv[0][i];
}
}
}
free_blas(uinv);
free(psr_e);
free(psr_toaidx);
logtchk("done whitening");
/*
* Now - if we are going to do a global fit, we store all the above for later
* otherwise
*/
if (doGlobalFit){
// we are going to do a global fit, so need to store the values for later
gX[ipsr] = psr_x;
gY[ipsr] = psr_y;
gW[ipsr] = psr_white_y;
gDM[ipsr] = designMatrix;
gWDM[ipsr] = white_designMatrix;
gCM[ipsr] = constraintsMatrix;
gNdata[ipsr] = psr_ndata;
totalGlobalData += psr_ndata;
totalGlobalParams += nParams - gParams;
totalGlobalConstraints += nConstraints - gConstraints;
} else {
// NOT GLOBAL
// so do one fit at a time...
double chisq; // the post-fit chi-squared
// allocate memory for the output of TKleastSquares
double* parameterEstimates = (double*)malloc(sizeof(double)*nParams);
double* errorEstimates = (double*)malloc(sizeof(double)*nParams);
/*
* Call TKleastSquares, or in fact, TKrobustConstrainedLeastSquares,
* since we might want robust fitting and/or constraints/
*
* The arguments here are explained in TKfit.C
*
*/
chisq = TKrobustConstrainedLeastSquares(psr_y,psr_white_y,
designMatrix,white_designMatrix,constraintsMatrix,
psr_ndata,nParams,nConstraints,
T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[ipsr].covar,
psr[ipsr].robust);
// update the pulsar struct as appropriate
psr[ipsr].fitChisq = chisq;
psr[ipsr].fitNfree = psr_ndata + nConstraints - nParams;
logdbg("Updating the parameters");
logtchk("updating the parameter values");
/*
* This routine calls the appropriate update functions to apply the result of the fit
* to the origianal (non-linearised) pulsar parameters.
*/
t2Fit_updateParameters(psr,ipsr,parameterEstimates,errorEstimates);
logtchk("complete updating the parameter values");
logdbg("Completed updating the parameters");
/*
* If we are not doing a global fit, we can clean up the memory for this pulsar.
* Might make a difference for very large datasets.
*/
logdbg("Free fit memory");
free(parameterEstimates);
free(errorEstimates);
free_blas(designMatrix);
free_blas(white_designMatrix);
if (constraintsMatrix) free_blas(constraintsMatrix);
free(psr_x);
free(psr_y);
free(psr_white_y);
}
}
if (doGlobalFit){
const unsigned int nobs = totalGlobalData;
double** designMatrix = malloc_blas(nobs,totalGlobalParams);
double** white_designMatrix = malloc_blas(nobs,totalGlobalParams);
double** constraintsMatrix = malloc_blas(totalGlobalConstraints,totalGlobalParams);
double *y = (double*)malloc(sizeof(double)*nobs);
double *white_y = (double*)malloc(sizeof(double)*nobs);
unsigned int off_f = gParams; // leave space for globals
unsigned int off_r = 0;
unsigned int off_c = gConstraints;
logdbg("Building matricies for global fit... npsr=%u",npsr);
logdbg("nobs=%u, totalGlobalParams=%u, totalGlobalConstraints=%u",nobs,totalGlobalParams,totalGlobalConstraints);
logwarn("This mode is not supported yet!!!");
for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){
unsigned int nLocal = psr[ipsr].fitinfo.nParams-gParams;
logdbg("ipsr=%u, off_r = %u, off_c=%u, off_f=%u, nlocal=%u",
ipsr,off_r,off_c,off_f,nLocal);
// the fit parameters
for(unsigned int i=0; i < gNdata[ipsr]; i++){
// the global params (they go first)
for(unsigned int g= 0; g < gParams; g++){
unsigned int j = g+nLocal;
if(ipsr==0 && i==0 && writeResiduals){
logmsg("Row %d = %s %s(%d)",g,"global",label_str[global_fitinfo.paramIndex[g]],global_fitinfo.paramCounters[g]);
}
designMatrix[i+off_r][g] = gDM[ipsr][i][j];
white_designMatrix[i+off_r][g] = gWDM[ipsr][i][j];
}
for(unsigned int j=0; j < nLocal; j++){
if(i==0 && writeResiduals){
logmsg("Row %d = %s %s(%d)",j+off_f,psr[ipsr].name,label_str[psr[ipsr].fitinfo.paramIndex[j]],psr[ipsr].fitinfo.paramCounters[j]);
}
designMatrix[i+off_r][j+off_f] = gDM[ipsr][i][j];
white_designMatrix[i+off_r][j+off_f] = gWDM[ipsr][i][j];
}
}
// the data
for(unsigned int i=0; i < gNdata[ipsr]; ++i){
y[i+off_r] = gY[ipsr][i];
white_y[i+off_r] = gW[ipsr][i];
}
for(unsigned int i=0; i < psr[ipsr].fitinfo.nConstraints; i++){
for(unsigned int j=0; j < nLocal; j++){
constraintsMatrix[i+off_c][j+off_f] = gCM[ipsr][i][j];
}
// the global params (they go first)
for(unsigned int g= 0; g < gParams; g++){
unsigned int j = g+nLocal;
constraintsMatrix[i+off_c][g] = gCM[ipsr][i][j];
}
}
off_r += gNdata[ipsr];
off_f += nLocal;
off_c += psr[ipsr].fitinfo.nConstraints;
free(gY[ipsr]);
free(gW[ipsr]);
free_blas(gDM[ipsr]);
if (gCM[ipsr]) free_blas(gCM[ipsr]);
free_blas(gWDM[ipsr]);
}
double chisq; // the post-fit chi-squared
double* parameterEstimates = (double*)malloc(sizeof(double)*totalGlobalParams);
double* errorEstimates = (double*)malloc(sizeof(double)*totalGlobalParams);
chisq = TKrobustConstrainedLeastSquares(y,white_y,
designMatrix,white_designMatrix,constraintsMatrix,
nobs,totalGlobalParams,totalGlobalConstraints,
T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[0].covar,
psr[0].robust);
// for now the CVM ends up in psr[0].covar.
int off_p = gParams;
for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){
// update the pulsar struct as appropriate
psr[ipsr].fitChisq = chisq;
psr[ipsr].fitNfree = nobs + totalGlobalConstraints - totalGlobalParams;
double* psr_parameterEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams);
double* psr_errorEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams);
const unsigned np = psr[ipsr].fitinfo.nParams-gParams;
/* extract the fit output for the individual pulsars.
* I.e. detangle the global fit
* Notice: Globals go at the end of the individual pulsar arrays.
*/
for (unsigned i = 0; i < np; ++i){
psr_parameterEstimates[i] = parameterEstimates[off_p];
psr_errorEstimates[i] = errorEstimates[off_p];
++off_p;
}
for (unsigned i = 0; i < gParams; ++i){
psr_parameterEstimates[i+np] = parameterEstimates[i];
psr_errorEstimates[i] = errorEstimates[off_p];
}
logdbg("Updating the parameters");
logtchk("updating the parameter values");
/*
* This routine calls the appropriate update functions to apply the result of the fit
* to the origianal (non-linearised) pulsar parameters.
*/
t2Fit_updateParameters(psr,ipsr,psr_parameterEstimates,psr_errorEstimates);
logtchk("complete updating the parameter values");
logdbg("Completed updating the parameters");
free(psr_parameterEstimates);
free(psr_errorEstimates);
}
free(parameterEstimates);
free(errorEstimates);
free(white_y);
free(y);
free_blas(designMatrix);
free_blas(white_designMatrix);
free_blas(constraintsMatrix);
}
}
/**
* UINV is a lower triangluar matrix.
* Matricies are row-major order, i.e. uinv[r][c].
* returns 0 if ok.
*/
int cholesky_formUinv(double **uinv,double** m,int np){
int i,j,k;
logtchk("forming Cholesky matrix ... do Cholesky decomposition");
#ifdef ACCEL_UINV
if(useT2accel){
logdbg("Doing ACCELERATED Chol Decomp (M.Keith/LAPACK method)");
for(i =0;i<np;i++){
memcpy(uinv[i],m[i],np*sizeof(double));
}
int ret = accel_uinv(uinv[0],np);
if (ret != 0) return ret;
logtchk("forming Cholesky matrix ... complete calculate uinv");
} else {
#endif
double sum;
double *cholp = (double *)malloc(sizeof(double)*(np+1));
logmsg("Doing Cholesky decomp and inverting matrix (SLOW method)");
if(!cholp)logerr("Could not allocate enough memory");
T2cholDecomposition(m,np,cholp);
logtchk("forming Cholesky matrix ... complete do Cholesky decomposition");
// Now calculate uinv
logtchk("forming Cholesky matrix ... calculate uinv");
for (i=0;i<np;i++)
{
m[i][i] = 1.0/cholp[i];
uinv[i][i] = m[i][i];
for (j=0;j<i;j++)
uinv[j][i] = 0.0;
for (j=i+1;j<np;j++)
{
sum=0.0;
for (k=i;k<j;k++) sum-=m[j][k]*m[k][i];
m[j][i]=sum/cholp[j];
uinv[j][i] = m[j][i];
}
}
logtchk("forming Cholesky matrix ... complete calculate uinv");
if (debugFlag)
{
logdbg("uinv = ");
for (i=0;i<5;i++)
{
for (j=0;j<5;j++) fprintf(LOG_OUTFILE,"%10g ",uinv[i][j]);
fprintf(LOG_OUTFILE,"\n");
}
fprintf(LOG_OUTFILE,"\n");
}
logtchk("forming Cholesky matrix ... free memory");
free(cholp);
logtchk("forming Cholesky matrix ... complete free memory");
#ifdef ACCEL_UINV
} // end the if clause when we have the option of accelerated cholesky.
#endif
if(debugFlag){
logdbg("Write uinv");
FILE* file=fopen("chol.uinv","w");
for(i =0;i<np;i++){
for(j =0;j<np;j++){
fprintf(file,"%d %d %lg\n",i,j,uinv[i][j]);
}
fprintf(file,"\n");
}
fclose(file);
}
return 0;
}
void cholesky_covarFunc2matrix(double** m, double* covarFunc, int ndays,double *resx,double *resy,double *rese,int np, int nc){
double escaleFactor = 1.0;
int i,j,k;
int ix,iy;
double t0,cint,t;
int t1,t2;
logtchk("forming Cholesky matrix ... determing m[ix][iy] = fabs(resx[ix]-resx[iy])");
for (ix=0;ix<(np);ix++)
{
for (iy=0;iy<(np);iy++)
m[ix][iy] = fabs(resx[ix]-resx[iy]);
}
logtchk("forming Cholesky matrix ... complete determing m[ix][iy] = fabs(resx[ix]-resx[iy])");
if (debugFlag==1)
{
logdbg("First m = ");
for (i=0;i<5;i++)
{
for (j=0;j<5;j++) fprintf(LOG_OUTFILE,"%10g ",m[i][j]);
fprintf(LOG_OUTFILE,"\n");
}
fprintf(LOG_OUTFILE,"\n");
logdbg("CovarFunc = ");
for (i=0;i<10;i++)
{
fprintf(LOG_OUTFILE,"%10g\n",covarFunc[i]);
}
}
// Insert the covariance which depends only on the time difference.
// Linearly interpolate between elements on the covariance function because
// valid covariance matrix must have decreasing off diagonal elements.
// logdbg("Inserting into the covariance matrix");
logtchk("forming Cholesky matrix ... determing covariance based on time difference");
for (ix=0;ix<(np);ix++)
{
for (iy=0;iy<(np);iy++)
{
if (ix >= np-nc || iy >= np-nc)
{
m[ix][iy] = 0;
}
else
{
t0 = m[ix][iy];
t1 = (int)floor(t0);
t2 = t1+1;
t = t0-t1;
if (t1 > ndays || t2 > ndays)
{
logerr("Problem that t1 or t2 > ndays: t1 = %d, t2 = %d, ndays = %d, ix = %d, iy = %d, np = %d",t1,t2,ndays,ix,iy,np);
exit(1);
}
cint = covarFunc[t1]*(1-t)+covarFunc[t2]*t; // Linear interpolation
m[ix][iy] = cint;
}
}
}
logtchk("forming Cholesky matrix ... complete determing covariance based on time difference");
// add the values for the constraints
// Constraints are not covariant with anything so it's all zero!
for (i=np-nc; i < np; i++){
for (j=0; j < np; j++){
m[i][j]=0;
m[j][i]=0;
}
}
}