// routine for pulsar fitting.
void TKleastSquares_single_pulsar(double *x,double *y,int n,double *outP,double *e,int nf,double **cvm, double *chisq, void (*fitFuncs)(double, double [], int,pulsar *,int, int),pulsar *psr,double tol, int *ip,char rescale_errors, double **uinv) {
double **designMatrix, **white_designMatrix, **constraintsMatrix;
double *b,*white_b;
constraintsMatrix=NULL;
TKfit_getPulsarDesignMatrix(x,y,n,nf,fitFuncs,psr,ip,uinv,0,&designMatrix,&white_designMatrix,&b,&white_b);
if(psr->nconstraints > 0){
logmsg("Get constraint weights");
computeConstraintWeights(psr);
logmsg("fill constraints matrix");
constraintsMatrix = malloc_blas(psr->nconstraints,nf);
for (int ic=0; ic < psr->nconstraints; ic++){
CONSTRAINTfuncs(psr,0,nf,psr->constraints[ic],constraintsMatrix[ic]);
}
}
*chisq = TKrobustConstrainedLeastSquares(b,white_b,designMatrix,white_designMatrix,constraintsMatrix,
n,nf,psr->nconstraints,tol,rescale_errors,
outP,e,cvm,psr->robust);
free_blas(designMatrix); // free-TKleastSquares_svd_psr_dcm-designMatrix**
free_blas(white_designMatrix); // free-TKleastSquares_svd_psr_dcm-white_designMatrix**
if(psr->nconstraints > 0) free_blas(constraintsMatrix);
free(b);
free(white_b);
}
void destroyOne (pulsar *psr)
{
if (psr->obsn)
free (psr->obsn);
free_blas(psr->covar);
for (int k=0; k < MAX_PARAMS; ++k){
if(psr->constraint_special[k]){
free(psr->constraint_special[k]);
}
}
destroyMemory(psr);
}
// same as above but without a uinv matrix.
void TKleastSquares_svd_psr(double *x,double *y,double *sig,int n,double *p,double *e,int nf,double **cvm, double *chisq, void (*fitFuncs)(double, double [], int,pulsar *,int,int),int weight,pulsar *psr,double tol, int *ip)
{
logmsg("Warning: Deprecated method TKleastSquares_svd_psr() -> TKleastSquares_single_pulsar()");
int i;
double ** uinv=malloc_blas(1,n);
if (weight==1){
for (i=0; i<n;i++){
uinv[0][i]=1.0/sig[i];
}
} else{
for (i=0; i<n;i++){
uinv[0][i]=1.0;
}
}
TKleastSquares_single_pulsar(x,y,n,p,e,nf,cvm,chisq,fitFuncs,psr,tol,ip,(weight==0 || (weight==1 && psr->rescaleErrChisq==1)),uinv);
free_blas(uinv);
}
void TKleastSquares_global_pulsar(double **x,double **y,int *n,
double *outP,double *e,int* nf, int nglobal,double **cvm, double *chisq, void (*fitFuncs)(double, double [], int,pulsar *,int,int),pulsar *psr,double tol, int **ip,char rescale_errors, double ***uinv, int npsr) {
double **designMatrix, **white_designMatrix;
double **constraintsMatrix;
double **psr_DM, **psr_wDM;
double *b,*white_b, *psr_b,*psr_wb;
int ipsr;
int totalFit=0;
int totalObs=0;
int totalConstraints=0;
int i,j;
int off_r=0;
int off_f=0;
int off_c=0;
for (ipsr=0; ipsr < npsr; ipsr++){
totalFit+=nf[ipsr];
totalObs+=n[ipsr];
totalConstraints+=psr[ipsr].nconstraints;
}
totalFit+=nglobal;
white_designMatrix=malloc_blas(totalObs,totalFit);
designMatrix=malloc_blas(totalObs,totalFit);
constraintsMatrix=malloc_blas(totalConstraints,totalFit);
b=(double*)calloc(totalObs,sizeof(double));
white_b=(double*)calloc(totalObs,sizeof(double));
for (ipsr=0; ipsr < npsr; ipsr++){
logdbg("Getting design matrix / whitened residuals for psr %d off_r=%d off_f=%d nglobal=%d",ipsr,off_r,off_f,nglobal);
TKfit_getPulsarDesignMatrix(x[ipsr],y[ipsr],n[ipsr],nf[ipsr]+nglobal,fitFuncs,psr,ip[ipsr],uinv[ipsr],ipsr,&psr_DM,&psr_wDM,&psr_b,&psr_wb);
// the global fit parameters
for(i=0; i < n[ipsr]; i++){
for(j=0; j < nglobal; j++){
designMatrix[i+off_r][j] = psr_DM[i][j];
white_designMatrix[i+off_r][j] = psr_wDM[i][j];
}
}
// the regular fit parameters
for(i=0; i < n[ipsr]; i++){
for(j=0; j < nf[ipsr]; j++){
designMatrix[i+off_r][j+off_f+nglobal] = psr_DM[i][j+nglobal];
white_designMatrix[i+off_r][j+off_f+nglobal] = psr_wDM[i][j+nglobal];
}
}
// the residuals
for(i=0; i < n[ipsr]; i++){
b[i+off_r] = psr_b[i];
white_b[i+off_r] = psr_wb[i];
}
if(psr[ipsr].nconstraints > 0){
logmsg("Get constraint weights");
computeConstraintWeights(psr+ipsr);
logmsg("fill constraints matrix");
for (int ic=0; ic < psr[ipsr].nconstraints; ic++){
CONSTRAINTfuncs(psr,ipsr,nf[ipsr],psr->constraints[ic],constraintsMatrix[ic+off_c]+off_f);
}
}
// increment the offset.
off_r += n[ipsr];
off_f += nf[ipsr];
off_c += psr[ipsr].nconstraints;
// free temp matricies.
free_blas(psr_DM);
free_blas(psr_wDM);
free(psr_b);
free(psr_wb);
}
// go ahead and do the fit!
*chisq = TKrobustConstrainedLeastSquares(b,white_b,designMatrix,white_designMatrix,
constraintsMatrix,
totalObs,totalFit,totalConstraints,tol,rescale_errors,
outP,e,cvm,psr[0].robust);
free_blas(designMatrix); // free-TKleastSquares_svd_psr_dcm-designMatrix**
free_blas(white_designMatrix); // free-TKleastSquares_svd_psr_dcm-white_designMatrix**
free_blas(constraintsMatrix); // free-TKleastSquares_svd_psr_dcm-white_designMatrix**
free(b);
free(white_b);
}
void t2Fit(pulsar *psr,unsigned int npsr, const char *covarFuncFile){
// if we have a model for the data covariance function, then use it.
// Otherwise we we will just whiten using the error bars.
bool haveCovar = (covarFuncFile!=NULL && strcmp(covarFuncFile,"NULL"));
/**
* Find out if there are any global parameters and what they are...
*/
FitInfo global_fitinfo;
t2Fit_fillGlobalFitInfo(psr,npsr,global_fitinfo);
logdbg("Nglobal parameters = %d",global_fitinfo.nParams);
// If we had any global parameters (or constraints) then we need to do a global fit
// otherwise we can do a fit for each pulsar individually, which is quicker
// and saves memory.
bool doGlobalFit = (global_fitinfo.nParams > 0) || (global_fitinfo.nConstraints > 0);
unsigned long long totalGlobalData=0; // the number of data points across all pulsars
unsigned int gParams=global_fitinfo.nParams; // the number of global fit parameters
unsigned int gConstraints=global_fitinfo.nConstraints; // the number of global constraints
unsigned long long totalGlobalParams=gParams;
unsigned long long totalGlobalConstraints=gConstraints;
double** gUinvs[MAX_PSR]; // whitening matrix for each pulsar
double* gX[MAX_PSR]; // "x" values for each pulsar
double* gY[MAX_PSR]; // "y" values for each pulsar
double* gW[MAX_PSR]; // whitened "y" values for each pulsar
double** gDM[MAX_PSR]; // design matrix for each pulsar
double** gWDM[MAX_PSR]; // whitened design matrix for each pulsar
double** gCM[MAX_PSR]; // constraints matrix for each pulsar
unsigned int gNdata[MAX_PSR]; // number of data points for each pulsar (size of x and y)
logmsg("NEW fit routine. GlobalFit=%s",doGlobalFit ? "true" : "false");
/**
* However we are going to do the fit, we want to loop over all the pulsars
* to get the input data and design matricies etc.
*/
for (size_t ipsr=0; ipsr < npsr; ipsr++) {
double *psr_x = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_white_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_e = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
int *psr_toaidx = (int*)malloc(sizeof(int)*psr[ipsr].nobs); // mapping from fit data to observation number
double** uinv; // the whitening matrix.
/**
* Working out which data contributes to the fit is done in this routine.
* Basically gets values for all observations within START and FINISH which are
* not deleted.
*
* returns the number of data points.
*/
const unsigned int psr_ndata = t2Fit_getFitData(psr+ipsr,psr_x,psr_y,psr_e,psr_toaidx);
assert(psr_ndata > 0u);
psr[ipsr].nFit = psr_ndata; // pulsar.nFit is the number of data points used in the fit.
/**
* Now we work out which parameters are being fit for, how many parameters,
* and determine the gradient functions for the design matrix and the update functions
* which update the pulsar struct.
*/
t2Fit_fillFitInfo(psr+ipsr,psr[ipsr].fitinfo,global_fitinfo);
/**
* The whitening matrix behaves diferently if we have a covariance matrix.
* If we have a covariance matrix, uinv is an ndata x ndata triangular matrix.
* Otherwise, it only has diagonal elements, so we efficiently store it as
* a 1-d ndata array.
*/
if (haveCovar) {
// ToAs must be sorted for covariance function code
sortToAs(psr+ipsr);
// malloc_uinv does a blas-compatible allocation of a 2-d array.
uinv = malloc_uinv(psr_ndata);
psr[ipsr].fitMode=1; // Note: forcing this to 1 as the Cholesky fit is a weighted fit
logmsg("Doing a FULL COVARIANCE MATRIX fit");
} else {
// Here the whitening matrix is just a diagonal
// weighting matrix. Store diagonal matrix as 1xN
// so that types match later.
uinv=malloc_blas(1,psr_ndata);
if(psr[ipsr].fitMode == 0){
// if we are doing an unweighted fit then we should set the errors to 1.0
// to give uniform weighting.
logdbg("Doing an UNWEIGHTED fit");
for (unsigned int i=0; i < psr_ndata; i++){
psr_e[i]=1.0;
}
} else {
logdbg("Doing a WEIGHTED fit");
}
}
assert(uinv!=NULL);
/**
* Now we form the whitening matrix, uinv.
* Note that getCholeskyMatrix() is clever enough to see that we
* have created a 1 x ndata matrix if we have only diagonal elements.
*/
getCholeskyMatrix(uinv,covarFuncFile,psr+ipsr,
psr_x,psr_y,psr_e,
psr_ndata,0,psr_toaidx);
logtchk("got Uinv");
// define some convinience variables
const unsigned nParams=psr[ipsr].fitinfo.nParams;
const unsigned nConstraints=psr[ipsr].fitinfo.nConstraints;
/**
* The design matrix is the matrix of gradients for the least-squares.
* If the design matrix is M, parameters p, and data d, we are solving
* M.p = d
* It is ndata x nparams in size. We also allocate the whitened DM here.
*/
double** designMatrix = malloc_blas(psr_ndata,nParams);
double** white_designMatrix = malloc_blas(psr_ndata,nParams);
for (unsigned int idata =0; idata < psr_ndata; ++idata){
// t2Fit_buildDesignMatrix is a replacement for the old FITfuncs routine.
// it fills one row of the design matrix.
t2Fit_buildDesignMatrix(psr,ipsr,psr_x[idata], psr_toaidx[idata], designMatrix[idata]);
}
logtchk("made design matrix");
/**
* The constraints matrix is similar to the design matrix, but here we are solving:
* B.p = 0
* Where B is the constraints matrix and p is the parameters. we solve both this
* and the DM equation set simultaniously. TKleastSquares will do this for us.
*
* If there are no constraints we leave it as NULL, which is detected in TKfit as
* no constraints anyway.
*/
double** constraintsMatrix =NULL;
if(psr[ipsr].fitinfo.nConstraints > 0){
computeConstraintWeights(psr+ipsr);
constraintsMatrix = malloc_blas(nConstraints,nParams);
for (unsigned int iconstraint =0; iconstraint < nConstraints; ++iconstraint){
// similar to t2Fit_buildDesignMatrix, t2Fit_buildConstraintsMatrix
// creates one row of the constraints matrix.
t2Fit_buildConstraintsMatrix(psr, ipsr, iconstraint, constraintsMatrix[iconstraint]);
}
}
logtchk("made constraints matrix");
/**
* Now we multiply the design matrix and the data vector by the whitening matrix.
* If we just have variances (uinv is diagonal) then we do it traditionally, otherwise
* we use TKmultMatrix as this is usually backed by LAPACK and so is fast :)
*/
if(haveCovar){
TKmultMatrixVec(uinv,psr_y,psr_ndata,psr_ndata,psr_white_y);
TKmultMatrix_sq(uinv,designMatrix,psr_ndata,nParams,white_designMatrix);
} else {
for(unsigned i=0;i<psr_ndata;++i){
psr_white_y[i]=psr_y[i]*uinv[0][i];
for(unsigned j=0;j<nParams;++j){
white_designMatrix[i][j] = designMatrix[i][j]*uinv[0][i];
}
}
}
free_blas(uinv);
free(psr_e);
free(psr_toaidx);
logtchk("done whitening");
/*
* Now - if we are going to do a global fit, we store all the above for later
* otherwise
*/
if (doGlobalFit){
// we are going to do a global fit, so need to store the values for later
gX[ipsr] = psr_x;
gY[ipsr] = psr_y;
gW[ipsr] = psr_white_y;
gDM[ipsr] = designMatrix;
gWDM[ipsr] = white_designMatrix;
gCM[ipsr] = constraintsMatrix;
gNdata[ipsr] = psr_ndata;
totalGlobalData += psr_ndata;
totalGlobalParams += nParams - gParams;
totalGlobalConstraints += nConstraints - gConstraints;
} else {
// NOT GLOBAL
// so do one fit at a time...
double chisq; // the post-fit chi-squared
// allocate memory for the output of TKleastSquares
double* parameterEstimates = (double*)malloc(sizeof(double)*nParams);
double* errorEstimates = (double*)malloc(sizeof(double)*nParams);
/*
* Call TKleastSquares, or in fact, TKrobustConstrainedLeastSquares,
* since we might want robust fitting and/or constraints/
*
* The arguments here are explained in TKfit.C
*
*/
chisq = TKrobustConstrainedLeastSquares(psr_y,psr_white_y,
designMatrix,white_designMatrix,constraintsMatrix,
psr_ndata,nParams,nConstraints,
T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[ipsr].covar,
psr[ipsr].robust);
// update the pulsar struct as appropriate
psr[ipsr].fitChisq = chisq;
psr[ipsr].fitNfree = psr_ndata + nConstraints - nParams;
logdbg("Updating the parameters");
logtchk("updating the parameter values");
/*
* This routine calls the appropriate update functions to apply the result of the fit
* to the origianal (non-linearised) pulsar parameters.
*/
t2Fit_updateParameters(psr,ipsr,parameterEstimates,errorEstimates);
logtchk("complete updating the parameter values");
logdbg("Completed updating the parameters");
/*
* If we are not doing a global fit, we can clean up the memory for this pulsar.
* Might make a difference for very large datasets.
*/
logdbg("Free fit memory");
free(parameterEstimates);
free(errorEstimates);
free_blas(designMatrix);
free_blas(white_designMatrix);
if (constraintsMatrix) free_blas(constraintsMatrix);
free(psr_x);
free(psr_y);
free(psr_white_y);
}
}
if (doGlobalFit){
const unsigned int nobs = totalGlobalData;
double** designMatrix = malloc_blas(nobs,totalGlobalParams);
double** white_designMatrix = malloc_blas(nobs,totalGlobalParams);
double** constraintsMatrix = malloc_blas(totalGlobalConstraints,totalGlobalParams);
double *y = (double*)malloc(sizeof(double)*nobs);
double *white_y = (double*)malloc(sizeof(double)*nobs);
unsigned int off_f = gParams; // leave space for globals
unsigned int off_r = 0;
unsigned int off_c = gConstraints;
logdbg("Building matricies for global fit... npsr=%u",npsr);
logdbg("nobs=%u, totalGlobalParams=%u, totalGlobalConstraints=%u",nobs,totalGlobalParams,totalGlobalConstraints);
logwarn("This mode is not supported yet!!!");
for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){
unsigned int nLocal = psr[ipsr].fitinfo.nParams-gParams;
logdbg("ipsr=%u, off_r = %u, off_c=%u, off_f=%u, nlocal=%u",
ipsr,off_r,off_c,off_f,nLocal);
// the fit parameters
for(unsigned int i=0; i < gNdata[ipsr]; i++){
// the global params (they go first)
for(unsigned int g= 0; g < gParams; g++){
unsigned int j = g+nLocal;
if(ipsr==0 && i==0 && writeResiduals){
logmsg("Row %d = %s %s(%d)",g,"global",label_str[global_fitinfo.paramIndex[g]],global_fitinfo.paramCounters[g]);
}
designMatrix[i+off_r][g] = gDM[ipsr][i][j];
white_designMatrix[i+off_r][g] = gWDM[ipsr][i][j];
}
for(unsigned int j=0; j < nLocal; j++){
if(i==0 && writeResiduals){
logmsg("Row %d = %s %s(%d)",j+off_f,psr[ipsr].name,label_str[psr[ipsr].fitinfo.paramIndex[j]],psr[ipsr].fitinfo.paramCounters[j]);
}
designMatrix[i+off_r][j+off_f] = gDM[ipsr][i][j];
white_designMatrix[i+off_r][j+off_f] = gWDM[ipsr][i][j];
}
}
// the data
for(unsigned int i=0; i < gNdata[ipsr]; ++i){
y[i+off_r] = gY[ipsr][i];
white_y[i+off_r] = gW[ipsr][i];
}
for(unsigned int i=0; i < psr[ipsr].fitinfo.nConstraints; i++){
for(unsigned int j=0; j < nLocal; j++){
constraintsMatrix[i+off_c][j+off_f] = gCM[ipsr][i][j];
}
// the global params (they go first)
for(unsigned int g= 0; g < gParams; g++){
unsigned int j = g+nLocal;
constraintsMatrix[i+off_c][g] = gCM[ipsr][i][j];
}
}
off_r += gNdata[ipsr];
off_f += nLocal;
off_c += psr[ipsr].fitinfo.nConstraints;
free(gY[ipsr]);
free(gW[ipsr]);
free_blas(gDM[ipsr]);
if (gCM[ipsr]) free_blas(gCM[ipsr]);
free_blas(gWDM[ipsr]);
}
double chisq; // the post-fit chi-squared
double* parameterEstimates = (double*)malloc(sizeof(double)*totalGlobalParams);
double* errorEstimates = (double*)malloc(sizeof(double)*totalGlobalParams);
chisq = TKrobustConstrainedLeastSquares(y,white_y,
designMatrix,white_designMatrix,constraintsMatrix,
nobs,totalGlobalParams,totalGlobalConstraints,
T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[0].covar,
psr[0].robust);
// for now the CVM ends up in psr[0].covar.
int off_p = gParams;
for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){
// update the pulsar struct as appropriate
psr[ipsr].fitChisq = chisq;
psr[ipsr].fitNfree = nobs + totalGlobalConstraints - totalGlobalParams;
double* psr_parameterEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams);
double* psr_errorEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams);
const unsigned np = psr[ipsr].fitinfo.nParams-gParams;
/* extract the fit output for the individual pulsars.
* I.e. detangle the global fit
* Notice: Globals go at the end of the individual pulsar arrays.
*/
for (unsigned i = 0; i < np; ++i){
psr_parameterEstimates[i] = parameterEstimates[off_p];
psr_errorEstimates[i] = errorEstimates[off_p];
++off_p;
}
for (unsigned i = 0; i < gParams; ++i){
psr_parameterEstimates[i+np] = parameterEstimates[i];
psr_errorEstimates[i] = errorEstimates[off_p];
}
logdbg("Updating the parameters");
logtchk("updating the parameter values");
/*
* This routine calls the appropriate update functions to apply the result of the fit
* to the origianal (non-linearised) pulsar parameters.
*/
t2Fit_updateParameters(psr,ipsr,psr_parameterEstimates,psr_errorEstimates);
logtchk("complete updating the parameter values");
logdbg("Completed updating the parameters");
free(psr_parameterEstimates);
free(psr_errorEstimates);
}
free(parameterEstimates);
free(errorEstimates);
free(white_y);
free(y);
free_blas(designMatrix);
free_blas(white_designMatrix);
free_blas(constraintsMatrix);
}
}
