void matrixDMConstraintWeights(pulsar *psr){
int i,j,k;
int nobs=0;
int nfit=psr->dmoffsDMnum+psr->dmoffsCMnum;
int nDM=psr->dmoffsDMnum;
double x;
if (psr->param[param_dmmodel].fitFlag[0]==1)
{
logdbg("Getting DM constraints for %s",psr->name);
// find out how many obs we have.
for(i=0; i < psr->nobs; i++){
if (psr->obsn[i].deleted==0)
{
char okay=1;
/* Check for START and FINISH flags */
if (psr->param[param_start].paramSet[0]==1 && psr->param[param_start].fitFlag[0]==1 &&
(psr->param[param_start].val[0] > psr->obsn[i].sat))
okay=0;
if (psr->param[param_finish].paramSet[0]==1 && psr->param[param_finish].fitFlag[0]==1 &&
psr->param[param_finish].val[0] < psr->obsn[i].sat)
okay=0;
if (okay==1)
{
nobs++;
}
}
}
// originally was sizeof(double*)*nobs.
double ** designMatrix=(double**)malloc_uinv(nobs);
double *e=(double*)malloc(sizeof(double)*nobs);
nobs=0;
// printf("c0: %d %s\n",psr->nobs,psr->name);
for(i=0; i < psr->nobs; i++){
// Check for "ok" and start/finish coppied from dofit.
// Needs to be the same so that the weighting is the same as the fit,
// but in practice probably doesn't matter.
// printf("c1 checking: %d %d %s\n",psr->obsn[i].deleted,psr->nobs,psr->name);
if (psr->obsn[i].deleted==0)
{
char okay=1;
/* Check for START and FINISH flags */
if (psr->param[param_start].paramSet[0]==1 && psr->param[param_start].fitFlag[0]==1 &&
(psr->param[param_start].val[0] > psr->obsn[i].sat))
okay=0;
if (psr->param[param_finish].paramSet[0]==1 && psr->param[param_finish].fitFlag[0]==1 &&
psr->param[param_finish].val[0] < psr->obsn[i].sat)
okay=0;
// printf("c2 checking: %d\n",okay);
if (okay==1)
{
x = (double)(psr->obsn[i].bbat-psr->param[param_pepoch].val[0]);
double sig=psr->obsn[i].toaErr*1e-6;
double dmf = 1.0/(DM_CONST*powl(psr->obsn[i].freqSSB/1.0e6,2));
for (k=0; k < nfit;k++){
if(k<nDM)
designMatrix[nobs][k]=dmf*getParamDeriv(psr,i,x,param_dmmodel,k)/sig;
else
designMatrix[nobs][k]=getParamDeriv(psr,i,x,param_dmmodel,k)/sig;
}
nobs++;
}
}
}
printf("Calling TKleastSquares %d %d\n",nobs,nfit); fflush(stdout);
TKleastSquares(NULL,NULL,designMatrix,designMatrix,nobs,nfit,1e-20,0,NULL,e,NULL);
printf("Returning from calling TKleastSquares\n"); fflush(stdout);
double sum_wDM=0;
double sum_wCM=0;
for (i=0;i<nfit;i++)
{
double sum=0.0;
if(i < nDM){
psr->dmoffsDM_weight[i]=1.0/e[i]/e[i];
printf("constraints.C here: %d %g\n",i,e[i]);
sum_wDM+=psr->dmoffsDM_weight[i];
} else {
psr->dmoffsCM_weight[i-nDM]=1.0/e[i]/e[i];
sum_wCM+=psr->dmoffsCM_weight[i-nDM];
}
}
//normalise the weights
for (i=0;i<psr->dmoffsDMnum;i++)
psr->dmoffsDM_weight[i]/=sum_wDM;
for (i=0;i<psr->dmoffsCMnum;i++)
psr->dmoffsCM_weight[i]/=sum_wCM;
// free everything .
free_uinv(designMatrix);
free(e);
}
}
void t2Fit(pulsar *psr,unsigned int npsr, const char *covarFuncFile){
// if we have a model for the data covariance function, then use it.
// Otherwise we we will just whiten using the error bars.
bool haveCovar = (covarFuncFile!=NULL && strcmp(covarFuncFile,"NULL"));
/**
* Find out if there are any global parameters and what they are...
*/
FitInfo global_fitinfo;
t2Fit_fillGlobalFitInfo(psr,npsr,global_fitinfo);
logdbg("Nglobal parameters = %d",global_fitinfo.nParams);
// If we had any global parameters (or constraints) then we need to do a global fit
// otherwise we can do a fit for each pulsar individually, which is quicker
// and saves memory.
bool doGlobalFit = (global_fitinfo.nParams > 0) || (global_fitinfo.nConstraints > 0);
unsigned long long totalGlobalData=0; // the number of data points across all pulsars
unsigned int gParams=global_fitinfo.nParams; // the number of global fit parameters
unsigned int gConstraints=global_fitinfo.nConstraints; // the number of global constraints
unsigned long long totalGlobalParams=gParams;
unsigned long long totalGlobalConstraints=gConstraints;
double** gUinvs[MAX_PSR]; // whitening matrix for each pulsar
double* gX[MAX_PSR]; // "x" values for each pulsar
double* gY[MAX_PSR]; // "y" values for each pulsar
double* gW[MAX_PSR]; // whitened "y" values for each pulsar
double** gDM[MAX_PSR]; // design matrix for each pulsar
double** gWDM[MAX_PSR]; // whitened design matrix for each pulsar
double** gCM[MAX_PSR]; // constraints matrix for each pulsar
unsigned int gNdata[MAX_PSR]; // number of data points for each pulsar (size of x and y)
logmsg("NEW fit routine. GlobalFit=%s",doGlobalFit ? "true" : "false");
/**
* However we are going to do the fit, we want to loop over all the pulsars
* to get the input data and design matricies etc.
*/
for (size_t ipsr=0; ipsr < npsr; ipsr++) {
double *psr_x = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_white_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
double *psr_e = (double*)malloc(sizeof(double)*psr[ipsr].nobs);
int *psr_toaidx = (int*)malloc(sizeof(int)*psr[ipsr].nobs); // mapping from fit data to observation number
double** uinv; // the whitening matrix.
/**
* Working out which data contributes to the fit is done in this routine.
* Basically gets values for all observations within START and FINISH which are
* not deleted.
*
* returns the number of data points.
*/
const unsigned int psr_ndata = t2Fit_getFitData(psr+ipsr,psr_x,psr_y,psr_e,psr_toaidx);
assert(psr_ndata > 0u);
psr[ipsr].nFit = psr_ndata; // pulsar.nFit is the number of data points used in the fit.
/**
* Now we work out which parameters are being fit for, how many parameters,
* and determine the gradient functions for the design matrix and the update functions
* which update the pulsar struct.
*/
t2Fit_fillFitInfo(psr+ipsr,psr[ipsr].fitinfo,global_fitinfo);
/**
* The whitening matrix behaves diferently if we have a covariance matrix.
* If we have a covariance matrix, uinv is an ndata x ndata triangular matrix.
* Otherwise, it only has diagonal elements, so we efficiently store it as
* a 1-d ndata array.
*/
if (haveCovar) {
// ToAs must be sorted for covariance function code
sortToAs(psr+ipsr);
// malloc_uinv does a blas-compatible allocation of a 2-d array.
uinv = malloc_uinv(psr_ndata);
psr[ipsr].fitMode=1; // Note: forcing this to 1 as the Cholesky fit is a weighted fit
logmsg("Doing a FULL COVARIANCE MATRIX fit");
} else {
// Here the whitening matrix is just a diagonal
// weighting matrix. Store diagonal matrix as 1xN
// so that types match later.
uinv=malloc_blas(1,psr_ndata);
if(psr[ipsr].fitMode == 0){
// if we are doing an unweighted fit then we should set the errors to 1.0
// to give uniform weighting.
logdbg("Doing an UNWEIGHTED fit");
for (unsigned int i=0; i < psr_ndata; i++){
psr_e[i]=1.0;
}
} else {
logdbg("Doing a WEIGHTED fit");
}
}
assert(uinv!=NULL);
/**
* Now we form the whitening matrix, uinv.
* Note that getCholeskyMatrix() is clever enough to see that we
* have created a 1 x ndata matrix if we have only diagonal elements.
*/
getCholeskyMatrix(uinv,covarFuncFile,psr+ipsr,
psr_x,psr_y,psr_e,
psr_ndata,0,psr_toaidx);
logtchk("got Uinv");
// define some convinience variables
const unsigned nParams=psr[ipsr].fitinfo.nParams;
const unsigned nConstraints=psr[ipsr].fitinfo.nConstraints;
/**
* The design matrix is the matrix of gradients for the least-squares.
* If the design matrix is M, parameters p, and data d, we are solving
* M.p = d
* It is ndata x nparams in size. We also allocate the whitened DM here.
*/
double** designMatrix = malloc_blas(psr_ndata,nParams);
double** white_designMatrix = malloc_blas(psr_ndata,nParams);
for (unsigned int idata =0; idata < psr_ndata; ++idata){
// t2Fit_buildDesignMatrix is a replacement for the old FITfuncs routine.
// it fills one row of the design matrix.
t2Fit_buildDesignMatrix(psr,ipsr,psr_x[idata], psr_toaidx[idata], designMatrix[idata]);
}
logtchk("made design matrix");
/**
* The constraints matrix is similar to the design matrix, but here we are solving:
* B.p = 0
* Where B is the constraints matrix and p is the parameters. we solve both this
* and the DM equation set simultaniously. TKleastSquares will do this for us.
*
* If there are no constraints we leave it as NULL, which is detected in TKfit as
* no constraints anyway.
*/
double** constraintsMatrix =NULL;
if(psr[ipsr].fitinfo.nConstraints > 0){
computeConstraintWeights(psr+ipsr);
constraintsMatrix = malloc_blas(nConstraints,nParams);
for (unsigned int iconstraint =0; iconstraint < nConstraints; ++iconstraint){
// similar to t2Fit_buildDesignMatrix, t2Fit_buildConstraintsMatrix
// creates one row of the constraints matrix.
t2Fit_buildConstraintsMatrix(psr, ipsr, iconstraint, constraintsMatrix[iconstraint]);
}
}
logtchk("made constraints matrix");
/**
* Now we multiply the design matrix and the data vector by the whitening matrix.
* If we just have variances (uinv is diagonal) then we do it traditionally, otherwise
* we use TKmultMatrix as this is usually backed by LAPACK and so is fast :)
*/
if(haveCovar){
TKmultMatrixVec(uinv,psr_y,psr_ndata,psr_ndata,psr_white_y);
TKmultMatrix_sq(uinv,designMatrix,psr_ndata,nParams,white_designMatrix);
} else {
for(unsigned i=0;i<psr_ndata;++i){
psr_white_y[i]=psr_y[i]*uinv[0][i];
for(unsigned j=0;j<nParams;++j){
white_designMatrix[i][j] = designMatrix[i][j]*uinv[0][i];
}
}
}
free_blas(uinv);
free(psr_e);
free(psr_toaidx);
logtchk("done whitening");
/*
* Now - if we are going to do a global fit, we store all the above for later
* otherwise
*/
if (doGlobalFit){
// we are going to do a global fit, so need to store the values for later
gX[ipsr] = psr_x;
gY[ipsr] = psr_y;
gW[ipsr] = psr_white_y;
gDM[ipsr] = designMatrix;
gWDM[ipsr] = white_designMatrix;
gCM[ipsr] = constraintsMatrix;
gNdata[ipsr] = psr_ndata;
totalGlobalData += psr_ndata;
totalGlobalParams += nParams - gParams;
totalGlobalConstraints += nConstraints - gConstraints;
} else {
// NOT GLOBAL
// so do one fit at a time...
double chisq; // the post-fit chi-squared
// allocate memory for the output of TKleastSquares
double* parameterEstimates = (double*)malloc(sizeof(double)*nParams);
double* errorEstimates = (double*)malloc(sizeof(double)*nParams);
/*
* Call TKleastSquares, or in fact, TKrobustConstrainedLeastSquares,
* since we might want robust fitting and/or constraints/
*
* The arguments here are explained in TKfit.C
*
*/
chisq = TKrobustConstrainedLeastSquares(psr_y,psr_white_y,
designMatrix,white_designMatrix,constraintsMatrix,
psr_ndata,nParams,nConstraints,
T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[ipsr].covar,
psr[ipsr].robust);
// update the pulsar struct as appropriate
psr[ipsr].fitChisq = chisq;
psr[ipsr].fitNfree = psr_ndata + nConstraints - nParams;
logdbg("Updating the parameters");
logtchk("updating the parameter values");
/*
* This routine calls the appropriate update functions to apply the result of the fit
* to the origianal (non-linearised) pulsar parameters.
*/
t2Fit_updateParameters(psr,ipsr,parameterEstimates,errorEstimates);
logtchk("complete updating the parameter values");
logdbg("Completed updating the parameters");
/*
* If we are not doing a global fit, we can clean up the memory for this pulsar.
* Might make a difference for very large datasets.
*/
logdbg("Free fit memory");
free(parameterEstimates);
free(errorEstimates);
free_blas(designMatrix);
free_blas(white_designMatrix);
if (constraintsMatrix) free_blas(constraintsMatrix);
free(psr_x);
free(psr_y);
free(psr_white_y);
}
}
if (doGlobalFit){
const unsigned int nobs = totalGlobalData;
double** designMatrix = malloc_blas(nobs,totalGlobalParams);
double** white_designMatrix = malloc_blas(nobs,totalGlobalParams);
double** constraintsMatrix = malloc_blas(totalGlobalConstraints,totalGlobalParams);
double *y = (double*)malloc(sizeof(double)*nobs);
double *white_y = (double*)malloc(sizeof(double)*nobs);
unsigned int off_f = gParams; // leave space for globals
unsigned int off_r = 0;
unsigned int off_c = gConstraints;
logdbg("Building matricies for global fit... npsr=%u",npsr);
logdbg("nobs=%u, totalGlobalParams=%u, totalGlobalConstraints=%u",nobs,totalGlobalParams,totalGlobalConstraints);
logwarn("This mode is not supported yet!!!");
for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){
unsigned int nLocal = psr[ipsr].fitinfo.nParams-gParams;
logdbg("ipsr=%u, off_r = %u, off_c=%u, off_f=%u, nlocal=%u",
ipsr,off_r,off_c,off_f,nLocal);
// the fit parameters
for(unsigned int i=0; i < gNdata[ipsr]; i++){
// the global params (they go first)
for(unsigned int g= 0; g < gParams; g++){
unsigned int j = g+nLocal;
if(ipsr==0 && i==0 && writeResiduals){
logmsg("Row %d = %s %s(%d)",g,"global",label_str[global_fitinfo.paramIndex[g]],global_fitinfo.paramCounters[g]);
}
designMatrix[i+off_r][g] = gDM[ipsr][i][j];
white_designMatrix[i+off_r][g] = gWDM[ipsr][i][j];
}
for(unsigned int j=0; j < nLocal; j++){
if(i==0 && writeResiduals){
logmsg("Row %d = %s %s(%d)",j+off_f,psr[ipsr].name,label_str[psr[ipsr].fitinfo.paramIndex[j]],psr[ipsr].fitinfo.paramCounters[j]);
}
designMatrix[i+off_r][j+off_f] = gDM[ipsr][i][j];
white_designMatrix[i+off_r][j+off_f] = gWDM[ipsr][i][j];
}
}
// the data
for(unsigned int i=0; i < gNdata[ipsr]; ++i){
y[i+off_r] = gY[ipsr][i];
white_y[i+off_r] = gW[ipsr][i];
}
for(unsigned int i=0; i < psr[ipsr].fitinfo.nConstraints; i++){
for(unsigned int j=0; j < nLocal; j++){
constraintsMatrix[i+off_c][j+off_f] = gCM[ipsr][i][j];
}
// the global params (they go first)
for(unsigned int g= 0; g < gParams; g++){
unsigned int j = g+nLocal;
constraintsMatrix[i+off_c][g] = gCM[ipsr][i][j];
}
}
off_r += gNdata[ipsr];
off_f += nLocal;
off_c += psr[ipsr].fitinfo.nConstraints;
free(gY[ipsr]);
free(gW[ipsr]);
free_blas(gDM[ipsr]);
if (gCM[ipsr]) free_blas(gCM[ipsr]);
free_blas(gWDM[ipsr]);
}
double chisq; // the post-fit chi-squared
double* parameterEstimates = (double*)malloc(sizeof(double)*totalGlobalParams);
double* errorEstimates = (double*)malloc(sizeof(double)*totalGlobalParams);
chisq = TKrobustConstrainedLeastSquares(y,white_y,
designMatrix,white_designMatrix,constraintsMatrix,
nobs,totalGlobalParams,totalGlobalConstraints,
T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[0].covar,
psr[0].robust);
// for now the CVM ends up in psr[0].covar.
int off_p = gParams;
for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){
// update the pulsar struct as appropriate
psr[ipsr].fitChisq = chisq;
psr[ipsr].fitNfree = nobs + totalGlobalConstraints - totalGlobalParams;
double* psr_parameterEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams);
double* psr_errorEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams);
const unsigned np = psr[ipsr].fitinfo.nParams-gParams;
/* extract the fit output for the individual pulsars.
* I.e. detangle the global fit
* Notice: Globals go at the end of the individual pulsar arrays.
*/
for (unsigned i = 0; i < np; ++i){
psr_parameterEstimates[i] = parameterEstimates[off_p];
psr_errorEstimates[i] = errorEstimates[off_p];
++off_p;
}
for (unsigned i = 0; i < gParams; ++i){
psr_parameterEstimates[i+np] = parameterEstimates[i];
psr_errorEstimates[i] = errorEstimates[off_p];
}
logdbg("Updating the parameters");
logtchk("updating the parameter values");
/*
* This routine calls the appropriate update functions to apply the result of the fit
* to the origianal (non-linearised) pulsar parameters.
*/
t2Fit_updateParameters(psr,ipsr,psr_parameterEstimates,psr_errorEstimates);
logtchk("complete updating the parameter values");
logdbg("Completed updating the parameters");
free(psr_parameterEstimates);
free(psr_errorEstimates);
}
free(parameterEstimates);
free(errorEstimates);
free(white_y);
free(y);
free_blas(designMatrix);
free_blas(white_designMatrix);
free_blas(constraintsMatrix);
}
}
/**
* Do the least squares using QR decomposition
*/
double accel_lsq_qr(double** A, double* data, double* oparam, int ndata, int nparam, double** Ocvm){
int nhrs=1;
int nwork = -1;
int info=0;
int i,j;
double iwork;
// workspace query - works out optimal size for work array
F77_dgels("T", &nparam, &ndata, &nhrs, A[0], &nparam, data, &ndata, &iwork, &nwork, &info);
nwork=(int)iwork;
logdbg("nwork = %d (%lf)",nwork,iwork);
double* work = static_cast<double*>(malloc(sizeof(double)*nwork));
logdbg("accel_lsq_qr ndata=%d nparam=%d",ndata,nparam);
// as usual we prentend that the matrix is transposed to deal with C->fortran conversion
// degls does a least-squares using QR decomposition. Fast and robust.
F77_dgels("T", &nparam, &ndata, &nhrs, A[0], &nparam, data, &ndata, work, &nwork, &info);
free(work);
// if info is not zero then the fit failed.
if(info!=0){
logerr("Error in lapack DEGLS. INFO=%d See full logs for explanation.",info);
logmsg("");
logmsg("From: http://www.netlib.org/lapack/explore-html/d8/dde/dgels_8f.html");
logmsg("INFO is INTEGER");
logmsg(" = 0: successful exit");
logmsg(" < 0: if INFO = -i, the i-th argument had an illegal value");
logmsg(" > 0: if INFO = i, the i-th diagonal element of the");
logmsg(" triangular factor of A is zero, so that A does not have");
logmsg(" full rank; the least squares solution could not be");
logmsg(" computed.");
logmsg("");
if(info > 0){
logerr("It appears that you are fitting for a 'bad' parameter - E.g A jump on a non-existant flag.");
logmsg(" TEMPO2 will NOT attempt to deal with this!");
logmsg("Cannot continue. Abort fit.");
return -1;
} else {
logmsg("Cannot continue. Abort fit.");
return -1;
}
}
assert(info==0);
if(oparam!=NULL){
// copy out the output parameters, which are written into the "data" array.
memcpy(oparam,data,sizeof(double)*nparam);
}
double chisq=0;
for( i = nparam; i < ndata; i++ ) chisq += data[i] * data[i];
if (Ocvm != NULL){
int n=nparam;
// packed triangular matrix.
double* _t=(double*)malloc(sizeof(double)*(n*(n+1))/2);
// This code taken from the LAPACK documentation
// to pack a triangular matrix.
// we want the upper triangular matrix part of A.
//
// pack upper triangle like (r,c)
// (1,1) (1,2) (2,2)
// i+(2n-j)(j-1)/2
int jc=0;
for (j=0;j<n;j++){ // cols
for (i=0; i <=j; i++) { // rows
_t[jc] = A[i][j]; // A came from fortran, so is in [col][row] ordering
// BUT - we have transposed A, so we have to un-transpose it
++jc;
}
}
logdbg("Inverting...");
F77_dtptri("U","N",&n,_t,&i);
if(i!=0){
logerr("Error in lapack DTPTRI. INFO=%d",i);
logmsg("From: http://www.netlib.org/lapack/explore-html/d8/d05/dtptri_8f.html");
logmsg("INFO is INTEGER");
logmsg(" = 0: successful exit");
logmsg(" < 0: if INFO = -i, the i-th argument had an illegal value");
logmsg(" > 0: if INFO = i, A(i,i) is exactly zero. The triangular");
logmsg(" matrix is singular and its inverse can not be computed.");
logmsg("Cannot continue - abort fit");
return -1;
}
double **Rinv = malloc_uinv(n);
for (j=0;j<n;j++){ // cols
for (i=0;i<n;i++){ //rows
Ocvm[i][j]=0;
}
}
// Unpack the triangular matrix using reverse of above
// We will put it in fortran, so continue to use [col][row] order
jc=0;
for (j=0;j<n;j++){ // cols
for (i=0; i <=j; i++) { // rows
Rinv[j][i] = _t[jc];
Ocvm[j][i] = _t[jc];
++jc;
}
}
free(_t);
double a=chisq/(double)(ndata-nparam);
// (X^T X)^-1 = Rinv.Rinv^T gives parameter covariance matrix
// Note that Ocvm is input and output
// and that covar matrix will be transposed, but it is
// symetric so it doesn't matter!
F77_dtrmm( "R", "U", "T", "N", &n, &n, &a, *Rinv, &n, *Ocvm, &n);
// DTRMM ( SIDE, UPLO, TRANSA, DIAG, M, N, ALPHA, A , LDA, B, LDB )
if(debugFlag){
for(i=0;i<n;i++){
for(j=0;j<n;j++){
logdbg("COVAR %d %d %lg",i,j,Ocvm[i][j]);
}
}
}
free_uinv(Rinv);
}
return chisq;
}
