void matrixDMConstraintWeights(pulsar *psr){ int i,j,k; int nobs=0; int nfit=psr->dmoffsDMnum+psr->dmoffsCMnum; int nDM=psr->dmoffsDMnum; double x; if (psr->param[param_dmmodel].fitFlag[0]==1) { logdbg("Getting DM constraints for %s",psr->name); // find out how many obs we have. for(i=0; i < psr->nobs; i++){ if (psr->obsn[i].deleted==0) { char okay=1; /* Check for START and FINISH flags */ if (psr->param[param_start].paramSet[0]==1 && psr->param[param_start].fitFlag[0]==1 && (psr->param[param_start].val[0] > psr->obsn[i].sat)) okay=0; if (psr->param[param_finish].paramSet[0]==1 && psr->param[param_finish].fitFlag[0]==1 && psr->param[param_finish].val[0] < psr->obsn[i].sat) okay=0; if (okay==1) { nobs++; } } } // originally was sizeof(double*)*nobs. double ** designMatrix=(double**)malloc_uinv(nobs); double *e=(double*)malloc(sizeof(double)*nobs); nobs=0; // printf("c0: %d %s\n",psr->nobs,psr->name); for(i=0; i < psr->nobs; i++){ // Check for "ok" and start/finish coppied from dofit. // Needs to be the same so that the weighting is the same as the fit, // but in practice probably doesn't matter. // printf("c1 checking: %d %d %s\n",psr->obsn[i].deleted,psr->nobs,psr->name); if (psr->obsn[i].deleted==0) { char okay=1; /* Check for START and FINISH flags */ if (psr->param[param_start].paramSet[0]==1 && psr->param[param_start].fitFlag[0]==1 && (psr->param[param_start].val[0] > psr->obsn[i].sat)) okay=0; if (psr->param[param_finish].paramSet[0]==1 && psr->param[param_finish].fitFlag[0]==1 && psr->param[param_finish].val[0] < psr->obsn[i].sat) okay=0; // printf("c2 checking: %d\n",okay); if (okay==1) { x = (double)(psr->obsn[i].bbat-psr->param[param_pepoch].val[0]); double sig=psr->obsn[i].toaErr*1e-6; double dmf = 1.0/(DM_CONST*powl(psr->obsn[i].freqSSB/1.0e6,2)); for (k=0; k < nfit;k++){ if(k<nDM) designMatrix[nobs][k]=dmf*getParamDeriv(psr,i,x,param_dmmodel,k)/sig; else designMatrix[nobs][k]=getParamDeriv(psr,i,x,param_dmmodel,k)/sig; } nobs++; } } } printf("Calling TKleastSquares %d %d\n",nobs,nfit); fflush(stdout); TKleastSquares(NULL,NULL,designMatrix,designMatrix,nobs,nfit,1e-20,0,NULL,e,NULL); printf("Returning from calling TKleastSquares\n"); fflush(stdout); double sum_wDM=0; double sum_wCM=0; for (i=0;i<nfit;i++) { double sum=0.0; if(i < nDM){ psr->dmoffsDM_weight[i]=1.0/e[i]/e[i]; printf("constraints.C here: %d %g\n",i,e[i]); sum_wDM+=psr->dmoffsDM_weight[i]; } else { psr->dmoffsCM_weight[i-nDM]=1.0/e[i]/e[i]; sum_wCM+=psr->dmoffsCM_weight[i-nDM]; } } //normalise the weights for (i=0;i<psr->dmoffsDMnum;i++) psr->dmoffsDM_weight[i]/=sum_wDM; for (i=0;i<psr->dmoffsCMnum;i++) psr->dmoffsCM_weight[i]/=sum_wCM; // free everything . free_uinv(designMatrix); free(e); } }
void t2Fit(pulsar *psr,unsigned int npsr, const char *covarFuncFile){ // if we have a model for the data covariance function, then use it. // Otherwise we we will just whiten using the error bars. bool haveCovar = (covarFuncFile!=NULL && strcmp(covarFuncFile,"NULL")); /** * Find out if there are any global parameters and what they are... */ FitInfo global_fitinfo; t2Fit_fillGlobalFitInfo(psr,npsr,global_fitinfo); logdbg("Nglobal parameters = %d",global_fitinfo.nParams); // If we had any global parameters (or constraints) then we need to do a global fit // otherwise we can do a fit for each pulsar individually, which is quicker // and saves memory. bool doGlobalFit = (global_fitinfo.nParams > 0) || (global_fitinfo.nConstraints > 0); unsigned long long totalGlobalData=0; // the number of data points across all pulsars unsigned int gParams=global_fitinfo.nParams; // the number of global fit parameters unsigned int gConstraints=global_fitinfo.nConstraints; // the number of global constraints unsigned long long totalGlobalParams=gParams; unsigned long long totalGlobalConstraints=gConstraints; double** gUinvs[MAX_PSR]; // whitening matrix for each pulsar double* gX[MAX_PSR]; // "x" values for each pulsar double* gY[MAX_PSR]; // "y" values for each pulsar double* gW[MAX_PSR]; // whitened "y" values for each pulsar double** gDM[MAX_PSR]; // design matrix for each pulsar double** gWDM[MAX_PSR]; // whitened design matrix for each pulsar double** gCM[MAX_PSR]; // constraints matrix for each pulsar unsigned int gNdata[MAX_PSR]; // number of data points for each pulsar (size of x and y) logmsg("NEW fit routine. GlobalFit=%s",doGlobalFit ? "true" : "false"); /** * However we are going to do the fit, we want to loop over all the pulsars * to get the input data and design matricies etc. */ for (size_t ipsr=0; ipsr < npsr; ipsr++) { double *psr_x = (double*)malloc(sizeof(double)*psr[ipsr].nobs); double *psr_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs); double *psr_white_y = (double*)malloc(sizeof(double)*psr[ipsr].nobs); double *psr_e = (double*)malloc(sizeof(double)*psr[ipsr].nobs); int *psr_toaidx = (int*)malloc(sizeof(int)*psr[ipsr].nobs); // mapping from fit data to observation number double** uinv; // the whitening matrix. /** * Working out which data contributes to the fit is done in this routine. * Basically gets values for all observations within START and FINISH which are * not deleted. * * returns the number of data points. */ const unsigned int psr_ndata = t2Fit_getFitData(psr+ipsr,psr_x,psr_y,psr_e,psr_toaidx); assert(psr_ndata > 0u); psr[ipsr].nFit = psr_ndata; // pulsar.nFit is the number of data points used in the fit. /** * Now we work out which parameters are being fit for, how many parameters, * and determine the gradient functions for the design matrix and the update functions * which update the pulsar struct. */ t2Fit_fillFitInfo(psr+ipsr,psr[ipsr].fitinfo,global_fitinfo); /** * The whitening matrix behaves diferently if we have a covariance matrix. * If we have a covariance matrix, uinv is an ndata x ndata triangular matrix. * Otherwise, it only has diagonal elements, so we efficiently store it as * a 1-d ndata array. */ if (haveCovar) { // ToAs must be sorted for covariance function code sortToAs(psr+ipsr); // malloc_uinv does a blas-compatible allocation of a 2-d array. uinv = malloc_uinv(psr_ndata); psr[ipsr].fitMode=1; // Note: forcing this to 1 as the Cholesky fit is a weighted fit logmsg("Doing a FULL COVARIANCE MATRIX fit"); } else { // Here the whitening matrix is just a diagonal // weighting matrix. Store diagonal matrix as 1xN // so that types match later. uinv=malloc_blas(1,psr_ndata); if(psr[ipsr].fitMode == 0){ // if we are doing an unweighted fit then we should set the errors to 1.0 // to give uniform weighting. logdbg("Doing an UNWEIGHTED fit"); for (unsigned int i=0; i < psr_ndata; i++){ psr_e[i]=1.0; } } else { logdbg("Doing a WEIGHTED fit"); } } assert(uinv!=NULL); /** * Now we form the whitening matrix, uinv. * Note that getCholeskyMatrix() is clever enough to see that we * have created a 1 x ndata matrix if we have only diagonal elements. */ getCholeskyMatrix(uinv,covarFuncFile,psr+ipsr, psr_x,psr_y,psr_e, psr_ndata,0,psr_toaidx); logtchk("got Uinv"); // define some convinience variables const unsigned nParams=psr[ipsr].fitinfo.nParams; const unsigned nConstraints=psr[ipsr].fitinfo.nConstraints; /** * The design matrix is the matrix of gradients for the least-squares. * If the design matrix is M, parameters p, and data d, we are solving * M.p = d * It is ndata x nparams in size. We also allocate the whitened DM here. */ double** designMatrix = malloc_blas(psr_ndata,nParams); double** white_designMatrix = malloc_blas(psr_ndata,nParams); for (unsigned int idata =0; idata < psr_ndata; ++idata){ // t2Fit_buildDesignMatrix is a replacement for the old FITfuncs routine. // it fills one row of the design matrix. t2Fit_buildDesignMatrix(psr,ipsr,psr_x[idata], psr_toaidx[idata], designMatrix[idata]); } logtchk("made design matrix"); /** * The constraints matrix is similar to the design matrix, but here we are solving: * B.p = 0 * Where B is the constraints matrix and p is the parameters. we solve both this * and the DM equation set simultaniously. TKleastSquares will do this for us. * * If there are no constraints we leave it as NULL, which is detected in TKfit as * no constraints anyway. */ double** constraintsMatrix =NULL; if(psr[ipsr].fitinfo.nConstraints > 0){ computeConstraintWeights(psr+ipsr); constraintsMatrix = malloc_blas(nConstraints,nParams); for (unsigned int iconstraint =0; iconstraint < nConstraints; ++iconstraint){ // similar to t2Fit_buildDesignMatrix, t2Fit_buildConstraintsMatrix // creates one row of the constraints matrix. t2Fit_buildConstraintsMatrix(psr, ipsr, iconstraint, constraintsMatrix[iconstraint]); } } logtchk("made constraints matrix"); /** * Now we multiply the design matrix and the data vector by the whitening matrix. * If we just have variances (uinv is diagonal) then we do it traditionally, otherwise * we use TKmultMatrix as this is usually backed by LAPACK and so is fast :) */ if(haveCovar){ TKmultMatrixVec(uinv,psr_y,psr_ndata,psr_ndata,psr_white_y); TKmultMatrix_sq(uinv,designMatrix,psr_ndata,nParams,white_designMatrix); } else { for(unsigned i=0;i<psr_ndata;++i){ psr_white_y[i]=psr_y[i]*uinv[0][i]; for(unsigned j=0;j<nParams;++j){ white_designMatrix[i][j] = designMatrix[i][j]*uinv[0][i]; } } } free_blas(uinv); free(psr_e); free(psr_toaidx); logtchk("done whitening"); /* * Now - if we are going to do a global fit, we store all the above for later * otherwise */ if (doGlobalFit){ // we are going to do a global fit, so need to store the values for later gX[ipsr] = psr_x; gY[ipsr] = psr_y; gW[ipsr] = psr_white_y; gDM[ipsr] = designMatrix; gWDM[ipsr] = white_designMatrix; gCM[ipsr] = constraintsMatrix; gNdata[ipsr] = psr_ndata; totalGlobalData += psr_ndata; totalGlobalParams += nParams - gParams; totalGlobalConstraints += nConstraints - gConstraints; } else { // NOT GLOBAL // so do one fit at a time... double chisq; // the post-fit chi-squared // allocate memory for the output of TKleastSquares double* parameterEstimates = (double*)malloc(sizeof(double)*nParams); double* errorEstimates = (double*)malloc(sizeof(double)*nParams); /* * Call TKleastSquares, or in fact, TKrobustConstrainedLeastSquares, * since we might want robust fitting and/or constraints/ * * The arguments here are explained in TKfit.C * */ chisq = TKrobustConstrainedLeastSquares(psr_y,psr_white_y, designMatrix,white_designMatrix,constraintsMatrix, psr_ndata,nParams,nConstraints, T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[ipsr].covar, psr[ipsr].robust); // update the pulsar struct as appropriate psr[ipsr].fitChisq = chisq; psr[ipsr].fitNfree = psr_ndata + nConstraints - nParams; logdbg("Updating the parameters"); logtchk("updating the parameter values"); /* * This routine calls the appropriate update functions to apply the result of the fit * to the origianal (non-linearised) pulsar parameters. */ t2Fit_updateParameters(psr,ipsr,parameterEstimates,errorEstimates); logtchk("complete updating the parameter values"); logdbg("Completed updating the parameters"); /* * If we are not doing a global fit, we can clean up the memory for this pulsar. * Might make a difference for very large datasets. */ logdbg("Free fit memory"); free(parameterEstimates); free(errorEstimates); free_blas(designMatrix); free_blas(white_designMatrix); if (constraintsMatrix) free_blas(constraintsMatrix); free(psr_x); free(psr_y); free(psr_white_y); } } if (doGlobalFit){ const unsigned int nobs = totalGlobalData; double** designMatrix = malloc_blas(nobs,totalGlobalParams); double** white_designMatrix = malloc_blas(nobs,totalGlobalParams); double** constraintsMatrix = malloc_blas(totalGlobalConstraints,totalGlobalParams); double *y = (double*)malloc(sizeof(double)*nobs); double *white_y = (double*)malloc(sizeof(double)*nobs); unsigned int off_f = gParams; // leave space for globals unsigned int off_r = 0; unsigned int off_c = gConstraints; logdbg("Building matricies for global fit... npsr=%u",npsr); logdbg("nobs=%u, totalGlobalParams=%u, totalGlobalConstraints=%u",nobs,totalGlobalParams,totalGlobalConstraints); logwarn("This mode is not supported yet!!!"); for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){ unsigned int nLocal = psr[ipsr].fitinfo.nParams-gParams; logdbg("ipsr=%u, off_r = %u, off_c=%u, off_f=%u, nlocal=%u", ipsr,off_r,off_c,off_f,nLocal); // the fit parameters for(unsigned int i=0; i < gNdata[ipsr]; i++){ // the global params (they go first) for(unsigned int g= 0; g < gParams; g++){ unsigned int j = g+nLocal; if(ipsr==0 && i==0 && writeResiduals){ logmsg("Row %d = %s %s(%d)",g,"global",label_str[global_fitinfo.paramIndex[g]],global_fitinfo.paramCounters[g]); } designMatrix[i+off_r][g] = gDM[ipsr][i][j]; white_designMatrix[i+off_r][g] = gWDM[ipsr][i][j]; } for(unsigned int j=0; j < nLocal; j++){ if(i==0 && writeResiduals){ logmsg("Row %d = %s %s(%d)",j+off_f,psr[ipsr].name,label_str[psr[ipsr].fitinfo.paramIndex[j]],psr[ipsr].fitinfo.paramCounters[j]); } designMatrix[i+off_r][j+off_f] = gDM[ipsr][i][j]; white_designMatrix[i+off_r][j+off_f] = gWDM[ipsr][i][j]; } } // the data for(unsigned int i=0; i < gNdata[ipsr]; ++i){ y[i+off_r] = gY[ipsr][i]; white_y[i+off_r] = gW[ipsr][i]; } for(unsigned int i=0; i < psr[ipsr].fitinfo.nConstraints; i++){ for(unsigned int j=0; j < nLocal; j++){ constraintsMatrix[i+off_c][j+off_f] = gCM[ipsr][i][j]; } // the global params (they go first) for(unsigned int g= 0; g < gParams; g++){ unsigned int j = g+nLocal; constraintsMatrix[i+off_c][g] = gCM[ipsr][i][j]; } } off_r += gNdata[ipsr]; off_f += nLocal; off_c += psr[ipsr].fitinfo.nConstraints; free(gY[ipsr]); free(gW[ipsr]); free_blas(gDM[ipsr]); if (gCM[ipsr]) free_blas(gCM[ipsr]); free_blas(gWDM[ipsr]); } double chisq; // the post-fit chi-squared double* parameterEstimates = (double*)malloc(sizeof(double)*totalGlobalParams); double* errorEstimates = (double*)malloc(sizeof(double)*totalGlobalParams); chisq = TKrobustConstrainedLeastSquares(y,white_y, designMatrix,white_designMatrix,constraintsMatrix, nobs,totalGlobalParams,totalGlobalConstraints, T2_SVD_TOL,1,parameterEstimates,errorEstimates,psr[0].covar, psr[0].robust); // for now the CVM ends up in psr[0].covar. int off_p = gParams; for (unsigned int ipsr = 0; ipsr < npsr ; ++ipsr){ // update the pulsar struct as appropriate psr[ipsr].fitChisq = chisq; psr[ipsr].fitNfree = nobs + totalGlobalConstraints - totalGlobalParams; double* psr_parameterEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams); double* psr_errorEstimates = (double*)malloc(sizeof(double)*psr[ipsr].fitinfo.nParams); const unsigned np = psr[ipsr].fitinfo.nParams-gParams; /* extract the fit output for the individual pulsars. * I.e. detangle the global fit * Notice: Globals go at the end of the individual pulsar arrays. */ for (unsigned i = 0; i < np; ++i){ psr_parameterEstimates[i] = parameterEstimates[off_p]; psr_errorEstimates[i] = errorEstimates[off_p]; ++off_p; } for (unsigned i = 0; i < gParams; ++i){ psr_parameterEstimates[i+np] = parameterEstimates[i]; psr_errorEstimates[i] = errorEstimates[off_p]; } logdbg("Updating the parameters"); logtchk("updating the parameter values"); /* * This routine calls the appropriate update functions to apply the result of the fit * to the origianal (non-linearised) pulsar parameters. */ t2Fit_updateParameters(psr,ipsr,psr_parameterEstimates,psr_errorEstimates); logtchk("complete updating the parameter values"); logdbg("Completed updating the parameters"); free(psr_parameterEstimates); free(psr_errorEstimates); } free(parameterEstimates); free(errorEstimates); free(white_y); free(y); free_blas(designMatrix); free_blas(white_designMatrix); free_blas(constraintsMatrix); } }
/** * Do the least squares using QR decomposition */ double accel_lsq_qr(double** A, double* data, double* oparam, int ndata, int nparam, double** Ocvm){ int nhrs=1; int nwork = -1; int info=0; int i,j; double iwork; // workspace query - works out optimal size for work array F77_dgels("T", &nparam, &ndata, &nhrs, A[0], &nparam, data, &ndata, &iwork, &nwork, &info); nwork=(int)iwork; logdbg("nwork = %d (%lf)",nwork,iwork); double* work = static_cast<double*>(malloc(sizeof(double)*nwork)); logdbg("accel_lsq_qr ndata=%d nparam=%d",ndata,nparam); // as usual we prentend that the matrix is transposed to deal with C->fortran conversion // degls does a least-squares using QR decomposition. Fast and robust. F77_dgels("T", &nparam, &ndata, &nhrs, A[0], &nparam, data, &ndata, work, &nwork, &info); free(work); // if info is not zero then the fit failed. if(info!=0){ logerr("Error in lapack DEGLS. INFO=%d See full logs for explanation.",info); logmsg(""); logmsg("From: http://www.netlib.org/lapack/explore-html/d8/dde/dgels_8f.html"); logmsg("INFO is INTEGER"); logmsg(" = 0: successful exit"); logmsg(" < 0: if INFO = -i, the i-th argument had an illegal value"); logmsg(" > 0: if INFO = i, the i-th diagonal element of the"); logmsg(" triangular factor of A is zero, so that A does not have"); logmsg(" full rank; the least squares solution could not be"); logmsg(" computed."); logmsg(""); if(info > 0){ logerr("It appears that you are fitting for a 'bad' parameter - E.g A jump on a non-existant flag."); logmsg(" TEMPO2 will NOT attempt to deal with this!"); logmsg("Cannot continue. Abort fit."); return -1; } else { logmsg("Cannot continue. Abort fit."); return -1; } } assert(info==0); if(oparam!=NULL){ // copy out the output parameters, which are written into the "data" array. memcpy(oparam,data,sizeof(double)*nparam); } double chisq=0; for( i = nparam; i < ndata; i++ ) chisq += data[i] * data[i]; if (Ocvm != NULL){ int n=nparam; // packed triangular matrix. double* _t=(double*)malloc(sizeof(double)*(n*(n+1))/2); // This code taken from the LAPACK documentation // to pack a triangular matrix. // we want the upper triangular matrix part of A. // // pack upper triangle like (r,c) // (1,1) (1,2) (2,2) // i+(2n-j)(j-1)/2 int jc=0; for (j=0;j<n;j++){ // cols for (i=0; i <=j; i++) { // rows _t[jc] = A[i][j]; // A came from fortran, so is in [col][row] ordering // BUT - we have transposed A, so we have to un-transpose it ++jc; } } logdbg("Inverting..."); F77_dtptri("U","N",&n,_t,&i); if(i!=0){ logerr("Error in lapack DTPTRI. INFO=%d",i); logmsg("From: http://www.netlib.org/lapack/explore-html/d8/d05/dtptri_8f.html"); logmsg("INFO is INTEGER"); logmsg(" = 0: successful exit"); logmsg(" < 0: if INFO = -i, the i-th argument had an illegal value"); logmsg(" > 0: if INFO = i, A(i,i) is exactly zero. The triangular"); logmsg(" matrix is singular and its inverse can not be computed."); logmsg("Cannot continue - abort fit"); return -1; } double **Rinv = malloc_uinv(n); for (j=0;j<n;j++){ // cols for (i=0;i<n;i++){ //rows Ocvm[i][j]=0; } } // Unpack the triangular matrix using reverse of above // We will put it in fortran, so continue to use [col][row] order jc=0; for (j=0;j<n;j++){ // cols for (i=0; i <=j; i++) { // rows Rinv[j][i] = _t[jc]; Ocvm[j][i] = _t[jc]; ++jc; } } free(_t); double a=chisq/(double)(ndata-nparam); // (X^T X)^-1 = Rinv.Rinv^T gives parameter covariance matrix // Note that Ocvm is input and output // and that covar matrix will be transposed, but it is // symetric so it doesn't matter! F77_dtrmm( "R", "U", "T", "N", &n, &n, &a, *Rinv, &n, *Ocvm, &n); // DTRMM ( SIDE, UPLO, TRANSA, DIAG, M, N, ALPHA, A , LDA, B, LDB ) if(debugFlag){ for(i=0;i<n;i++){ for(j=0;j<n;j++){ logdbg("COVAR %d %d %lg",i,j,Ocvm[i][j]); } } } free_uinv(Rinv); } return chisq; }