void flattenDataToVector(omxMatrix* cov, omxMatrix* means, omxMatrix *obsThresholdMat,
std::vector< omxThresholdColumn > &thresholds, omxMatrix* vector) {
// TODO: vectorize data flattening
// if(OMX_DEBUG) { mxLog("Flattening out data vectors: cov 0x%x, mean 0x%x, thresh 0x%x[n=%d] ==> 0x%x",
// cov, means, thresholds, nThresholds, vector); }
int nextLoc = 0;
for(int j = 0; j < cov->rows; j++) {
for(int k = j; k < cov->rows; k++) {
omxSetVectorElement(vector, nextLoc, omxMatrixElement(cov, j, k)); // Use upper triangle in case of SYMM-style mat.
nextLoc++;
}
}
if (means != NULL) {
for(int j = 0; j < cov->rows; j++) {
omxSetVectorElement(vector, nextLoc, omxVectorElement(means, j));
nextLoc++;
}
}
for(int j = 0; j < int(thresholds.size()); j++) {
omxThresholdColumn* thresh = &thresholds[j];
for(int k = 0; k < thresh->numThresholds; k++) {
omxSetVectorElement(vector, nextLoc, omxMatrixElement(obsThresholdMat, k, thresh->column));
nextLoc++;
}
}
}
static void setLatentStartingValues(omxFitFunction *oo, FitContext *fc) //remove? TODO
{
BA81FitState *state = (BA81FitState*) oo->argStruct;
BA81Expect *estate = (BA81Expect*) oo->expectation->argStruct;
std::vector<int> &latentMap = state->latentMap;
ba81NormalQuad &quad = estate->getQuad();
int maxAbilities = quad.maxAbilities;
omxMatrix *estMean = estate->estLatentMean;
omxMatrix *estCov = estate->estLatentCov;
for (int a1 = 0; a1 < maxAbilities; ++a1) {
if (latentMap[a1] >= 0) {
int to = latentMap[a1];
fc->est[to] = omxVectorElement(estMean, a1);
}
for (int a2 = 0; a2 <= a1; ++a2) {
int to = latentMap[maxAbilities + triangleLoc1(a1) + a2];
if (to < 0) continue;
fc->est[to] = omxMatrixElement(estCov, a1, a2);
}
}
if (estate->verbose >= 1) {
mxLog("%s: set latent parameters for version %d",
oo->name(), estate->ElatentVersion);
}
}
double totalLogLikelihood(omxMatrix *fitMat)
{
if (fitMat->rows != 1) {
omxFitFunction *ff = fitMat->fitFunction;
if (strEQ(ff->fitType, "MxFitFunctionML") || strEQ(ff->fitType, "imxFitFunctionFIML")) {
// NOTE: Floating-point addition is not
// associative. If we compute this in parallel
// then we introduce non-determinancy.
double sum = 0;
for(int i = 0; i < fitMat->rows; i++) {
sum += log(omxVectorElement(fitMat, i));
}
if (!Global->rowLikelihoodsWarning) {
Rf_warning("%s does not evaluate to a 1x1 matrix. Fixing model by adding "
"mxAlgebra(-2*sum(log(%s)), 'm2ll'), mxFitFunctionAlgebra('m2ll')",
fitMat->name(), fitMat->name());
Global->rowLikelihoodsWarning = true;
}
return sum * Global->llScale;
} else {
omxRaiseErrorf("%s of type %s returned %d values instead of 1, not sure how to proceed",
fitMat->name(), ff->fitType, fitMat->rows);
return nan("unknown");
}
} else {
return fitMat->data[0];
}
}
void AlgebraFitFunction::compute(int want, FitContext *fc)
{
if (fc && varGroup != fc->varGroup) {
setVarGroup(fc->varGroup);
}
if (want & (FF_COMPUTE_FIT | FF_COMPUTE_INITIAL_FIT | FF_COMPUTE_PREOPTIMIZE)) {
if (algebra) {
omxRecompute(algebra, fc);
ff->matrix->data[0] = algebra->data[0];
} else {
ff->matrix->data[0] = 0;
}
}
if (gradMap.size() == 0) return;
if (gradient) {
omxRecompute(gradient, fc);
if (want & FF_COMPUTE_GRADIENT) {
for (size_t v1=0; v1 < gradMap.size(); ++v1) {
int to = gradMap[v1];
if (to < 0) continue;
fc->grad(to) += omxVectorElement(gradient, v1);
}
}
if (want & FF_COMPUTE_INFO && fc->infoMethod == INFO_METHOD_MEAT) {
std::vector<double> grad(varGroup->vars.size());
for (size_t v1=0; v1 < gradMap.size(); ++v1) {
int to = gradMap[v1];
if (to < 0) continue;
grad[to] += omxVectorElement(gradient, v1);
}
addSymOuterProd(1, grad.data(), varGroup->vars.size(), fc->infoB);
}
}
if (hessian && ((want & (FF_COMPUTE_HESSIAN | FF_COMPUTE_IHESSIAN)) ||
(want & FF_COMPUTE_INFO && fc->infoMethod == INFO_METHOD_HESSIAN))) {
omxRecompute(hessian, fc);
if (!vec2diag) {
HessianBlock *hb = new HessianBlock;
hb->vars.resize(numDeriv);
int vx=0;
for (size_t h1=0; h1 < gradMap.size(); ++h1) {
if (gradMap[h1] < 0) continue;
hb->vars[vx] = gradMap[h1];
++vx;
}
hb->mat.resize(numDeriv, numDeriv);
for (size_t d1=0, h1=0; h1 < gradMap.size(); ++h1) {
if (gradMap[h1] < 0) continue;
for (size_t d2=0, h2=0; h2 <= h1; ++h2) {
if (gradMap[h2] < 0) continue;
if (h1 == h2) {
hb->mat(d2,d1) = omxMatrixElement(hessian, h2, h1);
} else {
double coef1 = omxMatrixElement(hessian, h2, h1);
double coef2 = omxMatrixElement(hessian, h1, h2);
if (coef1 != coef2) {
Rf_warning("%s: Hessian algebra '%s' is not symmetric at [%d,%d]",
ff->matrix->name(), hessian->name(), 1+h2, 1+h1);
}
hb->mat(d2,d1) = coef1;
}
++d2;
}
++d1;
}
fc->queue(hb);
} else {
for (size_t h1=0; h1 < gradMap.size(); ++h1) {
int to = gradMap[h1];
if (to < 0) continue;
HessianBlock *hb = new HessianBlock;
hb->vars.assign(1, to);
hb->mat.resize(1,1);
hb->mat(0,0) = omxMatrixElement(hessian, h1, h1);
fc->queue(hb);
}
}
}
// complain if unimplemented FF_COMPUTE_INFO requested? TODO
}
static void omxRowFitFunctionSingleIteration(omxFitFunction *localobj, omxFitFunction *sharedobj, int rowbegin, int rowcount,
FitContext *fc) {
omxRowFitFunction* oro = ((omxRowFitFunction*) localobj->argStruct);
omxRowFitFunction* shared_oro = ((omxRowFitFunction*) sharedobj->argStruct);
omxMatrix *rowAlgebra, *rowResults;
omxMatrix *filteredDataRow, *dataRow, *existenceVector;
omxMatrix *dataColumns;
omxData *data;
int isContiguous, contiguousStart, contiguousLength;
int numCols, numRemoves;
rowAlgebra = oro->rowAlgebra;
rowResults = shared_oro->rowResults;
data = oro->data;
dataColumns = oro->dataColumns;
dataRow = oro->dataRow;
filteredDataRow = oro->filteredDataRow;
existenceVector = oro->existenceVector;
isContiguous = oro->contiguous.isContiguous;
contiguousStart = oro->contiguous.start;
contiguousLength = oro->contiguous.length;
Eigen::VectorXd oldDefs;
oldDefs.resize(data->defVars.size());
oldDefs.setConstant(NA_REAL);
numCols = dataColumns->cols;
int *toRemove = (int*) malloc(sizeof(int) * dataColumns->cols);
int *zeros = (int*) calloc(dataColumns->cols, sizeof(int));
for(int row = rowbegin; row < data->rows && (row - rowbegin) < rowcount; row++) {
data->handleDefinitionVarList(localobj->matrix->currentState, row, oldDefs.data());
omxStateNextRow(localobj->matrix->currentState); // Advance row
// Populate data row
numRemoves = 0;
if (isContiguous) {
omxContiguousDataRow(data, row, contiguousStart, contiguousLength, dataRow);
} else {
omxDataRow(data, row, dataColumns, dataRow); // Populate data row
}
markDataRowDependencies(localobj->matrix->currentState, oro);
for(int j = 0; j < dataColumns->cols; j++) {
double dataValue = omxVectorElement(dataRow, j);
if(std::isnan(dataValue)) {
numRemoves++;
toRemove[j] = 1;
omxSetVectorElement(existenceVector, j, 0);
} else {
toRemove[j] = 0;
omxSetVectorElement(existenceVector, j, 1);
}
}
// TODO: Determine if this is the correct response.
if(numRemoves == numCols) {
char *errstr = (char*) calloc(250, sizeof(char));
sprintf(errstr, "Row %d completely missing. omxRowFitFunction cannot have completely missing rows.", omxDataIndex(data, row));
omxRaiseError(errstr);
free(errstr);
continue;
}
omxCopyMatrix(filteredDataRow, dataRow);
omxRemoveRowsAndColumns(filteredDataRow, 0, numRemoves, zeros, toRemove);
omxRecompute(rowAlgebra, fc);
omxCopyMatrixToRow(rowAlgebra, omxDataIndex(data, row), rowResults);
}
free(toRemove);
free(zeros);
}
static void CallFIMLFitFunction(omxFitFunction *off, int want, FitContext *fc)
{
// TODO: Figure out how to give access to other per-iteration structures.
// TODO: Current implementation is slow: update by filtering correlations and thresholds.
// TODO: Current implementation does not implement speedups for sorting.
// TODO: Current implementation may fail on all-continuous-missing or all-ordinal-missing rows.
if (want & (FF_COMPUTE_PREOPTIMIZE)) return;
if(OMX_DEBUG) {
mxLog("Beginning Joint FIML Evaluation.");
}
int returnRowLikelihoods = 0;
omxFIMLFitFunction* ofiml = ((omxFIMLFitFunction*)off->argStruct);
omxMatrix* fitMatrix = off->matrix;
int numChildren = (int) fc->childList.size();
omxMatrix *cov = ofiml->cov;
omxMatrix *means = ofiml->means;
if (!means) {
omxRaiseErrorf("%s: raw data observed but no expected means "
"vector was provided. Add something like mxPath(from = 'one',"
" to = manifests) to your model.", off->name());
return;
}
omxData* data = ofiml->data; // read-only
omxMatrix *dataColumns = ofiml->dataColumns;
returnRowLikelihoods = ofiml->returnRowLikelihoods; // read-only
omxExpectation* expectation = off->expectation;
std::vector< omxThresholdColumn > &thresholdCols = expectation->thresholds;
if (data->defVars.size() == 0 && !strEQ(expectation->expType, "MxExpectationStateSpace")) {
if(OMX_DEBUG) {mxLog("Precalculating cov and means for all rows.");}
omxExpectationRecompute(fc, expectation);
// MCN Also do the threshold formulae!
for(int j=0; j < dataColumns->cols; j++) {
int var = omxVectorElement(dataColumns, j);
if (!omxDataColumnIsFactor(data, var)) continue;
if (j < int(thresholdCols.size()) && thresholdCols[j].numThresholds > 0) { // j is an ordinal column
omxMatrix* nextMatrix = thresholdCols[j].matrix;
omxRecompute(nextMatrix, fc);
checkIncreasing(nextMatrix, thresholdCols[j].column, thresholdCols[j].numThresholds, fc);
for(int index = 0; index < numChildren; index++) {
FitContext *kid = fc->childList[index];
omxMatrix *target = kid->lookupDuplicate(nextMatrix);
omxCopyMatrix(target, nextMatrix);
}
} else {
Rf_error("No threshold given for ordinal column '%s'",
omxDataColumnName(data, j));
}
}
double *corList = ofiml->corList;
double *weights = ofiml->weights;
if (corList) {
omxStandardizeCovMatrix(cov, corList, weights, fc); // Calculate correlation and covariance
}
for(int index = 0; index < numChildren; index++) {
FitContext *kid = fc->childList[index];
omxMatrix *childFit = kid->lookupDuplicate(fitMatrix);
omxFIMLFitFunction* childOfiml = ((omxFIMLFitFunction*) childFit->fitFunction->argStruct);
omxCopyMatrix(childOfiml->cov, cov);
omxCopyMatrix(childOfiml->means, means);
if (corList) {
memcpy(childOfiml->weights, weights, sizeof(double) * cov->rows);
memcpy(childOfiml->corList, corList, sizeof(double) * (cov->rows * (cov->rows - 1)) / 2);
}
}
if(OMX_DEBUG) { omxPrintMatrix(cov, "Cov"); }
if(OMX_DEBUG) { omxPrintMatrix(means, "Means"); }
}
memset(ofiml->rowLogLikelihoods->data, 0, sizeof(double) * data->rows);
int parallelism = (numChildren == 0) ? 1 : numChildren;
if (parallelism > data->rows) {
parallelism = data->rows;
}
FIMLSingleIterationType singleIter = ofiml->SingleIterFn;
bool failed = false;
if (parallelism > 1) {
int stride = (data->rows / parallelism);
#pragma omp parallel for num_threads(parallelism) reduction(||:failed)
for(int i = 0; i < parallelism; i++) {
FitContext *kid = fc->childList[i];
omxMatrix *childMatrix = kid->lookupDuplicate(fitMatrix);
omxFitFunction *childFit = childMatrix->fitFunction;
if (i == parallelism - 1) {
failed |= singleIter(kid, childFit, off, stride * i, data->rows - stride * i);
} else {
failed |= singleIter(kid, childFit, off, stride * i, stride);
}
}
} else {
failed |= singleIter(fc, off, off, 0, data->rows);
}
if (failed) {
omxSetMatrixElement(off->matrix, 0, 0, NA_REAL);
return;
}
if(!returnRowLikelihoods) {
double val, sum = 0.0;
// floating-point addition is not associative,
// so we serialized the following reduction operation.
for(int i = 0; i < data->rows; i++) {
val = omxVectorElement(ofiml->rowLogLikelihoods, i);
// mxLog("%d , %f, %llx\n", i, val, *((unsigned long long*) &val));
sum += val;
}
if(OMX_DEBUG) {mxLog("Total Likelihood is %3.3f", sum);}
omxSetMatrixElement(off->matrix, 0, 0, sum);
}
}
