static void gradCov(omxFitFunction *oo, FitContext *fc)
{
const double Scale = Global->llScale;
omxExpectation *expectation = oo->expectation;
BA81FitState *state = (BA81FitState*) oo->argStruct;
BA81Expect *estate = (BA81Expect*) expectation->argStruct;
if (estate->verbose >= 1) mxLog("%s: cross product approximation", oo->name());
estate->grp.ba81OutcomeProb(estate->itemParam->data, FALSE);
const int numThreads = Global->numThreads;
const int numUnique = estate->getNumUnique();
ba81NormalQuad &quad = estate->getQuad();
const int numSpecific = quad.numSpecific;
const int maxDims = quad.maxDims;
const int pDims = numSpecific? maxDims-1 : maxDims;
const int maxAbilities = quad.maxAbilities;
Eigen::MatrixXd icovMat(pDims, pDims);
if (maxAbilities) {
Eigen::VectorXd mean;
Eigen::MatrixXd srcMat;
estate->getLatentDistribution(fc, mean, srcMat);
icovMat = srcMat.topLeftCorner(pDims, pDims);
Matrix tmp(icovMat.data(), pDims, pDims);
int info = InvertSymmetricPosDef(tmp, 'U');
if (info) {
omxRaiseErrorf("%s: latent covariance matrix is not positive definite", oo->name());
return;
}
icovMat.triangularView<Eigen::Lower>() = icovMat.transpose().triangularView<Eigen::Lower>();
}
std::vector<int> &rowMap = estate->grp.rowMap;
double *rowWeight = estate->grp.rowWeight;
std::vector<bool> &rowSkip = estate->grp.rowSkip;
const int totalQuadPoints = quad.totalQuadPoints;
omxMatrix *itemParam = estate->itemParam;
omxBuffer<double> patternLik(numUnique);
const int priDerivCoef = pDims + triangleLoc1(pDims);
const int numLatents = maxAbilities + triangleLoc1(maxAbilities);
const int thrDerivSize = itemParam->cols * state->itemDerivPadSize;
const int totalOutcomes = estate->totalOutcomes();
const int numItems = state->freeItemParams? estate->numItems() : 0;
const size_t numParam = fc->varGroup->vars.size();
std::vector<double> thrGrad(numThreads * numParam);
std::vector<double> thrMeat(numThreads * numParam * numParam);
const double *wherePrep = quad.wherePrep.data();
if (numSpecific == 0) {
omxBuffer<double> thrLxk(totalQuadPoints * numThreads);
omxBuffer<double> derivCoef(totalQuadPoints * priDerivCoef);
if (state->freeLatents) {
#pragma omp parallel for num_threads(numThreads)
for (int qx=0; qx < totalQuadPoints; qx++) {
const double *where = wherePrep + qx * maxDims;
calcDerivCoef(fc, state, estate, icovMat.data(), where,
derivCoef.data() + qx * priDerivCoef);
}
}
#pragma omp parallel for num_threads(numThreads)
for (int px=0; px < numUnique; px++) {
if (rowSkip[px]) continue;
int thrId = omx_absolute_thread_num();
double *lxk = thrLxk.data() + thrId * totalQuadPoints;
omxBuffer<double> expected(totalOutcomes); // can use maxOutcomes instead TODO
std::vector<double> deriv0(thrDerivSize);
std::vector<double> latentGrad(numLatents);
std::vector<double> patGrad(numParam);
double *grad = thrGrad.data() + thrId * numParam;
double *meat = thrMeat.data() + thrId * numParam * numParam;
estate->grp.ba81LikelihoodSlow2(px, lxk);
// If patternLik is already valid, maybe could avoid this loop TODO
double patternLik1 = 0;
for (int qx=0; qx < totalQuadPoints; qx++) {
patternLik1 += lxk[qx];
}
patternLik[px] = patternLik1;
// if (!validPatternLik(state, patternLik1)) complain, TODO
for (int qx=0; qx < totalQuadPoints; qx++) {
double tmp = lxk[qx];
mapLatentDeriv(state, estate, tmp, derivCoef.data() + qx * priDerivCoef,
latentGrad.data());
for (int ix=0; ix < numItems; ++ix) {
int pick = estate->grp.dataColumns[ix][rowMap[px]];
if (pick == NA_INTEGER) continue;
OMXZERO(expected.data(), estate->itemOutcomes(ix));
expected[pick-1] = tmp;
const double *spec = estate->itemSpec(ix);
double *iparam = omxMatrixColumn(itemParam, ix);
const int id = spec[RPF_ISpecID];
double *myDeriv = deriv0.data() + ix * state->itemDerivPadSize;
(*Glibrpf_model[id].dLL1)(spec, iparam, wherePrep + qx * maxDims,
expected.data(), myDeriv);
}
}
gradCov_finish_1pat(1 / patternLik1, rowWeight[px], numItems, numLatents, numParam,
state, estate, itemParam, deriv0, latentGrad, Scale, patGrad, grad, meat);
}
} else {
const int totalPrimaryPoints = quad.totalPrimaryPoints;
const int specificPoints = quad.quadGridSize;
omxBuffer<double> thrLxk(totalQuadPoints * numSpecific * numThreads);
omxBuffer<double> thrEi(totalPrimaryPoints * numThreads);
omxBuffer<double> thrEis(totalPrimaryPoints * numSpecific * numThreads);
const int derivPerPoint = priDerivCoef + 2 * numSpecific;
omxBuffer<double> derivCoef(totalQuadPoints * derivPerPoint);
if (state->freeLatents) {
#pragma omp parallel for num_threads(numThreads)
for (int qx=0; qx < totalQuadPoints; qx++) {
const double *where = wherePrep + qx * maxDims;
calcDerivCoef(fc, state, estate, icovMat.data(), where,
derivCoef.data() + qx * derivPerPoint);
for (int Sgroup=0; Sgroup < numSpecific; ++Sgroup) {
calcDerivCoef1(fc, state, estate, where, Sgroup,
derivCoef.data() + qx * derivPerPoint + priDerivCoef + 2 * Sgroup);
}
}
}
#pragma omp parallel for num_threads(numThreads)
for (int px=0; px < numUnique; px++) {
if (rowSkip[px]) continue;
int thrId = omx_absolute_thread_num();
double *lxk = thrLxk.data() + totalQuadPoints * numSpecific * thrId;
double *Ei = thrEi.data() + totalPrimaryPoints * thrId;
double *Eis = thrEis.data() + totalPrimaryPoints * numSpecific * thrId;
omxBuffer<double> expected(totalOutcomes); // can use maxOutcomes instead TODO
std::vector<double> deriv0(thrDerivSize);
std::vector<double> latentGrad(numLatents);
std::vector<double> patGrad(numParam);
double *grad = thrGrad.data() + thrId * numParam;
double *meat = thrMeat.data() + thrId * numParam * numParam;
estate->grp.cai2010EiEis(px, lxk, Eis, Ei);
for (int qx=0, qloc = 0; qx < totalPrimaryPoints; qx++) {
for (int sgroup=0; sgroup < numSpecific; ++sgroup) {
Eis[qloc] = Ei[qx] / Eis[qloc];
++qloc;
}
}
for (int qloc=0, eisloc=0, qx=0; eisloc < totalPrimaryPoints * numSpecific; eisloc += numSpecific) {
for (int sx=0; sx < specificPoints; sx++) {
mapLatentDeriv(state, estate, Eis[eisloc] * lxk[qloc],
derivCoef.data() + qx * derivPerPoint,
latentGrad.data());
for (int Sgroup=0; Sgroup < numSpecific; Sgroup++) {
double lxk1 = lxk[qloc];
double Eis1 = Eis[eisloc + Sgroup];
double tmp = Eis1 * lxk1;
mapLatentDerivS(state, estate, Sgroup, tmp,
derivCoef.data() + qx * derivPerPoint + priDerivCoef + 2 * Sgroup,
latentGrad.data());
for (int ix=0; ix < numItems; ++ix) {
if (estate->grp.Sgroup[ix] != Sgroup) continue;
int pick = estate->grp.dataColumns[ix][rowMap[px]];
if (pick == NA_INTEGER) continue;
OMXZERO(expected.data(), estate->itemOutcomes(ix));
expected[pick-1] = tmp;
const double *spec = estate->itemSpec(ix);
double *iparam = omxMatrixColumn(itemParam, ix);
const int id = spec[RPF_ISpecID];
const int dims = spec[RPF_ISpecDims];
double *myDeriv = deriv0.data() + ix * state->itemDerivPadSize;
const double *where = wherePrep + qx * maxDims;
Eigen::VectorXd ptheta(dims);
for (int dx=0; dx < dims; dx++) {
ptheta[dx] = where[std::min(dx, maxDims-1)];
}
(*Glibrpf_model[id].dLL1)(spec, iparam, ptheta.data(),
expected.data(), myDeriv);
}
++qloc;
}
++qx;
}
}
// If patternLik is already valid, maybe could avoid this loop TODO
double patternLik1 = 0;
for (int qx=0; qx < totalPrimaryPoints; ++qx) {
patternLik1 += Ei[qx];
}
patternLik[px] = patternLik1;
gradCov_finish_1pat(1 / patternLik1, rowWeight[px], numItems, numLatents, numParam,
state, estate, itemParam, deriv0, latentGrad, Scale, patGrad, grad, meat);
}
}
for (int tx=1; tx < numThreads; ++tx) {
double *th = thrGrad.data() + tx * numParam;
for (size_t en=0; en < numParam; ++en) {
thrGrad[en] += th[en];
}
}
for (int tx=1; tx < numThreads; ++tx) {
double *th = thrMeat.data() + tx * numParam * numParam;
for (size_t en=0; en < numParam * numParam; ++en) {
thrMeat[en] += th[en];
}
}
for (size_t d1=0; d1 < numParam; ++d1) {
fc->grad(d1) += thrGrad[d1];
}
if (fc->infoB) {
for (size_t d1=0; d1 < numParam; ++d1) {
for (size_t d2=0; d2 < numParam; ++d2) {
int cell = d1 * numParam + d2;
fc->infoB[cell] += thrMeat[cell];
}
}
}
}
void omxInitExpectationBA81(omxExpectation* oo) {
omxState* currentState = oo->currentState;
SEXP rObj = oo->rObj;
SEXP tmp;
if(OMX_DEBUG) {
mxLog("Initializing %s.", oo->name);
}
if (!Glibrpf_model) {
#if USE_EXTERNAL_LIBRPF
get_librpf_t get_librpf = (get_librpf_t) R_GetCCallable("rpf", "get_librpf_model_GPL");
(*get_librpf)(LIBIFA_RPF_API_VERSION, &Glibrpf_numModels, &Glibrpf_model);
#else
// if linking against included source code
Glibrpf_numModels = librpf_numModels;
Glibrpf_model = librpf_model;
#endif
}
BA81Expect *state = new BA81Expect;
// These two constants should be as identical as possible
state->name = oo->name;
if (0) {
state->LogLargestDouble = 0.0;
state->LargestDouble = 1.0;
} else {
state->LogLargestDouble = log(std::numeric_limits<double>::max()) - 1;
state->LargestDouble = exp(state->LogLargestDouble);
ba81NormalQuad &quad = state->getQuad();
quad.setOne(state->LargestDouble);
}
state->expectedUsed = false;
state->estLatentMean = NULL;
state->estLatentCov = NULL;
state->type = EXPECTATION_OBSERVED;
state->itemParam = NULL;
state->EitemParam = NULL;
state->itemParamVersion = 0;
state->latentParamVersion = 0;
oo->argStruct = (void*) state;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("data")));
state->data = omxDataLookupFromState(tmp, currentState);
}
if (strcmp(omxDataType(state->data), "raw") != 0) {
omxRaiseErrorf("%s unable to handle data type %s", oo->name, omxDataType(state->data));
return;
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("verbose")));
state->verbose = Rf_asInteger(tmp);
}
int targetQpoints;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("qpoints")));
targetQpoints = Rf_asInteger(tmp);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("qwidth")));
state->grp.setGridFineness(Rf_asReal(tmp), targetQpoints);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("ItemSpec")));
state->grp.importSpec(tmp);
if (state->verbose >= 2) mxLog("%s: found %d item specs", oo->name, state->numItems());
}
state->_latentMeanOut = omxNewMatrixFromSlot(rObj, currentState, "mean");
state->_latentCovOut = omxNewMatrixFromSlot(rObj, currentState, "cov");
state->itemParam = omxNewMatrixFromSlot(rObj, currentState, "item");
state->grp.param = state->itemParam->data; // algebra not allowed yet TODO
const int numItems = state->itemParam->cols;
if (state->numItems() != numItems) {
omxRaiseErrorf("ItemSpec length %d must match the number of item columns (%d)",
state->numItems(), numItems);
return;
}
if (state->itemParam->rows != state->grp.impliedParamRows) {
omxRaiseErrorf("item matrix must have %d rows", state->grp.impliedParamRows);
return;
}
state->grp.paramRows = state->itemParam->rows;
// for algebra item param, will need to defer until later?
state->grp.learnMaxAbilities();
int maxAbilities = state->grp.itemDims;
state->grp.setFactorNames(state->itemParam->rownames);
{
ProtectedSEXP tmp2(R_do_slot(rObj, Rf_install(".detectIndependence")));
state->grp.detectIndependence = Rf_asLogical(tmp2);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("EstepItem")));
if (!Rf_isNull(tmp)) {
int rows, cols;
getMatrixDims(tmp, &rows, &cols);
if (rows != state->itemParam->rows || cols != state->itemParam->cols) {
Rf_error("EstepItem must have the same dimensions as the item MxMatrix");
}
state->EitemParam = REAL(tmp);
}
}
oo->computeFun = ba81compute;
oo->setVarGroup = ignoreSetVarGroup;
oo->destructFun = ba81Destroy;
oo->populateAttrFun = ba81PopulateAttributes;
oo->componentFun = getComponent;
oo->canDuplicate = false;
// TODO: Exactly identical rows do not contribute any information.
// The sorting algorithm ought to remove them so we get better cache behavior.
// The following summary stats would be cheaper to calculate too.
omxData *data = state->data;
if (data->hasDefinitionVariables()) Rf_error("%s: not implemented yet", oo->name);
std::vector<int> &rowMap = state->grp.rowMap;
int weightCol;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("weightColumn")));
weightCol = INTEGER(tmp)[0];
}
if (weightCol == NA_INTEGER) {
// Should rowMap be part of omxData? This is essentially a
// generic compression step that shouldn't be specific to IFA models.
state->grp.rowWeight = (double*) R_alloc(data->rows, sizeof(double));
rowMap.resize(data->rows);
int numUnique = 0;
for (int rx=0; rx < data->rows; ) {
int rw = 1;
state->grp.rowWeight[numUnique] = rw;
rowMap[numUnique] = rx;
rx += rw;
++numUnique;
}
rowMap.resize(numUnique);
state->weightSum = state->data->rows;
}
else {
if (omxDataColumnIsFactor(data, weightCol)) {
omxRaiseErrorf("%s: weightColumn %d is a factor", oo->name, 1 + weightCol);
return;
}
state->grp.rowWeight = omxDoubleDataColumn(data, weightCol);
state->weightSum = 0;
for (int rx=0; rx < data->rows; ++rx) { state->weightSum += state->grp.rowWeight[rx]; }
rowMap.resize(data->rows);
for (size_t rx=0; rx < rowMap.size(); ++rx) {
rowMap[rx] = rx;
}
}
// complain about non-integral rowWeights (EAP can't work) TODO
auto colMap = oo->getDataColumns();
for (int cx = 0; cx < numItems; cx++) {
int *col = omxIntDataColumnUnsafe(data, colMap[cx]);
state->grp.dataColumns.push_back(col);
}
// sanity check data
for (int cx = 0; cx < numItems; cx++) {
if (!omxDataColumnIsFactor(data, colMap[cx])) {
data->omxPrintData("diagnostic", 3);
omxRaiseErrorf("%s: column %d is not a factor", oo->name, int(1 + colMap[cx]));
return;
}
}
// TODO the max outcome should be available from omxData
for (int rx=0; rx < data->rows; rx++) {
int cols = 0;
for (int cx = 0; cx < numItems; cx++) {
const int *col = state->grp.dataColumns[cx];
int pick = col[rx];
if (pick == NA_INTEGER) continue;
++cols;
const int no = state->grp.itemOutcomes[cx];
if (pick > no) {
Rf_error("Data for item '%s' has at least %d outcomes, not %d",
state->itemParam->colnames[cx], pick, no);
}
}
if (cols == 0) {
Rf_error("Row %d has all NAs", 1+rx);
}
}
if (state->_latentMeanOut && state->_latentMeanOut->rows * state->_latentMeanOut->cols != maxAbilities) {
Rf_error("The mean matrix '%s' must be a row or column vector of size %d",
state->_latentMeanOut->name(), maxAbilities);
}
if (state->_latentCovOut && (state->_latentCovOut->rows != maxAbilities ||
state->_latentCovOut->cols != maxAbilities)) {
Rf_error("The cov matrix '%s' must be %dx%d",
state->_latentCovOut->name(), maxAbilities, maxAbilities);
}
state->grp.setLatentDistribution(state->_latentMeanOut? state->_latentMeanOut->data : NULL,
state->_latentCovOut? state->_latentCovOut->data : NULL);
{
EigenArrayAdaptor Eparam(state->itemParam);
Eigen::Map< Eigen::VectorXd > meanVec(state->grp.mean, maxAbilities);
Eigen::Map< Eigen::MatrixXd > covMat(state->grp.cov, maxAbilities, maxAbilities);
state->grp.quad.setStructure(state->grp.qwidth, state->grp.qpoints,
Eparam, meanVec, covMat);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("minItemsPerScore")));
state->grp.setMinItemsPerScore(Rf_asInteger(tmp));
}
state->grp.buildRowSkip();
if (isErrorRaised()) return;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("debugInternal")));
state->debugInternal = Rf_asLogical(tmp);
}
state->ElatentVersion = 0;
if (state->_latentMeanOut) {
state->estLatentMean = omxInitMatrix(maxAbilities, 1, TRUE, currentState);
omxCopyMatrix(state->estLatentMean, state->_latentMeanOut); // rename matrices TODO
}
if (state->_latentCovOut) {
state->estLatentCov = omxInitMatrix(maxAbilities, maxAbilities, TRUE, currentState);
omxCopyMatrix(state->estLatentCov, state->_latentCovOut);
}
}