void FitMultigroup::init()
{
auto *oo = this;
FitMultigroup *mg =this;
SEXP rObj = oo->rObj;
if (!rObj) return;
if (mg->fits.size()) return; // hack to prevent double initialization, remove TOOD
oo->units = FIT_UNITS_UNINITIALIZED;
oo->gradientAvailable = TRUE;
oo->hessianAvailable = TRUE;
oo->canDuplicate = true;
omxState *os = oo->matrix->currentState;
ProtectedSEXP Rverb(R_do_slot(rObj, Rf_install("verbose")));
mg->verbose = Rf_asInteger(Rverb);
ProtectedSEXP Rgroups(R_do_slot(rObj, Rf_install("groups")));
int *fits = INTEGER(Rgroups);
for(int gx = 0; gx < Rf_length(Rgroups); gx++) {
if (isErrorRaised()) break;
omxMatrix *mat;
if (fits[gx] >= 0) {
mat = os->algebraList[fits[gx]];
} else {
mxThrow("Can only add algebra and fitfunction");
}
if (mat == oo->matrix) mxThrow("Cannot add multigroup to itself");
mg->fits.push_back(mat);
if (mat->fitFunction) {
omxCompleteFitFunction(mat);
oo->gradientAvailable = (oo->gradientAvailable && mat->fitFunction->gradientAvailable);
oo->hessianAvailable = (oo->hessianAvailable && mat->fitFunction->hessianAvailable);
} else {
oo->gradientAvailable = FALSE;
oo->hessianAvailable = FALSE;
}
}
}
void omxComputeNumericDeriv::computeImpl(FitContext *fc)
{
if (fc->fitUnits == FIT_UNITS_SQUARED_RESIDUAL ||
fc->fitUnits == FIT_UNITS_SQUARED_RESIDUAL_CHISQ) { // refactor TODO
numParams = 0;
if (verbose >= 1) mxLog("%s: derivatives %s units are meaningless",
name, fitUnitsToName(fc->fitUnits));
return; //Possible TODO: calculate Hessian anyway?
}
int newWanted = fc->wanted | FF_COMPUTE_GRADIENT;
if (wantHessian) newWanted |= FF_COMPUTE_HESSIAN;
int nf = fc->calcNumFree();
if (numParams != 0 && numParams != nf) {
mxThrow("%s: number of parameters changed from %d to %d",
name, numParams, nf);
}
numParams = nf;
if (numParams <= 0) { complainNoFreeParam(); return; }
optima.resize(numParams);
fc->copyEstToOptimizer(optima);
paramMap.resize(numParams);
for (int px=0,ex=0; px < numParams; ++ex) {
if (fc->profiledOut[ex]) continue;
paramMap[px++] = ex;
}
omxAlgebraPreeval(fitMat, fc);
fc->createChildren(fitMat); // allow FIML rowwiseParallel even when parallel=false
fc->state->countNonlinearConstraints(fc->state->numEqC, fc->state->numIneqC, false);
int c_n = fc->state->numEqC + fc->state->numIneqC;
fc->constraintFunVals.resize(c_n);
fc->constraintJacobian.resize(c_n, numParams);
if(c_n){
omxCalcFinalConstraintJacobian(fc, numParams);
}
// TODO: Allow more than one hessian value for calculation
int numChildren = 1;
if (parallel && !fc->openmpUser && fc->childList.size()) numChildren = fc->childList.size();
if (!fc->haveReferenceFit(fitMat)) return;
minimum = fc->fit;
hessWorkVector = new hess_struct[numChildren];
if (numChildren == 1) {
omxPopulateHessianWork(hessWorkVector, fc);
} else {
for(int i = 0; i < numChildren; i++) {
omxPopulateHessianWork(hessWorkVector + i, fc->childList[i]);
}
}
if(verbose >= 1) mxLog("Numerical Hessian approximation (%d children, ref fit %.2f)",
numChildren, minimum);
hessian = NULL;
if (wantHessian) {
hessian = fc->getDenseHessUninitialized();
Eigen::Map< Eigen::MatrixXd > eH(hessian, numParams, numParams);
eH.setConstant(NA_REAL);
if (knownHessian) {
int khSize = int(khMap.size());
Eigen::Map< Eigen::MatrixXd > kh(knownHessian, khSize, khMap.size());
for (int rx=0; rx < khSize; ++rx) {
for (int cx=0; cx < khSize; ++cx) {
if (khMap[rx] < 0 || khMap[cx] < 0) continue;
eH(khMap[rx], khMap[cx]) = kh(rx, cx);
}
}
}
}
if (detail) {
recordDetail = false; // already done it once
} else {
Rf_protect(detail = Rf_allocVector(VECSXP, 4));
SET_VECTOR_ELT(detail, 0, Rf_allocVector(LGLSXP, numParams));
for (int gx=0; gx < 3; ++gx) {
SET_VECTOR_ELT(detail, 1+gx, Rf_allocVector(REALSXP, numParams));
}
SEXP detailCols;
Rf_protect(detailCols = Rf_allocVector(STRSXP, 4));
Rf_setAttrib(detail, R_NamesSymbol, detailCols);
SET_STRING_ELT(detailCols, 0, Rf_mkChar("symmetric"));
SET_STRING_ELT(detailCols, 1, Rf_mkChar("forward"));
SET_STRING_ELT(detailCols, 2, Rf_mkChar("central"));
SET_STRING_ELT(detailCols, 3, Rf_mkChar("backward"));
SEXP detailRowNames;
Rf_protect(detailRowNames = Rf_allocVector(STRSXP, numParams));
Rf_setAttrib(detail, R_RowNamesSymbol, detailRowNames);
for (int nx=0; nx < int(numParams); ++nx) {
SET_STRING_ELT(detailRowNames, nx, Rf_mkChar(fc->varGroup->vars[nx]->name));
}
markAsDataFrame(detail);
}
gforward = REAL(VECTOR_ELT(detail, 1));
gcentral = REAL(VECTOR_ELT(detail, 2));
gbackward = REAL(VECTOR_ELT(detail, 3));
Eigen::Map< Eigen::ArrayXd > Gf(gforward, numParams);
Eigen::Map< Eigen::ArrayXd > Gc(gcentral, numParams);
Eigen::Map< Eigen::ArrayXd > Gb(gbackward, numParams);
Gf.setConstant(NA_REAL);
Gc.setConstant(NA_REAL);
Gb.setConstant(NA_REAL);
calcHessianEntry che(this);
CovEntrywiseParallel(numChildren, che);
for(int i = 0; i < numChildren; i++) {
struct hess_struct *hw = hessWorkVector + i;
totalProbeCount += hw->probeCount;
}
delete [] hessWorkVector;
if (isErrorRaised()) return;
Eigen::Map< Eigen::ArrayXi > Gsymmetric(LOGICAL(VECTOR_ELT(detail, 0)), numParams);
double gradNorm = 0.0;
double feasibilityTolerance = Global->feasibilityTolerance;
for (int px=0; px < numParams; ++px) {
// factor out simliar code in ComputeNR
omxFreeVar &fv = *fc->varGroup->vars[ paramMap[px] ];
if ((fabs(optima[px] - fv.lbound) < feasibilityTolerance && Gc[px] > 0) ||
(fabs(optima[px] - fv.ubound) < feasibilityTolerance && Gc[px] < 0)) {
Gsymmetric[px] = false;
continue;
}
gradNorm += Gc[px] * Gc[px];
double relsym = 2 * fabs(Gf[px] + Gb[px]) / (Gb[px] - Gf[px]);
Gsymmetric[px] = (Gf[px] < 0 && 0 < Gb[px] && relsym < 1.5);
if (checkGradient && verbose >= 2 && !Gsymmetric[px]) {
mxLog("%s: param[%d] %d %f", name, px, Gsymmetric[px], relsym);
}
}
fc->grad.resize(fc->numParam);
fc->grad.setZero();
fc->copyGradFromOptimizer(Gc);
if(c_n){
fc->inequality.resize(fc->state->numIneqC);
fc->analyticIneqJacTmp.resize(fc->state->numIneqC, numParams);
fc->myineqFun(true, verbose, omxConstraint::LESS_THAN, false);
}
gradNorm = sqrt(gradNorm);
double gradThresh = Global->getGradientThreshold(minimum);
//The gradient will generally not be near zero at a local minimum if there are equality constraints
//or active inequality constraints:
if ( checkGradient && gradNorm > gradThresh && !(fc->state->numEqC || fc->inequality.array().sum()) ) {
if (verbose >= 1) {
mxLog("Some gradient entries are too large, norm %f", gradNorm);
}
if (fc->getInform() < INFORM_NOT_AT_OPTIMUM) fc->setInform(INFORM_NOT_AT_OPTIMUM);
}
fc->setEstFromOptimizer(optima);
// auxillary information like per-row likelihoods need a refresh
ComputeFit(name, fitMat, FF_COMPUTE_FIT, fc);
fc->wanted = newWanted;
}
void omxCompleteExpectation(omxExpectation *ox) {
if(ox->isComplete) return;
if (ox->rObj) {
omxState *os = ox->currentState;
SEXP rObj = ox->rObj;
SEXP slot;
{ScopedProtect(slot, R_do_slot(rObj, Rf_install("container")));
if (Rf_length(slot) == 1) {
int ex = INTEGER(slot)[0];
ox->container = os->expectationList.at(ex);
}
}
{ScopedProtect(slot, R_do_slot(rObj, Rf_install("submodels")));
if (Rf_length(slot)) {
int numSubmodels = Rf_length(slot);
int *submodel = INTEGER(slot);
for (int ex=0; ex < numSubmodels; ex++) {
int sx = submodel[ex];
ox->submodels.push_back(omxExpectationFromIndex(sx, os));
}
}
}
}
omxExpectationProcessDataStructures(ox, ox->rObj);
int numSubmodels = (int) ox->submodels.size();
for (int ex=0; ex < numSubmodels; ex++) {
omxCompleteExpectation(ox->submodels[ex]);
}
ox->initFun(ox);
if(ox->computeFun == NULL) {
if (isErrorRaised()) {
Rf_error("Failed to initialize '%s' of type %s: %s", ox->name, ox->expType, Global->getBads());
} else {
Rf_error("Failed to initialize '%s' of type %s", ox->name, ox->expType);
}
}
if (OMX_DEBUG) {
omxData *od = ox->data;
omxState *state = ox->currentState;
std::string msg = string_snprintf("Expectation '%s' of type '%s' has"
" %d definition variables:\n", ox->name, ox->expType,
int(od->defVars.size()));
for (int dx=0; dx < int(od->defVars.size()); ++dx) {
omxDefinitionVar &dv = od->defVars[dx];
msg += string_snprintf("[%d] column '%s' ->", dx, omxDataColumnName(od, dv.column));
for (int lx=0; lx < dv.numLocations; ++lx) {
msg += string_snprintf(" %s[%d,%d]", state->matrixToName(~dv.matrices[lx]),
dv.rows[lx], dv.cols[lx]);
}
msg += "\n dirty:";
for (int mx=0; mx < dv.numDeps; ++mx) {
msg += string_snprintf(" %s", state->matrixToName(dv.deps[mx]));
}
msg += "\n";
}
mxLogBig(msg);
}
ox->isComplete = TRUE;
}
void omxSD(GradientOptimizerContext &rf)
{
int maxIter = rf.maxMajorIterations;
if (maxIter == -1) maxIter = 50000;
Eigen::VectorXd currEst(rf.numFree);
rf.copyToOptimizer(currEst.data());
int iter = 0;
double priorSpeed = 1.0, shrinkage = 0.7;
rf.setupSimpleBounds();
rf.informOut = INFORM_UNINITIALIZED;
{
int mode = 0;
rf.solFun(currEst.data(), &mode);
if (mode == -1) {
rf.informOut = INFORM_STARTING_VALUES_INFEASIBLE;
return;
}
}
double refFit = rf.getFit();
rf.grad.resize(rf.numFree);
fit_functional ff(rf);
Eigen::VectorXd majorEst = currEst;
while(++iter < maxIter && !isErrorRaised()) {
gradient_with_ref(rf.gradientAlgo, 1, rf.gradientIterations, rf.gradientStepSize,
ff, refFit, majorEst, rf.grad);
if (rf.verbose >= 3) mxPrintMat("grad", rf.grad);
if(rf.grad.norm() == 0)
{
rf.informOut = INFORM_CONVERGED_OPTIMUM;
if(rf.verbose >= 2) mxLog("After %i iterations, gradient achieves zero!", iter);
break;
}
int retries = 300;
double speed = std::min(priorSpeed, 1.0);
double bestSpeed = speed;
bool foundBetter = false;
Eigen::VectorXd bestEst(majorEst.size());
Eigen::VectorXd prevEst(majorEst.size());
Eigen::VectorXd searchDir = rf.grad;
searchDir /= searchDir.norm();
prevEst.setConstant(nan("uninit"));
while (--retries > 0 && !isErrorRaised()){
Eigen::VectorXd nextEst = majorEst - speed * searchDir;
nextEst = nextEst.cwiseMax(rf.solLB).cwiseMin(rf.solUB);
if (nextEst == prevEst) break;
prevEst = nextEst;
rf.checkActiveBoxConstraints(nextEst);
int mode = 0;
double fit = rf.solFun(nextEst.data(), &mode);
if (fit < refFit) {
foundBetter = true;
refFit = rf.getFit();
bestSpeed = speed;
bestEst = nextEst;
break;
}
speed *= shrinkage;
}
if (false && foundBetter) {
// In some tests, this did not help so it is not enabled.
// It might be worth testing more.
mxLog("trying larger step size");
retries = 3;
while (--retries > 0 && !isErrorRaised()){
speed *= 1.01;
Eigen::VectorXd nextEst = majorEst - speed * searchDir;
nextEst = nextEst.cwiseMax(rf.solLB).cwiseMin(rf.solUB);
rf.checkActiveBoxConstraints(nextEst);
int mode = 0;
double fit = rf.solFun(nextEst.data(), &mode);
if (fit < refFit) {
foundBetter = true;
refFit = rf.getFit();
bestSpeed = speed;
bestEst = nextEst;
}
}
}
if (!foundBetter) {
rf.informOut = INFORM_CONVERGED_OPTIMUM;
if(rf.verbose >= 2) mxLog("After %i iterations, cannot find better estimation along the gradient direction", iter);
break;
}
if (rf.verbose >= 2) mxLog("major fit %f bestSpeed %g", refFit, bestSpeed);
majorEst = bestEst;
priorSpeed = bestSpeed * 1.1;
}
rf.est = majorEst;
if ((rf.grad.array().abs() > 0.1).any()) {
rf.informOut = INFORM_NOT_AT_OPTIMUM;
}
if (iter >= maxIter - 1) {
rf.informOut = INFORM_ITERATION_LIMIT;
if(rf.verbose >= 2) mxLog("Maximum iteration achieved!");
}
if(rf.verbose >= 1) mxLog("Status code : %i", rf.informOut);
}
void omxInitExpectationBA81(omxExpectation* oo) {
omxState* currentState = oo->currentState;
SEXP rObj = oo->rObj;
SEXP tmp;
if(OMX_DEBUG) {
mxLog("Initializing %s.", oo->name);
}
if (!Glibrpf_model) {
#if USE_EXTERNAL_LIBRPF
get_librpf_t get_librpf = (get_librpf_t) R_GetCCallable("rpf", "get_librpf_model_GPL");
(*get_librpf)(LIBIFA_RPF_API_VERSION, &Glibrpf_numModels, &Glibrpf_model);
#else
// if linking against included source code
Glibrpf_numModels = librpf_numModels;
Glibrpf_model = librpf_model;
#endif
}
BA81Expect *state = new BA81Expect;
// These two constants should be as identical as possible
state->name = oo->name;
if (0) {
state->LogLargestDouble = 0.0;
state->LargestDouble = 1.0;
} else {
state->LogLargestDouble = log(std::numeric_limits<double>::max()) - 1;
state->LargestDouble = exp(state->LogLargestDouble);
ba81NormalQuad &quad = state->getQuad();
quad.setOne(state->LargestDouble);
}
state->expectedUsed = false;
state->estLatentMean = NULL;
state->estLatentCov = NULL;
state->type = EXPECTATION_OBSERVED;
state->itemParam = NULL;
state->EitemParam = NULL;
state->itemParamVersion = 0;
state->latentParamVersion = 0;
oo->argStruct = (void*) state;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("data")));
state->data = omxDataLookupFromState(tmp, currentState);
}
if (strcmp(omxDataType(state->data), "raw") != 0) {
omxRaiseErrorf("%s unable to handle data type %s", oo->name, omxDataType(state->data));
return;
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("verbose")));
state->verbose = Rf_asInteger(tmp);
}
int targetQpoints;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("qpoints")));
targetQpoints = Rf_asInteger(tmp);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("qwidth")));
state->grp.setGridFineness(Rf_asReal(tmp), targetQpoints);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("ItemSpec")));
state->grp.importSpec(tmp);
if (state->verbose >= 2) mxLog("%s: found %d item specs", oo->name, state->numItems());
}
state->_latentMeanOut = omxNewMatrixFromSlot(rObj, currentState, "mean");
state->_latentCovOut = omxNewMatrixFromSlot(rObj, currentState, "cov");
state->itemParam = omxNewMatrixFromSlot(rObj, currentState, "item");
state->grp.param = state->itemParam->data; // algebra not allowed yet TODO
const int numItems = state->itemParam->cols;
if (state->numItems() != numItems) {
omxRaiseErrorf("ItemSpec length %d must match the number of item columns (%d)",
state->numItems(), numItems);
return;
}
if (state->itemParam->rows != state->grp.impliedParamRows) {
omxRaiseErrorf("item matrix must have %d rows", state->grp.impliedParamRows);
return;
}
state->grp.paramRows = state->itemParam->rows;
// for algebra item param, will need to defer until later?
state->grp.learnMaxAbilities();
int maxAbilities = state->grp.itemDims;
state->grp.setFactorNames(state->itemParam->rownames);
{
ProtectedSEXP tmp2(R_do_slot(rObj, Rf_install(".detectIndependence")));
state->grp.detectIndependence = Rf_asLogical(tmp2);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("EstepItem")));
if (!Rf_isNull(tmp)) {
int rows, cols;
getMatrixDims(tmp, &rows, &cols);
if (rows != state->itemParam->rows || cols != state->itemParam->cols) {
Rf_error("EstepItem must have the same dimensions as the item MxMatrix");
}
state->EitemParam = REAL(tmp);
}
}
oo->computeFun = ba81compute;
oo->setVarGroup = ignoreSetVarGroup;
oo->destructFun = ba81Destroy;
oo->populateAttrFun = ba81PopulateAttributes;
oo->componentFun = getComponent;
oo->canDuplicate = false;
// TODO: Exactly identical rows do not contribute any information.
// The sorting algorithm ought to remove them so we get better cache behavior.
// The following summary stats would be cheaper to calculate too.
omxData *data = state->data;
if (data->hasDefinitionVariables()) Rf_error("%s: not implemented yet", oo->name);
std::vector<int> &rowMap = state->grp.rowMap;
int weightCol;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("weightColumn")));
weightCol = INTEGER(tmp)[0];
}
if (weightCol == NA_INTEGER) {
// Should rowMap be part of omxData? This is essentially a
// generic compression step that shouldn't be specific to IFA models.
state->grp.rowWeight = (double*) R_alloc(data->rows, sizeof(double));
rowMap.resize(data->rows);
int numUnique = 0;
for (int rx=0; rx < data->rows; ) {
int rw = 1;
state->grp.rowWeight[numUnique] = rw;
rowMap[numUnique] = rx;
rx += rw;
++numUnique;
}
rowMap.resize(numUnique);
state->weightSum = state->data->rows;
}
else {
if (omxDataColumnIsFactor(data, weightCol)) {
omxRaiseErrorf("%s: weightColumn %d is a factor", oo->name, 1 + weightCol);
return;
}
state->grp.rowWeight = omxDoubleDataColumn(data, weightCol);
state->weightSum = 0;
for (int rx=0; rx < data->rows; ++rx) { state->weightSum += state->grp.rowWeight[rx]; }
rowMap.resize(data->rows);
for (size_t rx=0; rx < rowMap.size(); ++rx) {
rowMap[rx] = rx;
}
}
// complain about non-integral rowWeights (EAP can't work) TODO
auto colMap = oo->getDataColumns();
for (int cx = 0; cx < numItems; cx++) {
int *col = omxIntDataColumnUnsafe(data, colMap[cx]);
state->grp.dataColumns.push_back(col);
}
// sanity check data
for (int cx = 0; cx < numItems; cx++) {
if (!omxDataColumnIsFactor(data, colMap[cx])) {
data->omxPrintData("diagnostic", 3);
omxRaiseErrorf("%s: column %d is not a factor", oo->name, int(1 + colMap[cx]));
return;
}
}
// TODO the max outcome should be available from omxData
for (int rx=0; rx < data->rows; rx++) {
int cols = 0;
for (int cx = 0; cx < numItems; cx++) {
const int *col = state->grp.dataColumns[cx];
int pick = col[rx];
if (pick == NA_INTEGER) continue;
++cols;
const int no = state->grp.itemOutcomes[cx];
if (pick > no) {
Rf_error("Data for item '%s' has at least %d outcomes, not %d",
state->itemParam->colnames[cx], pick, no);
}
}
if (cols == 0) {
Rf_error("Row %d has all NAs", 1+rx);
}
}
if (state->_latentMeanOut && state->_latentMeanOut->rows * state->_latentMeanOut->cols != maxAbilities) {
Rf_error("The mean matrix '%s' must be a row or column vector of size %d",
state->_latentMeanOut->name(), maxAbilities);
}
if (state->_latentCovOut && (state->_latentCovOut->rows != maxAbilities ||
state->_latentCovOut->cols != maxAbilities)) {
Rf_error("The cov matrix '%s' must be %dx%d",
state->_latentCovOut->name(), maxAbilities, maxAbilities);
}
state->grp.setLatentDistribution(state->_latentMeanOut? state->_latentMeanOut->data : NULL,
state->_latentCovOut? state->_latentCovOut->data : NULL);
{
EigenArrayAdaptor Eparam(state->itemParam);
Eigen::Map< Eigen::VectorXd > meanVec(state->grp.mean, maxAbilities);
Eigen::Map< Eigen::MatrixXd > covMat(state->grp.cov, maxAbilities, maxAbilities);
state->grp.quad.setStructure(state->grp.qwidth, state->grp.qpoints,
Eparam, meanVec, covMat);
}
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("minItemsPerScore")));
state->grp.setMinItemsPerScore(Rf_asInteger(tmp));
}
state->grp.buildRowSkip();
if (isErrorRaised()) return;
{ScopedProtect p1(tmp, R_do_slot(rObj, Rf_install("debugInternal")));
state->debugInternal = Rf_asLogical(tmp);
}
state->ElatentVersion = 0;
if (state->_latentMeanOut) {
state->estLatentMean = omxInitMatrix(maxAbilities, 1, TRUE, currentState);
omxCopyMatrix(state->estLatentMean, state->_latentMeanOut); // rename matrices TODO
}
if (state->_latentCovOut) {
state->estLatentCov = omxInitMatrix(maxAbilities, maxAbilities, TRUE, currentState);
omxCopyMatrix(state->estLatentCov, state->_latentCovOut);
}
}
