void Segmentor::getGoldActions(const vector<Instance>& vecInsts, vector<vector<CAction> >& vecActions) {
vecActions.clear();
static Metric segEval, posEval;
static CStateItem state[m_classifier.MAX_SENTENCE_SIZE];
static CResult output;
static CAction answer;
segEval.reset();
posEval.reset();
static int numInstance, actionNum;
vecActions.resize(vecInsts.size());
for (numInstance = 0; numInstance < vecInsts.size(); numInstance++) {
const Instance &instance = vecInsts[numInstance];
actionNum = 0;
state[actionNum].initSentence(&instance.chars, &instance.candidateLabels);
state[actionNum].clear();
while (!state[actionNum].IsTerminated()) {
state[actionNum].getGoldAction(instance, m_classifier.fe._postagAlphabet, answer);
vecActions[numInstance].push_back(answer);
state[actionNum].move(state + actionNum + 1, answer, m_classifier.fe._postagAlphabet);
actionNum++;
}
if (actionNum - 1 != instance.charsize()) {
std::cout << "action number is not correct, please check" << std::endl;
}
state[actionNum].getSegPosResults(output);
instance.evaluate(output, segEval, posEval);
if (!segEval.bIdentical() || !posEval.bIdentical()) {
std::cout << "error state conversion!" << std::endl;
std::cout << "output instance:" << std::endl;
for (int tmpK = 0; tmpK < instance.words.size(); tmpK++) {
std::cout << instance.words[tmpK] << "_" << instance.postags[tmpK] << " ";
}
std::cout << std::endl;
std::cout << "predicated instance:" << std::endl;
for (int tmpK = 0; tmpK < output.size(); tmpK++) {
std::cout << output.words[tmpK] << "_" << output.postags[tmpK] << " ";
}
std::cout << std::endl;
exit(0);
}
if ((numInstance + 1) % m_options.verboseIter == 0) {
cout << numInstance + 1 << " ";
if ((numInstance + 1) % (40 * m_options.verboseIter) == 0)
cout << std::endl;
cout.flush();
}
if (m_options.maxInstance > 0 && numInstance == m_options.maxInstance)
break;
}
}
// all linear features are extracted from positive examples
int Segmentor::createAlphabet(const vector<Instance>& vecInsts) {
cout << "Creating Alphabet..." << endl;
int numInstance = vecInsts.size();
hash_map<string, int> action_stat;
hash_map<string, int> feat_stat;
hash_map<string, int> postag_stat;
assert(numInstance > 0);
static Metric segEval, posEval;
static CStateItem state[m_classifier.MAX_SENTENCE_SIZE];
static Feature feat;
static CResult output;
static CAction answer;
static int actionNum;
m_classifier.initAlphabet();
segEval.reset();
posEval.reset();
int maxFreqChar = -1;
int maxFreqWord = -1;
for (numInstance = 0; numInstance < vecInsts.size(); numInstance++) {
const Instance &instance = vecInsts[numInstance];
for (int idx = 0; idx < instance.postagsize(); idx++) {
postag_stat[instance.postags[idx]];
m_classifier.fe._tagConstraints.addWordPOSPair(instance.words[idx], instance.postags[idx]);
}
}
m_classifier.addToPostagAlphabet(postag_stat);
for (numInstance = 0; numInstance < vecInsts.size(); numInstance++) {
const Instance &instance = vecInsts[numInstance];
actionNum = 0;
state[actionNum].initSentence(&instance.chars, &instance.candidateLabels);
state[actionNum].clear();
while (!state[actionNum].IsTerminated()) {
state[actionNum].getGoldAction(instance, m_classifier.fe._postagAlphabet, answer);
action_stat[answer.str()]++;
m_classifier.extractFeature(state + actionNum, answer, feat);
for (int idx = 0; idx < feat._strSparseFeat.size(); idx++) {
feat_stat[feat._strSparseFeat[idx]]++;
}
state[actionNum].move(state + actionNum + 1, answer, m_classifier.fe._postagAlphabet);
actionNum++;
}
if (actionNum - 1 != instance.charsize()) {
std::cout << "action number is not correct, please check" << std::endl;
}
state[actionNum].getSegPosResults(output);
instance.evaluate(output, segEval, posEval);
if (!segEval.bIdentical() || !posEval.bIdentical()) {
std::cout << "error state conversion!" << std::endl;
std::cout << "output instance:" << std::endl;
for (int tmpK = 0; tmpK < instance.words.size(); tmpK++) {
std::cout << instance.words[tmpK] << "_" << instance.postags[tmpK] << " ";
}
std::cout << std::endl;
std::cout << "predicated instance:" << std::endl;
for (int tmpK = 0; tmpK < output.size(); tmpK++) {
std::cout << output.words[tmpK] << "_" << output.postags[tmpK] << " ";
}
std::cout << std::endl;
exit(0);
}
if ((numInstance + 1) % m_options.verboseIter == 0) {
cout << numInstance + 1 << " ";
if ((numInstance + 1) % (40 * m_options.verboseIter) == 0)
cout << std::endl;
cout.flush();
}
if (m_options.maxInstance > 0 && numInstance == m_options.maxInstance)
break;
}
m_classifier.addToActionAlphabet(action_stat);
m_classifier.addToFeatureAlphabet(feat_stat, m_options.featCutOff);
cout << numInstance << " " << endl;
cout << "Action num: " << m_classifier.fe._actionAlphabet.size() << endl;
cout << "Pos num: " << m_classifier.fe._postagAlphabet.size() << endl;
cout << "Total feat num: " << feat_stat.size() << endl;
cout << "Remain feat num: " << m_classifier.fe._featAlphabet.size() << endl;
//m_classifier.setFeatureCollectionState(false);
return 0;
}
