SegUsageCalcMap(HadoopPipes::TaskContext& context) {
//inputWords = context.getCounter(WORDCOUNT, INPUT_WORDS);
const HadoopPipes::JobConf* job = context.getJobConf();
std::string inputDir = job->get("mapred.input.dir");
const char * uri = inputDir.c_str();
const char * fs_name = g_basename(inputDir.c_str());
printf("DBG:--input Dir:%s,fs_name:%s,uri:%s\n",inputDir.c_str(),fs_name,uri);
m_storage = init_storage_handler(uri);
HADOOP_ASSERT(m_storage != NULL, "failed to init storage handler ");
uint32_t segment_size;
uint32_t block_size;
uint64_t max_fs_size;
int ret = read_fs_meta(m_storage, &segment_size, &block_size,&max_fs_size);
printf("DBG:--segment size:%u,block size:%u,max fs size%llu\n",segment_size,block_size,max_fs_size);
//m_latest_inode = load_latest_inode(m_storage);
//HADOOP_ASSERT(m_latest_inode != NULL, "failed to load latest inode ");
m_block_size = block_size;
}
/*
* End implemented functions for local mode.
*/
SyntopReducer::SyntopReducer(HadoopPipes::TaskContext& context) { // NOLINT
SyntopParameters* params = new SyntopParameters();
params->set_vocab_size(context.getJobConf()->getInt("syntop.vocab.number"));
params->set_num_docs(context.getJobConf()->getInt("syntop.doc.number"));
params->set_num_topics(context.getJobConf()->getInt("syntop.topic.number"));
params->set_model_name(context.getJobConf()->get("syntop.model.name"));
params->set_finite(context.getJobConf()->getBoolean("syntop.model.finite"));
params->set_ignore_trans(context.getJobConf()->
getBoolean("syntop.model.ignore.trans"));
params->set_ignore_docs(context.getJobConf()->
getBoolean("syntop.model.ignore.docs"));
params->set_shortcut_gsl(context.getJobConf()->
getBoolean("syntop.model.shortcut.gsl"));
params->set_alpha_doc(context.getJobConf()->getFloat("syntop.alpha.doc"));
params->set_alpha_trans(context.getJobConf()->getFloat("syntop.alpha.trans"));
params->set_alpha_top(context.getJobConf()->getFloat("syntop.alpha.top"));
params_.reset(params);
vars_ = new VariationalParameters(*params_);
check_order_ = false;
lhood_ = false;
index = -1;
tau_coordinate_ = -1;
nu_coordinate_ = -1;
}
MergeReducer(HadoopPipes::TaskContext& context) {
trimmed = context.getCounter(MERGE, MERGE_TRIMMED);
bad = context.getCounter(MERGE, MERGE_BAD);
}
