Пример #1
0
        SegUsageCalcMap(HadoopPipes::TaskContext& context) {
            //inputWords = context.getCounter(WORDCOUNT, INPUT_WORDS);
            const HadoopPipes::JobConf* job = context.getJobConf();
            std::string inputDir = job->get("mapred.input.dir");
            const char * uri = inputDir.c_str();
            const char * fs_name = g_basename(inputDir.c_str());
	     printf("DBG:--input Dir:%s,fs_name:%s,uri:%s\n",inputDir.c_str(),fs_name,uri);
            m_storage = init_storage_handler(uri);
            HADOOP_ASSERT(m_storage != NULL, "failed to init storage handler ");
            uint32_t segment_size;
            uint32_t block_size;
            uint64_t max_fs_size;
            int ret = read_fs_meta(m_storage, &segment_size, &block_size,&max_fs_size);
            printf("DBG:--segment size:%u,block size:%u,max fs size%llu\n",segment_size,block_size,max_fs_size);
            //m_latest_inode = load_latest_inode(m_storage); 
     	    //HADOOP_ASSERT(m_latest_inode != NULL, "failed to load latest inode ");
            m_block_size = block_size;
        }            
Пример #2
0
/*
 * End implemented functions for local mode.
 */
SyntopReducer::SyntopReducer(HadoopPipes::TaskContext& context) { // NOLINT
  SyntopParameters* params = new SyntopParameters();

  params->set_vocab_size(context.getJobConf()->getInt("syntop.vocab.number"));
  params->set_num_docs(context.getJobConf()->getInt("syntop.doc.number"));
  params->set_num_topics(context.getJobConf()->getInt("syntop.topic.number"));
  params->set_model_name(context.getJobConf()->get("syntop.model.name"));

  params->set_finite(context.getJobConf()->getBoolean("syntop.model.finite"));
  params->set_ignore_trans(context.getJobConf()->
                           getBoolean("syntop.model.ignore.trans"));
  params->set_ignore_docs(context.getJobConf()->
                          getBoolean("syntop.model.ignore.docs"));
  params->set_shortcut_gsl(context.getJobConf()->
                           getBoolean("syntop.model.shortcut.gsl"));

  params->set_alpha_doc(context.getJobConf()->getFloat("syntop.alpha.doc"));
  params->set_alpha_trans(context.getJobConf()->getFloat("syntop.alpha.trans"));
  params->set_alpha_top(context.getJobConf()->getFloat("syntop.alpha.top"));

  params_.reset(params);

  vars_ = new VariationalParameters(*params_);

  check_order_ = false;
  lhood_ = false;

  index = -1;

  tau_coordinate_ = -1;
  nu_coordinate_ = -1;
}