int DASIncReader::init(const das_inc_conf_t &conf)
{
int ret = 0;
_max_lines_per_round = conf.max_lines_per_round;
_cur_lines = 0;
_desc = conf.pipe_name;
ret = _reader.init(conf.configio_xml_path.c_str());
if (0 != ret) {
DL_LOG_FATAL("fail to init InputObject, conf[%s].",
conf.configio_xml_path.c_str());
return -1;
}
// open and seek.
ret = _reader.open();
if (0 != ret) {
DL_LOG_FATAL("fail to open InputObject, conf[%s].",
conf.configio_xml_path.c_str());
return -1;
}
ret = _reader.seek(&conf.last_event_id, sizeof(conf.last_event_id));
if (ret < 0) {
DL_LOG_FATAL("fail to seek to event_id[%llu], conf[%s], ret[%d].",
conf.last_event_id, conf.configio_xml_path.c_str(),ret);
return -1;
}
_start_event_ids = conf.last_event_id;
_has_sought = (0 == ret);
_last_event_ids = conf.last_event_id;
return 0;
}
void* inc_run(das_lib::conf_t* conf)
{
ul_logstat_t ls;
ls.spec = FLAGS_log_to_screen ? UL_LOGTTY : 0;
ls.to_syslog = 15;
ls.events = 15;
if (ul_openlog_r("inc_thread", &ls) < 0) {
DL_LOG_FATAL("Fail to open log for inc_thread");
return NULL;
}
do {
int ret = 0;
DL_LOG_TRACE("Begin loading base indexes");
ret = conf->bdlib.start_service();
if (!ret) {
DL_LOG_FATAL("failed to load base index!");
break;
}
DL_LOG_TRACE("End loading base indexes");
system("mv data/inc/dsp.event.1 data/inc/dsp.event.1_bak");
system("echo all > dump_req");
for (long i = 0; 0 == FLAGS_inc_rounds || i < FLAGS_inc_rounds; ++i) {
//each time
DL_LOG_TRACE("Begin bdlib.handle_inc, round(%ld)", i);
ret = conf->bdlib.handle_inc();
if(!ret) {
DL_LOG_WARNING("Error occurred in bdlib.handle_inc()");
}
DL_LOG_TRACE("End bdlib.handle_inc");
if (FLAGS_sleep_interval > 0) {
DL_LOG_TRACE("Sleep for %d seconds", FLAGS_sleep_interval);
sleep(FLAGS_sleep_interval);
}
system("mv data/inc/dsp.event.1_bak data/dsp.event.1");
}
} while (0);
ul_closelog_r(0);
return NULL;
}
bool test_update_handler(TestUnitPlanTable &table, const IncRecordType &record)
{
static int aaa = 0;
aaa ++;
// DL_LOG_FATAL("!!!!!!!!!!test_update_handler called aaa[%d]",aaa);
uint32_t event_id = 0;
uint32_t op_type = 0;
uint32_t unit_id = 0;
uint32_t plan_id = 0;
if (0 != get_and_check_uint32(record, 1, "event_id", event_id, 0, UINT_MAX)) {
DL_LOG_FATAL("Failed to get event_id");
return -1;
}
if (0 != get_and_check_uint32(record, 2, "op_type", op_type, 0, 3)) {
DL_LOG_FATAL("Failed to get op_type");
return -1;
}
if (0 != get_and_check_uint32(record, 3, "uint_id", unit_id, 1, UINT_MAX)) {
DL_LOG_FATAL("Failed to get plan_id");
return -1;
}
if (0 != get_and_check_uint32(record, 4, "plan_id", plan_id, 1,UINT_MAX)) {
DL_LOG_FATAL("Failed to get plan_id");
return -1;
}
// DL_LOG_WARNING("!!![%u],[%u],[%u],[%u]",
// event_id,op_type,unit_id,plan_id);
const bool ERASE_TABLE = (op_type == 2);
const bool INSERT_TABLE = (op_type == 1);
// Do changes
if (ERASE_TABLE) {
table.erase<UNIT_ID>(unit_id);
}
if (INSERT_TABLE) {
table.insert(unit_id,plan_id);
}
return true;
}
int DASIncReader::read_next(configio::DynamicRecord &record)
{
int ret = 0;
unsigned long long cur_event_id;
// check lines.
if (_max_lines_per_round > 0 &&
_cur_lines == _max_lines_per_round) {
DL_LOG_TRACE("read enough lines[%d], max lines[%d]",
_cur_lines, _max_lines_per_round);
_cur_lines = 0;
return READ_END;
}
// read next record.
ret = _reader.get_next_record(record);
if (ret < 0) {
return READ_FAIL;
}
if (ret == 1) {
_cur_lines = 0;
return READ_END;
}
// check event id.
ret = record.get_uint64(1, cur_event_id);
if (0 != ret) {
DL_LOG_FATAL("fail to get event id from record");
return -1;
}
if (cur_event_id < _last_event_ids) {
DL_LOG_FATAL("current_event_id[%llu] is less than previous[%llu]",
cur_event_id,
_last_event_ids);
return -1;
}
_last_event_ids = cur_event_id;
_cur_lines++;
return 0;
}
int main(int argc,char** argv)
{
int ret = 0;
das_lib::conf_t conf;
FLAGS_flagfile="./conf/startup/gflags.conf";
ret = google::ParseCommandLineFlags(&argc, &argv, true);
if (ret < 0) {
DL_LOG_FATAL("Fail to parse flags");
return -1;
}
ul_logstat_t ls;
ls.spec = FLAGS_log_to_screen ? UL_LOGTTY : 0;
ls.to_syslog = 15;
ls.events = 15;
if (ul_openlog("./log", FLAGS_log_prefix_name.c_str(), &ls, 100) < 0) {
DL_LOG_FATAL("Fail to open log");
}
if (ret < 0) {
DL_LOG_FATAL ("Fail to load_conf!");
return -1;
}
mrf::ConfigureService &cs = *mrf::GlobalConfigureService::instance();
if (!cs.load("./conf/startup/bs.conf")) {
return -1;
}
ret = conf.bdlib.init(cs["lib_conf"]);
if (! ret ) {
DL_LOG_FATAL ("Fail to init bdlib!");
return -1;
}
DL_LOG_TRACE("Create inc thread");
inc_run(&conf);
ul_closelog(0);
return 0;
}
int load_test_unit_table(TestUnitTable &table, const char* fpath, const PartitionArg &)
{
if (NULL == fpath) {
DL_LOG_FATAL("Param[fpath] is NULL");
return -1;
}
return table.insert_by_file(fpath, '\t');
}
bool LibManager::load_base_indexes(const char *p_path, const char *p_name)
{
if (p_path == NULL || p_name == NULL) {
DL_LOG_FATAL("Null param %p%p",p_path,p_name);
return false;
}
int pos = -1;
pos = _vm_tg.create_version();
if (pos < 0) {
DL_LOG_FATAL("Failed to create table group verion");
return false;
}
_vm_tg[pos].load();
_vm_tg[pos].serialize();
_vm_tg.freeze_version(pos);
return true;
}
int load_test_unit_plan_table(TestUnitPlanTable &table, const char* fpath, const PartitionArg &)
{
if (NULL == fpath) {
DL_LOG_FATAL("Param[fpath] is NULL");
return -1;
}
char line[2048];
unit_plan_line_t node;
int ret = 0;
int line_no = 1;
int n_succ = 0;
FILE *fp = fopen(fpath, "r");
if (NULL == fp) {
DL_LOG_FATAL("Fail to open file at '%s'", fpath);
return -1;
}
table.clear();
for (; fgets(line, sizeof(line), fp); ++line_no) {
line[sizeof(line) - 1] = 0;
ret = parse_test_unit_plan_line(&node, line);
if (ret < 0) {
DL_LOG_WARNING("Ignore line %d", line_no);
continue;
}
if (0 == table.insert(node.unit_id, node.plan_id)) {
DL_LOG_FATAL("Fail to insert unit_id=%u plan_id=%u into"
" TestUnitPlanTable, quit loading",
node.unit_id, node.plan_id);
break;
}
++ n_succ;
}
fclose(fp);
return n_succ;
}
bool LibManager::handle_inc()
{
int ret = 0;
st::Timer tm;
event_id_t last_event_id = 0;
event_id_t cur_eid = 0;
uint32_t count = 0;
configio::DynamicRecord record;
ret = _inc_reader.read_next(record);
if (ret < 0) {
DL_LOG_FATAL("fail to read next inc");
return false;
} else if (DASIncReader::READ_END == ret) {
DL_LOG_TRACE("No DAS inc to handle since last round");
return true;
}
TableGroup* p_table = NULL;
DL_LOG_TRACE("Begin creating new version of main_table_group");
tm.start();
int pos = _vm_tg.create_version();
if (pos < 0) {
DL_LOG_FATAL("Fail to create a new version of main_table_group");
return false;
}
p_table = &_vm_tg[pos];
tm.stop();
DL_LOG_TRACE("End creating new version of main_table_group: %lums",
tm.m_elapsed());
tm.start();
do {
// get current event id.
count ++;
if (0 != get_and_check_uint64(record, 1, "event_id", cur_eid)) {
return -1;
}
ret = p_table->handle_inc(record);
if (ret < 0) {
DL_LOG_FATAL("fail to handle_inc, event_id=%llu", cur_eid);
return -1;
}
last_event_id = cur_eid;
// get next record
ret = _inc_reader.read_next(record);
if (ret < 0) {
DL_LOG_FATAL("fail to read next budget inc");
return -1;
} else if (DASIncReader::READ_END == ret) {
break;
}
} while (true);
_vm_tg[pos].serialize();
_vm_tg.freeze_version(pos);
return ret;
}
bool LibManager::init()
{
_partition_arg.set_partition_idx(1).set_partition_num(1);
int pos = -1;
int ret = -1;
_vm_tg.init(2,"table_group_version_manager");
pos = _vm_tg.create_version();
if (pos < 0) {
DL_LOG_FATAL("Failed to create table group verion");
return false;
}
TableGroup &table_group = _vm_tg[pos];
bool ret_b = false;
#define NEW_AND_REGISTER_TABLE(type, name, load_strategy, update_strategy, level, table_group) \
TableManager<type> * name = \
new(std::nothrow) TableManager<type>( #name, \
load_strategy, \
update_strategy, \
&(table_group)); \
if (NULL == name) { \
return false; \
} \
ret_b = table_group.register_table(name, level); \
if (!ret_b) { \
return false; \
}
//add tables
IBaseLoadStrategy<TestUnitPlanTable> *p_load_strategy =
new(std::nothrow) LiteralBaseLoadStrategy<TestUnitPlanTable>("xml1",test_update_handler);
if(NULL == p_load_strategy) {
DL_LOG_FATAL("fail to allocate load strategy");
return false;
}
IBaseUpdateStrategy<TestUnitPlanTable>* p_update_strategy
= new(std::nothrow) IncUpdateStrategy<TestUnitPlanTable>(test_update_handler);
NEW_AND_REGISTER_TABLE(TestUnitPlanTable, unit_plan_table, p_load_strategy, p_update_strategy,
DEFAULT_LEVEL, table_group);
IBaseLoadStrategy<TestUnitTable> *p_test_unit_load_strategy =
new(std::nothrow) LiteralBaseLoadStrategy<TestUnitTable>("xml2",test_unit_update_handler);
if(NULL == p_test_unit_load_strategy) {
DL_LOG_FATAL("fail to allocate load strategy");
return false;
}
NEW_AND_REGISTER_TABLE(TestUnitTable, unit_table, p_test_unit_load_strategy, NULL,
RELOAD_LEVEL, table_group);
// inv table
p_test_unit_load_strategy =
new(std::nothrow) ConnectorLoadStrategy<TestUnitTable, TestUnitCon>(
"inv_unit_con",
unit_plan_exp_connector_maker,
&table_group);
if(NULL == p_test_unit_load_strategy) {
DL_LOG_FATAL("fail to allocate load strategy");
return false;
}
NEW_AND_REGISTER_TABLE(TestUnitTable, inv_unit_table, p_test_unit_load_strategy, NULL,
RELOAD_LEVEL, table_group);
#undef NEW_AND_REGISTER_TABLE
ret = _vm_tg[pos].init();
if (ret < 0) {
DL_LOG_FATAL("Init table group version manager failed");
return false;
}
_vm_tg.freeze_version(pos);
//初始化增量模型
das_inc_conf_t das_inc_conf;
das_inc_conf.configio_xml_path = "./test_lib.xml";
das_inc_conf.last_event_id = 0;
das_inc_conf.max_lines_per_round = 1000;
das_inc_conf.pipe_name = "das_inc";
if (0 != _inc_reader.init(das_inc_conf)) {
DL_LOG_FATAL("fail to init das_inc_reader");
return false;
}
return true;
}
