int sl_write_delete_sql (char *class_name, MOBJ mclass, DB_VALUE * key) { PARSER_CONTEXT *parser; PARSER_VARCHAR *buffer = NULL, *pkey; PARSER_VARCHAR *select; parser = parser_create_parser (); pkey = sl_print_pk (parser, (SM_CLASS *) mclass, key); if (pkey == NULL) { parser_free_parser (parser); return ER_FAILED; } buffer = pt_append_nulstring (parser, buffer, "DELETE FROM ["); buffer = pt_append_nulstring (parser, buffer, class_name); buffer = pt_append_nulstring (parser, buffer, "] WHERE "); buffer = pt_append_varchar (parser, buffer, pkey); buffer = pt_append_nulstring (parser, buffer, ";"); select = sl_print_select (parser, class_name, pkey); if (sl_write_sql (buffer, select) != NO_ERROR) { parser_free_parser (parser); return ER_FAILED; } parser_free_parser (parser); return NO_ERROR; }
void load(const std::string& model_path) { if (model == NULL) { model = parser_create_parser(model_path.c_str()); } else { std::cerr << "Parser: Model reloaded!" << std::endl; } }
JNIEXPORT jint JNICALL Java_edu_hit_ir_ltp4j_Parser_create (JNIEnv * env, jclass obj, jstring model_path){ const char * str = env->GetStringUTFChars( model_path , 0); if(!parser){ parser = parser_create_parser(str); } env->ReleaseStringUTFChars( model_path, str); if(parser) { return 1; } return -1; }
TEST(parser_interface_unittest, test_empty_list) { void * engine = parser_create_parser("./ltp_data/parser.model"); EXPECT_TRUE(NULL != engine); std::vector<std::string> words; std::vector<std::string> tags; std::vector<int> heads; std::vector<std::string> deprels; int nr_words = parser_parse(engine, words, tags ,heads ,deprels); EXPECT_EQ(0, nr_words); parser_release_parser(engine); }
TEST(parser_interface_unittest, test_normal) { void * engine = parser_create_parser("./ltp_data/parser.model"); EXPECT_TRUE(NULL != engine); std::vector<std::string> words; std::vector<std::string> tags; std::vector<int> heads; std::vector<std::string> deprels; for (int i = 0; i < kNumNormalWords; ++ i) { words.push_back(kNormalWords[i]); tags.push_back(kNormalPostags[i]); } int nr_words = parser_parse(engine, words, tags ,heads ,deprels); // tagged words should be greater than 4 EXPECT_GT(nr_words, 0); parser_release_parser(engine); }
/* ====================================================== * * Parser related resource * * ====================================================== */ int LTPResource::LoadParserResource(const char * model_file) { if (m_isParserResourceLoaded) { return 0; } TRACE_LOG("Loading parser resource from \"%s\"", model_file); m_parser = parser_create_parser(model_file); if (!m_parser) { ERROR_LOG("Failed to create parser"); return -1; } TRACE_LOG("Parser is loaded."); m_isParserResourceLoaded = true; return 0; }
TEST(parser_interface_unittest, test_empty_tag) { void * engine = parser_create_parser("./ltp_data/parser.model"); EXPECT_TRUE(NULL != engine); std::vector<std::string> words; std::vector<std::string> tags; std::vector<int> heads; std::vector<std::string> deprels; for (int i = 0; i < kNumNormalWords; ++ i) { if (i == 2) { tags.push_back(""); } else { tags.push_back(kNormalPostags[i]); } words.push_back(kNormalWords[i]); } int nr_words = parser_parse(engine, words, tags, heads, deprels); EXPECT_EQ(0, nr_words); parser_release_parser(engine); }
int sl_write_insert_sql (DB_OTMPL * inst_tp, DB_VALUE * key) { PARSER_CONTEXT *parser; PARSER_VARCHAR *buffer = NULL; PARSER_VARCHAR *att_names, *att_values; PARSER_VARCHAR *pkey; PARSER_VARCHAR *select; parser = parser_create_parser (); att_names = sl_print_insert_att_names (parser, inst_tp->assignments, inst_tp->nassigns); att_values = sl_print_insert_att_values (parser, inst_tp->assignments, inst_tp->nassigns); buffer = pt_append_nulstring (parser, buffer, "INSERT INTO ["); buffer = pt_append_nulstring (parser, buffer, sm_ch_name ((MOBJ) (inst_tp->class_))); buffer = pt_append_nulstring (parser, buffer, "]("); buffer = pt_append_varchar (parser, buffer, att_names); buffer = pt_append_nulstring (parser, buffer, ") VALUES ("); buffer = pt_append_varchar (parser, buffer, att_values); buffer = pt_append_nulstring (parser, buffer, ");"); pkey = sl_print_pk (parser, inst_tp->class_, key); if (pkey == NULL) { parser_free_parser (parser); return ER_FAILED; } select = sl_print_select (parser, sm_ch_name ((MOBJ) (inst_tp->class_)), pkey); if (sl_write_sql (buffer, select) != NO_ERROR) { parser_free_parser (parser); return ER_FAILED; } parser_free_parser (parser); return NO_ERROR; }
TEST(parser_interface_unittest, test_speed) { void * engine = parser_create_parser("./ltp_data/parser.model"); EXPECT_TRUE(NULL != engine); std::ifstream ifs("./test_data/unittest/test_data.postaggered"); std::string line; std::string word; std::vector<std::string> words; std::vector<std::string> tags; std::vector<int> heads; std::vector<std::string> deprels; int nr_tokens = 0; long start_time = clock(); while (std::getline(ifs, line, '\n')) { std::stringstream S(line); words.clear(); tags.clear(); heads.clear(); deprels.clear(); while (S >> word) { std::vector<std::string> sep = ltp::strutils::rsplit_by_sep(word, "_", 1); if(sep.size()==2) { words.push_back(sep[0]); tags.push_back(sep[1]); } else { std::cerr << word<<std::endl; return; } } parser_parse(engine, words, tags, heads, deprels); nr_tokens += words.size(); } double throughput_per_millisecond = (nr_tokens / ((clock() -start_time) / 1000.)); std::cerr << throughput_per_millisecond << std::endl; parser_release_parser(engine); }
TEST(parser_interface_unittest, test_load_model_fail) { void * engine = parser_create_parser("/a/path/that/never/exist"); EXPECT_EQ(NULL, engine); }
TEST(parser_interface_unittest, test_load_model_success) { void * engine = parser_create_parser("./ltp_data/parser.model"); EXPECT_TRUE(NULL != engine); parser_release_parser(engine); }
int sl_write_statement_sql (char *class_name, char *db_user, int item_type, char *stmt_text, char *ha_sys_prm) { int rc, error = NO_ERROR; PARSER_CONTEXT *parser; PARSER_VARCHAR *buffer = NULL, *grant = NULL; PARSER_VARCHAR *set_param = NULL, *restore_param = NULL; char default_ha_prm[LINE_MAX]; parser = parser_create_parser (); buffer = pt_append_nulstring (parser, buffer, stmt_text); buffer = pt_append_nulstring (parser, buffer, ";"); if (ha_sys_prm != NULL) { set_param = pt_append_nulstring (parser, set_param, CA_MARK_TRAN_START); set_param = pt_append_nulstring (parser, set_param, " SET SYSTEM PARAMETERS '"); set_param = pt_append_nulstring (parser, set_param, ha_sys_prm); set_param = pt_append_nulstring (parser, set_param, "';"); rc = sysprm_make_default_values (ha_sys_prm, default_ha_prm, sizeof (default_ha_prm)); if (rc != PRM_ERR_NO_ERROR) { error = sysprm_set_error (rc, ha_sys_prm); goto end; } restore_param = pt_append_nulstring (parser, restore_param, CA_MARK_TRAN_END); restore_param = pt_append_nulstring (parser, restore_param, " SET SYSTEM PARAMETERS '"); restore_param = pt_append_nulstring (parser, restore_param, default_ha_prm); restore_param = pt_append_nulstring (parser, restore_param, "';"); if (sl_write_sql (set_param, NULL) != NO_ERROR) { error = ER_FAILED; goto end; } if (sl_write_sql (buffer, NULL) != NO_ERROR) { sl_write_sql (restore_param, NULL); error = ER_FAILED; goto end; } if (sl_write_sql (restore_param, NULL) != NO_ERROR) { error = ER_FAILED; goto end; } } else { if (sl_write_sql (buffer, NULL) != NO_ERROR) { error = ER_FAILED; goto end; } } if (item_type == CUBRID_STMT_CREATE_CLASS) { if (db_user != NULL && strlen (db_user) > 0) { grant = pt_append_nulstring (parser, grant, "GRANT ALL PRIVILEGES ON "); grant = pt_append_nulstring (parser, grant, class_name); grant = pt_append_nulstring (parser, grant, " TO "); grant = pt_append_nulstring (parser, grant, db_user); grant = pt_append_nulstring (parser, grant, ";"); if (sl_write_sql (grant, NULL) != NO_ERROR) { error = ER_FAILED; goto end; } } } end: parser_free_parser (parser); return error; }
int sl_write_update_sql (DB_OTMPL * inst_tp, DB_VALUE * key) { PARSER_CONTEXT *parser; PARSER_VARCHAR *buffer = NULL, *select = NULL; PARSER_VARCHAR *att_set, *pkey; PARSER_VARCHAR *serial_name, *serial_value; DB_VALUE *cur_value, *incr_value; DB_VALUE next_value; char str_next_value[NUMERIC_MAX_STRING_SIZE]; bool is_db_serial = false; int result; parser = parser_create_parser (); if (strcmp (sm_ch_name ((MOBJ) (inst_tp->class_)), "db_serial") != 0) { /* ordinary tables */ att_set = sl_print_update_att_set (parser, inst_tp->assignments, inst_tp->nassigns); pkey = sl_print_pk (parser, inst_tp->class_, key); if (pkey == NULL) { result = ER_FAILED; goto end; } buffer = pt_append_nulstring (parser, buffer, "UPDATE ["); buffer = pt_append_nulstring (parser, buffer, sm_ch_name ((MOBJ) (inst_tp->class_))); buffer = pt_append_nulstring (parser, buffer, "] SET "); buffer = pt_append_varchar (parser, buffer, att_set); buffer = pt_append_nulstring (parser, buffer, " WHERE "); buffer = pt_append_varchar (parser, buffer, pkey); buffer = pt_append_nulstring (parser, buffer, ";"); select = sl_print_select (parser, sm_ch_name ((MOBJ) (inst_tp->class_)), pkey); result = sl_write_sql (buffer, select); } else { /* db_serial */ serial_name = sl_print_att_value (parser, "name", inst_tp->assignments, inst_tp->nassigns); trim_single_quote (serial_name); cur_value = sl_find_att_value (parser, "current_val", inst_tp->assignments, inst_tp->nassigns); incr_value = sl_find_att_value (parser, "increment_val", inst_tp->assignments, inst_tp->nassigns); if (cur_value == NULL || incr_value == NULL) { result = ER_FAILED; goto end; } result = numeric_db_value_add (cur_value, incr_value, &next_value); if (result != NO_ERROR) { goto end; } numeric_db_value_print (&next_value, str_next_value); buffer = pt_append_nulstring (parser, buffer, "ALTER SERIAL ["); buffer = pt_append_varchar (parser, buffer, serial_name); buffer = pt_append_nulstring (parser, buffer, "] START WITH "); buffer = pt_append_nulstring (parser, buffer, str_next_value); buffer = pt_append_nulstring (parser, buffer, ";"); result = sl_write_sql (buffer, NULL); } end: parser_free_parser (parser); if (result != NO_ERROR) { return ER_FAILED; } return NO_ERROR; }