int
sl_write_delete_sql (char *class_name, MOBJ mclass, DB_VALUE * key)
{
PARSER_CONTEXT *parser;
PARSER_VARCHAR *buffer = NULL, *pkey;
PARSER_VARCHAR *select;
parser = parser_create_parser ();
pkey = sl_print_pk (parser, (SM_CLASS *) mclass, key);
if (pkey == NULL)
{
parser_free_parser (parser);
return ER_FAILED;
}
buffer = pt_append_nulstring (parser, buffer, "DELETE FROM [");
buffer = pt_append_nulstring (parser, buffer, class_name);
buffer = pt_append_nulstring (parser, buffer, "] WHERE ");
buffer = pt_append_varchar (parser, buffer, pkey);
buffer = pt_append_nulstring (parser, buffer, ";");
select = sl_print_select (parser, class_name, pkey);
if (sl_write_sql (buffer, select) != NO_ERROR)
{
parser_free_parser (parser);
return ER_FAILED;
}
parser_free_parser (parser);
return NO_ERROR;
}
void load(const std::string& model_path) {
if (model == NULL) {
model = parser_create_parser(model_path.c_str());
} else {
std::cerr << "Parser: Model reloaded!" << std::endl;
}
}
JNIEXPORT jint JNICALL Java_edu_hit_ir_ltp4j_Parser_create
(JNIEnv * env, jclass obj, jstring model_path){
const char * str = env->GetStringUTFChars( model_path , 0);
if(!parser){
parser = parser_create_parser(str);
}
env->ReleaseStringUTFChars( model_path, str);
if(parser) {
return 1;
}
return -1;
}
TEST(parser_interface_unittest, test_empty_list) {
void * engine = parser_create_parser("./ltp_data/parser.model");
EXPECT_TRUE(NULL != engine);
std::vector<std::string> words;
std::vector<std::string> tags;
std::vector<int> heads;
std::vector<std::string> deprels;
int nr_words = parser_parse(engine, words, tags ,heads ,deprels);
EXPECT_EQ(0, nr_words);
parser_release_parser(engine);
}
TEST(parser_interface_unittest, test_normal) {
void * engine = parser_create_parser("./ltp_data/parser.model");
EXPECT_TRUE(NULL != engine);
std::vector<std::string> words;
std::vector<std::string> tags;
std::vector<int> heads;
std::vector<std::string> deprels;
for (int i = 0; i < kNumNormalWords; ++ i) {
words.push_back(kNormalWords[i]);
tags.push_back(kNormalPostags[i]);
}
int nr_words = parser_parse(engine, words, tags ,heads ,deprels);
// tagged words should be greater than 4
EXPECT_GT(nr_words, 0);
parser_release_parser(engine);
}
/* ====================================================== *
* Parser related resource *
* ====================================================== */
int LTPResource::LoadParserResource(const char * model_file) {
if (m_isParserResourceLoaded) {
return 0;
}
TRACE_LOG("Loading parser resource from \"%s\"", model_file);
m_parser = parser_create_parser(model_file);
if (!m_parser) {
ERROR_LOG("Failed to create parser");
return -1;
}
TRACE_LOG("Parser is loaded.");
m_isParserResourceLoaded = true;
return 0;
}
TEST(parser_interface_unittest, test_empty_tag) {
void * engine = parser_create_parser("./ltp_data/parser.model");
EXPECT_TRUE(NULL != engine);
std::vector<std::string> words;
std::vector<std::string> tags;
std::vector<int> heads;
std::vector<std::string> deprels;
for (int i = 0; i < kNumNormalWords; ++ i) {
if (i == 2) {
tags.push_back("");
} else {
tags.push_back(kNormalPostags[i]);
}
words.push_back(kNormalWords[i]);
}
int nr_words = parser_parse(engine, words, tags, heads, deprels);
EXPECT_EQ(0, nr_words);
parser_release_parser(engine);
}
int
sl_write_insert_sql (DB_OTMPL * inst_tp, DB_VALUE * key)
{
PARSER_CONTEXT *parser;
PARSER_VARCHAR *buffer = NULL;
PARSER_VARCHAR *att_names, *att_values;
PARSER_VARCHAR *pkey;
PARSER_VARCHAR *select;
parser = parser_create_parser ();
att_names = sl_print_insert_att_names (parser, inst_tp->assignments, inst_tp->nassigns);
att_values = sl_print_insert_att_values (parser, inst_tp->assignments, inst_tp->nassigns);
buffer = pt_append_nulstring (parser, buffer, "INSERT INTO [");
buffer = pt_append_nulstring (parser, buffer, sm_ch_name ((MOBJ) (inst_tp->class_)));
buffer = pt_append_nulstring (parser, buffer, "](");
buffer = pt_append_varchar (parser, buffer, att_names);
buffer = pt_append_nulstring (parser, buffer, ") VALUES (");
buffer = pt_append_varchar (parser, buffer, att_values);
buffer = pt_append_nulstring (parser, buffer, ");");
pkey = sl_print_pk (parser, inst_tp->class_, key);
if (pkey == NULL)
{
parser_free_parser (parser);
return ER_FAILED;
}
select = sl_print_select (parser, sm_ch_name ((MOBJ) (inst_tp->class_)), pkey);
if (sl_write_sql (buffer, select) != NO_ERROR)
{
parser_free_parser (parser);
return ER_FAILED;
}
parser_free_parser (parser);
return NO_ERROR;
}
TEST(parser_interface_unittest, test_speed) {
void * engine = parser_create_parser("./ltp_data/parser.model");
EXPECT_TRUE(NULL != engine);
std::ifstream ifs("./test_data/unittest/test_data.postaggered");
std::string line;
std::string word;
std::vector<std::string> words;
std::vector<std::string> tags;
std::vector<int> heads;
std::vector<std::string> deprels;
int nr_tokens = 0;
long start_time = clock();
while (std::getline(ifs, line, '\n')) {
std::stringstream S(line);
words.clear();
tags.clear();
heads.clear();
deprels.clear();
while (S >> word) {
std::vector<std::string> sep = ltp::strutils::rsplit_by_sep(word, "_", 1);
if(sep.size()==2) {
words.push_back(sep[0]);
tags.push_back(sep[1]);
} else {
std::cerr << word<<std::endl;
return;
}
}
parser_parse(engine, words, tags, heads, deprels);
nr_tokens += words.size();
}
double throughput_per_millisecond = (nr_tokens / ((clock() -start_time) / 1000.));
std::cerr << throughput_per_millisecond << std::endl;
parser_release_parser(engine);
}
TEST(parser_interface_unittest, test_load_model_fail) {
void * engine = parser_create_parser("/a/path/that/never/exist");
EXPECT_EQ(NULL, engine);
}
TEST(parser_interface_unittest, test_load_model_success) {
void * engine = parser_create_parser("./ltp_data/parser.model");
EXPECT_TRUE(NULL != engine);
parser_release_parser(engine);
}
int
sl_write_statement_sql (char *class_name, char *db_user, int item_type, char *stmt_text, char *ha_sys_prm)
{
int rc, error = NO_ERROR;
PARSER_CONTEXT *parser;
PARSER_VARCHAR *buffer = NULL, *grant = NULL;
PARSER_VARCHAR *set_param = NULL, *restore_param = NULL;
char default_ha_prm[LINE_MAX];
parser = parser_create_parser ();
buffer = pt_append_nulstring (parser, buffer, stmt_text);
buffer = pt_append_nulstring (parser, buffer, ";");
if (ha_sys_prm != NULL)
{
set_param = pt_append_nulstring (parser, set_param, CA_MARK_TRAN_START);
set_param = pt_append_nulstring (parser, set_param, " SET SYSTEM PARAMETERS '");
set_param = pt_append_nulstring (parser, set_param, ha_sys_prm);
set_param = pt_append_nulstring (parser, set_param, "';");
rc = sysprm_make_default_values (ha_sys_prm, default_ha_prm, sizeof (default_ha_prm));
if (rc != PRM_ERR_NO_ERROR)
{
error = sysprm_set_error (rc, ha_sys_prm);
goto end;
}
restore_param = pt_append_nulstring (parser, restore_param, CA_MARK_TRAN_END);
restore_param = pt_append_nulstring (parser, restore_param, " SET SYSTEM PARAMETERS '");
restore_param = pt_append_nulstring (parser, restore_param, default_ha_prm);
restore_param = pt_append_nulstring (parser, restore_param, "';");
if (sl_write_sql (set_param, NULL) != NO_ERROR)
{
error = ER_FAILED;
goto end;
}
if (sl_write_sql (buffer, NULL) != NO_ERROR)
{
sl_write_sql (restore_param, NULL);
error = ER_FAILED;
goto end;
}
if (sl_write_sql (restore_param, NULL) != NO_ERROR)
{
error = ER_FAILED;
goto end;
}
}
else
{
if (sl_write_sql (buffer, NULL) != NO_ERROR)
{
error = ER_FAILED;
goto end;
}
}
if (item_type == CUBRID_STMT_CREATE_CLASS)
{
if (db_user != NULL && strlen (db_user) > 0)
{
grant = pt_append_nulstring (parser, grant, "GRANT ALL PRIVILEGES ON ");
grant = pt_append_nulstring (parser, grant, class_name);
grant = pt_append_nulstring (parser, grant, " TO ");
grant = pt_append_nulstring (parser, grant, db_user);
grant = pt_append_nulstring (parser, grant, ";");
if (sl_write_sql (grant, NULL) != NO_ERROR)
{
error = ER_FAILED;
goto end;
}
}
}
end:
parser_free_parser (parser);
return error;
}
int
sl_write_update_sql (DB_OTMPL * inst_tp, DB_VALUE * key)
{
PARSER_CONTEXT *parser;
PARSER_VARCHAR *buffer = NULL, *select = NULL;
PARSER_VARCHAR *att_set, *pkey;
PARSER_VARCHAR *serial_name, *serial_value;
DB_VALUE *cur_value, *incr_value;
DB_VALUE next_value;
char str_next_value[NUMERIC_MAX_STRING_SIZE];
bool is_db_serial = false;
int result;
parser = parser_create_parser ();
if (strcmp (sm_ch_name ((MOBJ) (inst_tp->class_)), "db_serial") != 0)
{
/* ordinary tables */
att_set = sl_print_update_att_set (parser, inst_tp->assignments, inst_tp->nassigns);
pkey = sl_print_pk (parser, inst_tp->class_, key);
if (pkey == NULL)
{
result = ER_FAILED;
goto end;
}
buffer = pt_append_nulstring (parser, buffer, "UPDATE [");
buffer = pt_append_nulstring (parser, buffer, sm_ch_name ((MOBJ) (inst_tp->class_)));
buffer = pt_append_nulstring (parser, buffer, "] SET ");
buffer = pt_append_varchar (parser, buffer, att_set);
buffer = pt_append_nulstring (parser, buffer, " WHERE ");
buffer = pt_append_varchar (parser, buffer, pkey);
buffer = pt_append_nulstring (parser, buffer, ";");
select = sl_print_select (parser, sm_ch_name ((MOBJ) (inst_tp->class_)), pkey);
result = sl_write_sql (buffer, select);
}
else
{
/* db_serial */
serial_name = sl_print_att_value (parser, "name", inst_tp->assignments, inst_tp->nassigns);
trim_single_quote (serial_name);
cur_value = sl_find_att_value (parser, "current_val", inst_tp->assignments, inst_tp->nassigns);
incr_value = sl_find_att_value (parser, "increment_val", inst_tp->assignments, inst_tp->nassigns);
if (cur_value == NULL || incr_value == NULL)
{
result = ER_FAILED;
goto end;
}
result = numeric_db_value_add (cur_value, incr_value, &next_value);
if (result != NO_ERROR)
{
goto end;
}
numeric_db_value_print (&next_value, str_next_value);
buffer = pt_append_nulstring (parser, buffer, "ALTER SERIAL [");
buffer = pt_append_varchar (parser, buffer, serial_name);
buffer = pt_append_nulstring (parser, buffer, "] START WITH ");
buffer = pt_append_nulstring (parser, buffer, str_next_value);
buffer = pt_append_nulstring (parser, buffer, ";");
result = sl_write_sql (buffer, NULL);
}
end:
parser_free_parser (parser);
if (result != NO_ERROR)
{
return ER_FAILED;
}
return NO_ERROR;
}