int ul_exchangedir(const char *path, const char *sname, const char *dname)
{
char fsname[PATH_SIZE], fdname[PATH_SIZE], ftname[PATH_SIZE];
if (ul_fullpath(path, sname, fsname, sizeof(fsname)) < 0) {
return -1;
}
if (ul_fullpath(path, dname, fdname, sizeof(fdname)) < 0) {
return -1;
}
if (ul_fullpath(path, "tmp.xxx", ftname, sizeof(ftname)) < 0) {
return -1;
}
if (rename(fdname, ftname) < 0) {
ul_writelog(UL_LOG_FATAL, "[exchangedir] rename(%s) to (%s) error", fdname, ftname);
return -1;
}
if (rename(fsname, fdname) < 0) {
ul_writelog(UL_LOG_FATAL, "[exchangedir] rename(%s) to (%s) error", fsname, fdname);
return -1;
}
if (rename(ftname, fsname) < 0) {
ul_writelog(UL_LOG_FATAL, "[exchangedir] rename(%s) to (%s) error", ftname, fsname);
return -1;
}
return 1;
}
int
talkwith (int &fd, nshead_t * req, nshead_t * res, int siz)
{
int timeout = 1000000;
int ret = 0;
if (fd < 0)
{
fd = MyConnectO (opt_data.ip, opt_data.port, &timeout, "tinyse-qtool");
if (fd <= 0)
{
ul_writelog (UL_LOG_WARNING, "connect err[%m] ret[%d]", fd);
return -1;
}
int on = 1;
setsockopt (fd, IPPROTO_TCP, TCP_NODELAY, &on, sizeof (on));
struct linger li;
memset (&li, 0, sizeof (li));
li.l_onoff = 1;
li.l_linger = 0;
setsockopt (fd, SOL_SOCKET, SO_LINGER, (const char *) &li,
sizeof (li));
}
req->log_id = rand ();
ret = nshead_write (fd, req, 1000);
if (ret != 0)
{
ul_writelog (UL_LOG_WARNING, "write err[%m], ret[%d]", ret);
goto fail;
}
ret = nshead_read (fd, res, siz, 10000);
if (ret != 0)
{
ul_writelog (UL_LOG_WARNING, "read err[%m], ret[%d]", ret);
goto fail;
}
{//短连接
close (fd);
fd = -1;
}
return 0;
fail:
if (fd > 0)
{
close (fd);
}
fd = -1;
return -1;
}
int globalInit()
{
g_pConf = &gConf;
if (0 != loadConfig(CONF_PATH, CONF_FILENAME)) {
ul_writelog(UL_LOG_FATAL,
"Can not load config path=%s,fileName=%s",
CONF_PATH, CONF_FILENAME);
return -1;
}
logstat.events = g_pConf->iLogLevel;
// 打开日志文件，打开之后正常日志默认写入diffPro.log文件中，错误日志写入diffPro.log.wf
if (0 != ul_openlog(g_pConf->pLogPath, g_pConf->pLogFilename, &logstat, 1024)) {
fprintf(stderr, "Fail to open log,path=%s,filename=%s",
g_pConf->pLogPath, g_pConf->pLogFilename);
return -1;
}
// 根据词表文件大小获取：开辟进程数和split词表文件时的时间戳。
if (getExtraArgs() != 0) {
return -1;
}
return 0;
}
int ul_appfile(const char *path, const char *fsrc, const char *fdest)
{
int pf_src, pf_dest;
char filename[PATH_SIZE];
char buf[TBUF_SIZE];
ssize_t real_read;
if (ul_fullpath(path, fsrc, filename, sizeof(filename)) < 0) {
return -1;
}
if (!ul_fexist(path, fsrc)) {
ul_writelog(UL_LOG_FATAL, "[appfile] file : %s do not exist.\n", fsrc);
return -1;
}
pf_src = ul_open(filename, O_RDONLY,0776);
if (pf_src < 0) {
ul_writelog(UL_LOG_FATAL,"[appfile] open %s error", fsrc);
return -1;
}
if (ul_fullpath(path, fdest, filename, sizeof(filename)) < 0) {
return -1;
}
pf_dest= ul_open(filename, O_WRONLY|O_APPEND|O_CREAT, 0666);
if (pf_dest < 0) {
ul_close(pf_src);
ul_writelog(UL_LOG_FATAL, "[appfile] open %s error", fdest);
return -1;
}
while((real_read=ul_read(pf_src, buf, TBUF_SIZE)) == TBUF_SIZE) {
ul_write(pf_dest, buf, TBUF_SIZE);
}
ul_write(pf_dest, buf, (size_t)real_read);
ul_close(pf_src);
ul_close(pf_dest);
return 1;
}
int ul_mvfile(const char *path, const char *fsrc, const char *fdest)
{
int reti;
char ffsrc[PATH_SIZE], ffdest[PATH_SIZE];
if (ul_fullpath(path, fsrc, ffsrc, sizeof(ffsrc)) < 0) {
return -1;
}
if (ul_fullpath(path, fdest, ffdest, sizeof(ffdest)) < 0) {
return -1;
}
reti = rename(ffsrc, ffdest);
if (reti < 0) {
ul_writelog(UL_LOG_FATAL, "[mvfile] rename (%s) to (%s) error", ffsrc, ffdest);
ul_cpfile(path, fsrc, fdest);
reti = unlink(ffsrc);
if (reti < 0) {
ul_writelog(UL_LOG_FATAL, "[mvfile] unlink file (%s) error", ffsrc);
return -1;
}
}
return reti;
}
ssize_t ul_writefile(const char *path, const char *fname, void *buf, int size)
{
int fno;
ssize_t rsize;
char fullname[PATH_SIZE];
if (ul_fullpath(path, fname, fullname, sizeof(fullname)) < 0) {
return -1;
}
fno = ul_open(fullname, O_WRONLY|O_CREAT|O_TRUNC, 0666);
fchmod (fno, 0666);
if (fno < 0) {
ul_writelog(UL_LOG_FATAL, "[writefile] open (%s) error", fullname);
return -1;
}
rsize = ul_write(fno, buf, size);
if (rsize < 0){
ul_writelog(UL_LOG_FATAL, "[writefile] write(%s) error", fullname);
}
ul_close(fno);
return rsize;
}
ssize_t ul_readfile(const char *path, const char *fname, void *buf, int size)
{
int fno ;
ssize_t rsize;
char fullname[PATH_SIZE];
if (ul_fullpath(path, fname, fullname, sizeof(fullname)) < 0) {
return -1;
}
fno = ul_open(fullname, O_RDONLY,0776);
if (fno < 0) {
ul_writelog(UL_LOG_FATAL, "[readfile] open (%s) error", fullname);
return -1;
}
rsize = ul_read(fno, buf, size);
if (rsize < 0) {
ul_writelog(UL_LOG_FATAL, "[readfile] read (%s) error", fullname);
}
ul_close(fno);
return rsize;
}
int main(){
int i;
assert(com_logstatus() == LOG_NOT_DEFINED);
com_loadlog("./", "test.conf");
for(int j = 0; j < 1000; j++){
ul_writelog(COMLOG_WARNING, "1 - UL [%d/%d]", i, j);
com_writelog(UL_LOG_FATAL, "1 - COM [%d/%d]", i, j);
sleep(1);
}
assert(com_logstatus() == USING_COMLOG);
com_closelog(0);
assert(com_logstatus() == LOG_NOT_DEFINED);
return 0;
}
int ul_nullfile(const char *path, const char *fname)
{
char full_path[PATH_SIZE];
FILE *pf;
if (ul_fullpath(path, fname, full_path, sizeof(full_path)) < 0) {
return -1;
}
pf = ul_fopen(full_path, "w");
if (NULL == pf){
ul_writelog(UL_LOG_FATAL, "[nullfile] open file (%s) error", full_path);
return -1;
}
ul_fclose(pf);
return 1;
}
int splitDictFile(int iPerSize)
{
char pStrCmd[MAX_NAME];
//按10M左右大小split，并且最多split10个文件。词表名：./g_pConf->pDictPath/g_pArgs->pDictName
sprintf(pStrCmd, "split -C %d -d -a 1 %s%s dict_", iPerSize, g_pConf->pDictPath, g_pArgs->pDictName);
if (System(pStrCmd) != 0) {
ul_writelog(UL_LOG_FATAL, "splitDictFile error!");
return -1;
}
// split后，将生成的词表文件mv到对应文件夹中 ./dict/timestamp
sprintf(pStrCmd, "mv dict_* %s%s", g_pConf->pDictPath, g_pArgs->pStrTimestamp);
if (System(pStrCmd) != 0) {
return -1;
}
// TODO:是否同时将最初的词表文件删除？
return 0;
}
//限制缓冲区长度版本
int ul_pack_getvalue_ex(ul_package_t *ppack, const char *pname, char *pvalue, size_t size)
{
int i;
if (NULL == ppack || NULL == pname || NULL == pvalue) {
return 0;
}
pvalue[0] = '\0';
for (i = 0; i < ppack->pairnum; i++) {
if (strcmp(ppack->name[i], pname) == 0) {
if (snprintf(pvalue, size, "%s", ppack->value[i]) > (ssize_t)size) {
ul_writelog(UL_LOG_WARNING, "%s ppack item length too long [size:%zu]", __func__, size);
}
return 1;
}
}
return 0;
}
int main(int argc, char *argv[])
{
// 读取传参，顺序如下：线上环境、线下环境、词表
if (argc != 4) {
printf("Usage: %s onlineEnv offlineEnv dictFilename\n", argv[0]);
exit(-1);
} else {
g_pArgs = &gArgs;
// strcpy(g_pArgs->pOnlineEnv, argv[1]);
// strcpy(g_pArgs->pOfflineEnv, argv[2]);
strcpy(g_pArgs->pOnlineEnv, "http://yf-map-mirror-wpng.vm.baidu.com:8000");
strcpy(g_pArgs->pOfflineEnv, "http://cp01-centos43-testing005.epc.baidu.com:8000");
strcpy(g_pArgs->pDictName, argv[3]);
}
// 读取配置文件初始化词表目录、数据结果目录
if (globalInit() != 0) {
ul_writelog(UL_LOG_FATAL, "globalInit fail, exit!!!");
exit(-1);
}
// strcpy(g_pArgs->pStrTimestamp, "201508052127");
// filterFields();
time_t start, end;
start = time(NULL);
// 根据词表文件多线程发送http请求
sendReqs();
end = time(NULL);
printf("Total cost is : %f\n", difftime(end, start));
// 生成diff结果后，开始对结果进行处理
// TODO
dealResultFiles();
return 0;
}
int main()
{
spreg_match_t *rmatch = spreg_match_init(MAX_SUB_MATCH);
if (NULL == rmatch) {
ul_writelog(UL_LOG_FATAL, "no memory for spreg_match_t");
return -1;
}
char regex_string[] = "[0-9]+([A-Z]+)";
char match_string[] = "M88734AADFBBahjdjd18734XADFBB";
int match_string_size = strlen(match_string);
char replace_string[] = "OK\\1";
char buffer[1024];
int buffer_size = sizeof(buffer);
char const *err = NULL;
int ret;
spreg_t *re = NULL;
//初始化
re = spreg_init(regex_string, &err);
if (err != NULL) {
//初始化失败
ul_writelog(UL_LOG_WARNING, "spreg_init error [%s]", err);
} else {
//不需要匹配结果, 只 需要判断match
ret = spreg_match(re, match_string, match_string_size);
if (SPREG_NOMATCH == ret) {
//SPREG_NOMATCH 小于0不能使用ret < 0 进行判断异常
ul_writelog(UL_LOG_TRACE, "spreg_match NO MATCH!");
} else if (ret < 0) {
ul_writelog(UL_LOG_WARNING, "spreg_match error [%d]", ret);
} else {
ul_writelog(UL_LOG_TRACE, "spreg_match MATCH");
}
//spreg_search的使用
ret = spreg_search(re, match_string, match_string_size, rmatch, MAX_SUB_MATCH);
if (ret < 0) {
ul_writelog(UL_LOG_WARNING, "spreg_search error [%d]", ret);
} else {
fprintf(stderr, "\nspreg_search result\n");
for (int i = 0; i < ret; ++i) {
fprintf(stderr, "%d : [%d %d] [%.*s]\n", i, rmatch[i].match_begin, rmatch[i].match_end,
rmatch[i].match_end - rmatch[i].match_begin,
match_string + rmatch[i].match_begin);
}
fprintf(stderr, "\n");
}
//spreg_search_all使用
ret = spreg_search_all(re, match_string, match_string_size, search_all_callback,
match_string);
if (ret < 0) {
ul_writelog(UL_LOG_WARNING, "spreg_search_all error [%d]", ret);
} else {
fprintf(stderr, "search %d match\n\n", ret);
}
ret = spreg_split(re, match_string, match_string_size, split_callback, match_string);
if (ret < 0) {
ul_writelog(UL_LOG_WARNING, "spreg_split error [%d]", ret);
} else {
fprintf(stderr, "split %d \n\n", ret);
}
ret = spreg_replace(re, match_string, match_string_size,
replace_string, buffer, buffer_size, 1);
if (ret < 0) {
ul_writelog(UL_LOG_WARNING, "spreg_replace error [%d]", ret);
} else {
//buffer不是以'\0'结尾的这里补上'\0';
buffer[ret] = '\0';
fprintf(stderr, "result string : %s\n", buffer);
}
spreg_destroy(re);
spreg_match_destroy(rmatch);
}
return 0;
}
int creat_sign_t256(char *psrc, int slen, unsigned int* psign1, unsigned int* psign2)
{
unsigned char *psrc0;
unsigned char psrc1[256];
unsigned char prearr1[256];
unsigned char psrc2[256];
unsigned char prearr2[256];
unsigned int sign0, sign1, sign2;
int i, j1, j2;
/* init vars */
psrc0 = (unsigned char *)psrc;
*psign1 = 0;
*psign2 = 0;
/* check the string length */
if ((slen >256) || (slen<0)){
/* comment for using new lib
sprintf(inf_buf, "string too long (%d>256)\n", slen);
writelog(pf_log, inf_buf, IPROG);
*/
ul_writelog(UL_LOG_WARNING,"string too long (%d > 256 )\n",slen);
*psign1=0xffffffff;
*psign2=0xffffffff;
return -1;
}
if (slen == 0){
return 0;
}
/* sign the short string */
if (slen <= 4){
memcpy(psign1, psrc, slen);
return 1;
}
if ((slen > 4) && (slen <= 8)){
memcpy(psign1, psrc, 4);
memcpy(psign2, psrc+4, slen-4);
return 1;
}
/* get the rearranged string */
j1=0;
j2=0;
for (i=0; i<256; i++){
if (rearr256_1[i] < slen){
prearr1[j1++]=rearr256_1[i];
}
if (rearr256_2[i] < slen){
prearr2[j2++]=rearr256_2[i];
}
}
for (i=0; i<slen; i++){
psrc1[i]= psrc0[prearr1[i]];
psrc2[i]= psrc0[prearr2[i]];
}
/* get source string's sign */
sign0 = 0;
for (i=0; i<slen; i++){
sign0 += ( (unsigned char)psrc0[i] * remain256_0[i] )%MAX_PRIME_0;
if (sign0 >= MAX_PRIME_0){
sign0 -= MAX_PRIME_0;
}
}
/* get new 1 string's sign */
sign1 = 0;
for (i=0; i<slen; i++){
sign1 += ( (unsigned char)psrc1[i] * remain256_1[i] )%MAX_PRIME_1;
if (sign1 >= MAX_PRIME_1){
sign1 -= MAX_PRIME_1;
}
}
/* get new 2 string's sign */
sign2 = 0;
for (i=0; i<slen; i++){
sign2 += ( (unsigned char)psrc2[i] * remain256_2[i] )%MAX_PRIME_2;
if (sign2 >= MAX_PRIME_2){
sign2 -= MAX_PRIME_2;
}
}
/* get result */
*psign1=sign0 ^ sign2<<20;
*psign2=sign1 ^ (sign2<<8 & 0xfff00000);
/* debug */
#ifdef __DEBUG
printf("trans1 :\n");
for (i=0; i<slen; i++){
printf("%d", prearr1[i]);
}
printf("\n");
printf("newstr1 :\n");
for (i=0; i<slen; i++){
printf("%c", psrc1[i]);
}
printf("\n");
printf("trans2 :\n");
for (i=0; i<slen; i++){
printf("%d", prearr2[i]);
}
printf("\n");
printf("newstr2 :\n");
for (i=0; i<slen; i++){
printf("%c", psrc2[i]);
}
printf("\n");
printf("source str : %s\n", psrc);
printf("sign0 : %9u sign1 %9u sign2 %9u\n", sign0, sign1, sign2);
printf("psign1 : %9u psign2 %9u\n", *psign1, *psign2);
#endif
/* return */
return 1;
}
int ul_pack_reado_ms(int sockfd, ul_package_t *ppack, int msec)
{
int size;
int headsize = -1;
int bodysize = -1;
int i = 0;
char sizebuf[SIZE_NUM];
char headbuf[MAX_HEAD_SIZE];
char *ph = headbuf;
char *q;
char tmp[10], eofstr[BUF_EOF_SIZE+1];
if (NULL == ppack) {
return -2;
}
memset(sizebuf, 0, sizeof(sizebuf));
size = ul_pack_readallo_ms(sockfd, sizebuf, sizeof(sizebuf), msec);
if (-1 == size) {
ul_writelog(UL_LOG_WARNING, "ul_pack_reado: overtime or socket error");
return -2;
} else if(size != SIZE_NUM || strncmp(sizebuf, BUF_START, BUF_START_SIZE) != 0) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: size number error.[%d:%d || %s].", size, SIZE_NUM, sizebuf);
if(ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
sscanf(sizebuf, "%s%d%d", tmp, &headsize, &bodysize);
if (headsize < 0 || bodysize < 0) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: size are negative [%d : %d]",
headsize, bodysize);
if(ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
if (bodysize > ppack->bufsize || headsize >= MAX_HEAD_SIZE) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: buffer size not enough [%d : %d]",
ppack->bufsize, bodysize);
if (ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
size = ul_pack_readallo_ms(sockfd, headbuf, (size_t)headsize, msec);
if (-1 == size) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: socket error");
return -2;
}else if(size != headsize) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: read head error.size[%d] not equal bodysize[%d]",
size, headsize);
if(ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
headbuf[headsize] = '\0';
size = ul_pack_readallo_ms(sockfd, ppack->buf, (size_t)bodysize, msec);
if (-1 == size) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: socket error");
return -2;
}
if (size != bodysize) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: read body error");
if (ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
size = ul_pack_readallo_ms(sockfd, eofstr, BUF_EOF_SIZE, msec);
eofstr[BUF_EOF_SIZE] = '\0';
if (-1 == size) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: socket error");
return -2;
}
if ((size_t)size != BUF_EOF_SIZE || strncmp(eofstr, BUF_EOF, BUF_EOF_SIZE) != 0 ) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: read EOF error,%d,%s", size, eofstr);
if(ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
while (*ph != '\0' && i < MAX_PACK_ITEM_NUM) {
q = strstr(ph, "\r\n");
if (NULL == q || q == ph)
break;
*q = '\0';
ppack->name[i][0] = '\0';
ppack->value[i][0] = '\0';
char *p ;
p = strstr(ph , " : ");
if (NULL == p || p-ph >= MAX_ITEM_SIZE || q-(p+3) >= MAX_ITEM_SIZE) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: head data error error [%s]", ph);
if(ul_pack_readendo_ms(sockfd, msec) < 0) {
return -2;
}
return -1;
}
snprintf(ppack->value[i], sizeof(ppack->value[i]), "%s", p+3);
sscanf(ph, "%s%s", ppack->name[i], tmp );
if (':' == tmp[0]) {
i++;
}
ph = q + 2;
}
int leftTag = 0;
if ((*ph != '\0') && (i == MAX_PACK_ITEM_NUM)) {
q = strstr(ph, "\r\n");
if (NULL == q || q == ph) {
leftTag = 0;
} else {
*q = '\0';
char *p;
p = strstr(ph, " : ");
if (NULL == p) {
leftTag = 0;
} else {
leftTag = 1;
}
}
}
if (1 == leftTag) {
ul_writelog(UL_LOG_WARNING, "ul_pack_read: left key-value pairs in head, curr recv[%d]", i);
}
ppack->pairnum = i;
return bodysize;
}
int ul_pack_writeo_ms(int sockfd, const ul_package_t *ppack, int msec)
{
char sizebuf[SIZE_NUM];
char headbuf[MAX_HEAD_SIZE];
int headsize = 0;
int bodysize = 0;
int size;
int i;
if (NULL == ppack) {
return -1;
}
headbuf[0] = '\0';
for (i = 0;i < ppack->pairnum; i++) {
if (ppack->name[i][0] != '\0') {
headsize += snprintf(headbuf + headsize, sizeof(headbuf) - headsize,
"%s : %s\r\n", ppack->name[i], ppack->value[i]);
}
if (headsize > MAX_HEAD_SIZE) {
ul_writelog(UL_LOG_WARNING, "ul_pack_writeo_ms: head size over default max head .%d.",
headsize);
return -2;
}
}
headsize += snprintf(headbuf + headsize, sizeof(headbuf) - headsize, "%s", "\r\n");
if (headsize > MAX_HEAD_SIZE) {
ul_writelog(UL_LOG_WARNING,"ul_pack_writeo_ms: head size over default max head size.%d.",
headsize);
return -2;
}
bodysize = ppack->bufsize;
size = snprintf(sizebuf, sizeof(sizebuf), "%s%d %d", BUF_START, headsize, bodysize);
if (size < SIZE_NUM - 1) {
sizebuf[size] = ' ';
for (i = size + 1; i < SIZE_NUM - 1; i++)
sizebuf[i] = '@';
sizebuf[SIZE_NUM-1] = '\0';
}
size = ul_pack_writeallo_ms(sockfd, sizebuf, SIZE_NUM, msec);
if (size != SIZE_NUM) {
ul_writelog(UL_LOG_WARNING,"ul_pack_writeo_ms: write size error %d:%d.", size, SIZE_NUM);
return -1;
}
size = ul_pack_writeallo_ms(sockfd, headbuf, (size_t)headsize, msec);
if (size != headsize) {
ul_writelog(UL_LOG_WARNING,"ul_pack_writeo_ms: write head error.size is %d:%d.", size, headsize);
return -1;
}
size = ul_pack_writeallo_ms(sockfd,ppack->buf, (size_t)bodysize, msec);
if (size != bodysize) {
ul_writelog(UL_LOG_WARNING,"ul_pack_writeo_ms: write body error.size is %d:%d.", size, bodysize);
return -1;
}
size = ul_pack_writeallo_ms(sockfd, BUF_EOF, BUF_EOF_SIZE, msec);
if ((size_t)size != BUF_EOF_SIZE) {
ul_writelog(UL_LOG_WARNING,"ul_pack_writeo_ms: write eof error.%d.",size);
return -1;
}
return 0;
}
int ul_pack_write_mo_ms(int sockfd, ul_package_t *ppack, int msec)
{
int headsize = 0;
int bodysize = 0;
int size;
int i;
char *tmpbuf = NULL;
int ret;
int pairlen;
size_t curpos = 0;
int maxLen;
if (NULL == ppack) {
return -1;
}
tmpbuf = (char*)malloc(SIZE_NUM+MAX_HEAD_SIZE + ppack->bufsize + 20UL);
if (NULL == tmpbuf) {
ul_writelog(UL_LOG_WARNING,
"[%s] malloc sendbuf(size=%d) for write ul_package_t failed."
"using ul_pack_writeo_ms",
__func__, ppack->bufsize);
return ul_pack_writeo_ms(sockfd,ppack, msec);
}
maxLen = SIZE_NUM + MAX_HEAD_SIZE + ppack->bufsize + 20UL;
curpos = SIZE_NUM;
for (i = 0;i < ppack->pairnum; i++) {
if (ppack->name[i][0] != '\0') {
pairlen = snprintf(tmpbuf+curpos, maxLen - curpos, "%s : %s\r\n",
ppack->name[i],ppack->value[i]);
if (pairlen >= (int)(maxLen-curpos)) {
ret = -1;
goto end;
}
/*
if (pairlen <= 0) {
ret = -1;
goto end;
}
*/
curpos += pairlen;
if ((curpos-SIZE_NUM) > MAX_HEAD_SIZE) {
ret =-1;
goto end;
}
}
}
curpos += snprintf(tmpbuf + curpos, maxLen - curpos, "%s", "\r\n");
headsize = (int)(curpos - SIZE_NUM);
if (headsize > MAX_HEAD_SIZE) {
ret = -1;
goto end;
}
bodysize = ppack->bufsize;
if ((size = snprintf(tmpbuf, SIZE_NUM, "%s%d %d", BUF_START, headsize, bodysize)) >= SIZE_NUM) {
ret = -1;
goto end;
}
if (size < SIZE_NUM - 1) {
tmpbuf[size] = ' ';
for (i = size + 1; i < SIZE_NUM - 1; i++)
tmpbuf[i] = '@';
tmpbuf[SIZE_NUM - 1] = '\0';
} else {
ret = -1;
goto end;
}
if (curpos > SIZE_NUM+MAX_HEAD_SIZE) {
ret = -1;
goto end;
}
if (bodysize != 0) {
if (NULL == ppack->buf) {
ret = -1;
goto end;
}
memcpy(tmpbuf + curpos, ppack->buf, (size_t)bodysize);
curpos += bodysize;
}
memcpy(tmpbuf + curpos, BUF_EOF, BUF_EOF_SIZE);
curpos += BUF_EOF_SIZE;
size = ul_pack_writeallo_ms(sockfd, tmpbuf, curpos, msec);
if ((size_t)size != curpos) {
ul_writelog(UL_LOG_WARNING,"%s: write size error %d:%zu.",
__func__, size, curpos);
ret = -1;
} else {
ret = 0;
}
end:
free(tmpbuf);
return ret;
}
int ul_pack_serialization(void *buf, int bufsize, ul_package_t *ppack)
{
int headsize = 0;
int bodysize = 0;
int size, i, pairlen, maxlen;
int ret = 0;
size_t curpos = 0;
char *tmpbuf;
if (buf == NULL || bufsize <= 0 || ppack == NULL) {
return -1;
}
maxlen = SIZE_NUM + MAX_HEAD_SIZE + ppack->bufsize + 20UL;
tmpbuf = (char *)malloc(maxlen);
if (tmpbuf == NULL) {
ul_writelog(UL_LOG_WARNING,
"[%s] malloc buf(size=%d) for serial ul_package_t failed.",
__func__, ppack->bufsize);
return -1;
}
curpos = SIZE_NUM;
for (i = 0; i < ppack->pairnum; i++) {
if (ppack->name[i][0] != '\0') {
pairlen = snprintf(tmpbuf + curpos, maxlen - curpos, "%s : %s\r\n",
ppack->name[i], ppack->value[i]);
if (pairlen >= (int)(maxlen - curpos)) {
ret = -1;
goto end;
}
curpos += pairlen;
if ((curpos - SIZE_NUM) > MAX_HEAD_SIZE) {
ret = -1;
goto end;
}
}
}
curpos += snprintf(tmpbuf + curpos, maxlen - curpos, "%s", "\r\n");
headsize = (int)(curpos - SIZE_NUM);
if (headsize > MAX_HEAD_SIZE) {
ret = -1;
goto end;
}
bodysize = ppack->bufsize;
if ((size = snprintf(tmpbuf, SIZE_NUM, "%s%d %d", BUF_START, headsize, bodysize)) >= SIZE_NUM) {
ret = -1;
goto end;
}
if (size < SIZE_NUM - 1) {
tmpbuf[size] = ' ';
for (i = size + 1; i < SIZE_NUM - 1; i++)
tmpbuf[i] = '@';
tmpbuf[SIZE_NUM - 1] = '\0';
} else {
ret = -1;
goto end;
}
if (curpos > SIZE_NUM + MAX_HEAD_SIZE) {
ret = -1;
goto end;
}
if (bodysize != 0) {
if (ppack->buf == NULL) {
ret = -1;
goto end;
}
memcpy(tmpbuf + curpos, ppack->buf, (size_t)bodysize);
curpos += bodysize;
}
memcpy(tmpbuf + curpos, BUF_EOF, BUF_EOF_SIZE);
curpos += BUF_EOF_SIZE;
if (curpos >= (size_t)bufsize) {
ret = -1;
} else {
memcpy(buf, tmpbuf, curpos);
ret = curpos;
}
end:
free(tmpbuf);
return ret;
}
int ul_pack_deserialization(void *buf, int bufsize, ul_package_t *ppack)
{
int headsize = -1;
int bodysize = -1;
int i = 0;
int packsize;
char sizebuf[SIZE_NUM];
char headbuf[MAX_HEAD_SIZE];
char tmp[10], eofstr[BUF_EOF_SIZE+1];
char *ph = headbuf;
char *q;
if (buf == NULL || bufsize <= 0 || ppack == NULL) {
return -1;
}
/* read STRART */
memcpy(sizebuf, buf, SIZE_NUM);
if(strncmp(sizebuf, BUF_START, BUF_START_SIZE) != 0) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: start buf error.[%d : %s].", SIZE_NUM, sizebuf);
return -1;
}
sscanf(sizebuf, "%s%d%d", tmp, &headsize, &bodysize);
if (headsize < 0 || bodysize < 0) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: size are negative [%d : %d]",
headsize, bodysize);
return -1;
}
if (bodysize > ppack->bufsize || headsize >= MAX_HEAD_SIZE) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: body buffer size not enough [%d : %d]",
ppack->bufsize, bodysize);
return -1;
}
packsize = SIZE_NUM + headsize + bodysize + BUF_EOF_SIZE;
if (bufsize < packsize) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: total buffer size not enough [%d : %d]",
bufsize, packsize);
return -1;
}
/* read HEAD */
memcpy(headbuf, (char *)buf + SIZE_NUM, headsize);
headbuf[headsize] = '\0';
/* read BODY */
memcpy(ppack->buf, (char *)buf + SIZE_NUM + headsize, bodysize);
ppack->bufsize = bodysize;
/* read END */
memcpy(eofstr, (char *)buf + SIZE_NUM + headsize + bodysize, BUF_EOF_SIZE);
eofstr[BUF_EOF_SIZE] = '\0';
if (strncmp(eofstr, BUF_EOF, BUF_EOF_SIZE) != 0 ) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: read EOF error.[%ld || %s].", BUF_EOF_SIZE, eofstr);
return -1;
}
/* parse HEAD */
while (*ph != '\0' && i < MAX_PACK_ITEM_NUM) {
q = strstr(ph, "\r\n");
if (NULL == q || q == ph)
break;
*q = '\0';
ppack->name[i][0] = '\0';
ppack->value[i][0] = '\0';
char *p ;
p = strstr(ph , " : ");
if (NULL == p || p-ph >= MAX_ITEM_SIZE || q-(p+3) >= MAX_ITEM_SIZE) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: head data error error [%s]", ph);
return -1;
}
snprintf(ppack->value[i], sizeof(ppack->value[i]), "%s", p+3);
sscanf(ph, "%s%s", ppack->name[i], tmp );
if (':' == tmp[0]) {
i++;
}
ph = q + 2;
}
int leftTag = 0;
if ((*ph != '\0') && (i == MAX_PACK_ITEM_NUM)) {
q = strstr(ph, "\r\n");
if (NULL == q || q == ph) {
leftTag = 0;
} else {
*q = '\0';
char *p;
p = strstr(ph, " : ");
if (NULL == p) {
leftTag = 0;
} else {
leftTag = 1;
}
}
}
if (1 == leftTag) {
ul_writelog(UL_LOG_WARNING, "ul_pack_deserialization: left key-value pairs in head, curr recv[%d]", i);
}
ppack->pairnum = i;
return packsize;
}
int redis_load_conf()
{
Ul_confdata *pconf;
pconf=ul_initconf(DEFAULT_CONF_NUM);
if(!pconf)
{
printf("\n[Fatal Error] init config file error! \n");
return FAIL;
}
memset(&redis_conf,0,sizeof(redis_conf));
strcpy(redis_conf.confpath,DEFAULT_CONFPATH_REDIS);
strcpy(redis_conf.conffile,DEFAULT_CONFFILE_REDIS);
int ret=ul_readconf(redis_conf.confpath,redis_conf.conffile,pconf);
if(ret<0){
printf("\n[Fatal Error] load config file error! path[%s] file[%s] \n",redis_conf.confpath,redis_conf.conffile);
return FAIL;
}
if (ul_getconfstr(pconf, "ip", redis_conf.redis_conf.ip))
{
printf("Conf: redis server ip: %s\n",redis_conf.redis_conf.ip);
ul_writelog(UL_LOG_NOTICE, "Conf: redis server ip: %s", redis_conf.redis_conf.ip);
}
else
{
strcpy( redis_conf.redis_conf.ip,DEFAULT_REDIS_SERVER_IP);
ul_writelog(UL_LOG_NOTICE, "Conf: redis server ip is: %s DEFAULT", DEFAULT_REDIS_SERVER_IP);
}
if (ul_getconfint(pconf, "port",&( redis_conf.redis_conf.port)))
{
ul_writelog(UL_LOG_NOTICE, "Conf: redis port: %d", redis_conf.redis_conf.port);
}
else
{
redis_conf.redis_conf.port = DEFAULT_REDIS_PORT;
ul_writelog(UL_LOG_NOTICE, "Conf: redis port is: %d DEFAULT", DEFAULT_REDIS_PORT);
}
if (ul_getconfint(pconf, "conn_timeout",(int *)&( redis_conf.redis_conf.conn_timeout.tv_sec)))
{
ul_writelog(UL_LOG_NOTICE, "Conf: redis conn_timeout: %d", redis_conf.redis_conf.conn_timeout.tv_sec);
}
else
{
redis_conf.redis_conf.conn_timeout.tv_sec = DEFAULT_CONN_TIMEOUT;
ul_writelog(UL_LOG_NOTICE, "Conf: redis conn_timeout is: %d DEFAULT", DEFAULT_CONN_TIMEOUT);
}
if (ul_getconfint(pconf, "rdwr_timeout",(int *)&( redis_conf.redis_conf.rdwr_timeout.tv_sec)))
{
ul_writelog(UL_LOG_NOTICE, "Conf: redis rdwr_timeout: %d", redis_conf.redis_conf.rdwr_timeout.tv_sec);
}
else
{
redis_conf.redis_conf.rdwr_timeout.tv_sec = DEFAULT_RDWR_TIMEOUT;
ul_writelog(UL_LOG_NOTICE, "Conf: redis rdwr_timeout is: %d DEFAULT", DEFAULT_RDWR_TIMEOUT);
}
ul_freeconf(pconf);
pconf=NULL;
return OK;
}
//加载model文件
//返回值<0 为出错，=0为正常
int CrfModel::load_model(const char *filename, float this_path_factor) {
//CHECK_FALSE(mmap_.open(filename1)) << mmap_.what();
//使用mmap读入模型文件
//原来使用mmap加载双数组和alpha数组，现在改为全部复制到model的内存区域中
Mmap <char> mmap_;
FILE* fp = fopen(filename, "r");
if(fp == NULL) {
ul_writelog(UL_LOG_FATAL, "[%s]: open model file[%s] failed! Error[%m]", __func__, filename);
return -1;
}
fclose(fp);
if (!mmap_.open(filename)) {
ul_writelog(UL_LOG_FATAL, "[%s]: open filename[%s] failed", __func__, filename);
return -1;
}
char *ptr = mmap_.begin();
//读入版本号
read_static<unsigned int>(&ptr, &version);
ul_writelog(UL_LOG_TRACE, "[%s]: VERSION = %d", __func__,version);
//CHECK_FALSE(version_ / 100 == version / 100)
// << "model version is different: " << version_
// << " vs " << version << " : " << filename1;
int type = 0;
read_static<int>(&ptr, &type);
read_static<double>(&ptr, &cost_factor_);
read_static<unsigned int>(&ptr, &maxid_);
read_static<unsigned int>(&ptr, &xsize_);
unsigned int dsize = 0;
read_static<unsigned int>(&ptr, &dsize);
//使用字符串数组代替vector<string>保存类别的字符表示y_
unsigned int y_str_size;
read_static<unsigned int>(&ptr, &y_str_size);
char *y_str = read_ptr(&ptr, y_str_size);
size_t pos = 0;
//减少y_占用内存
/*
ysize_ = 0;
while (pos < y_str_size) {
//y_.push_back(y_str + pos);
strncpy(y_[ysize_], y_str + pos, CRF_MAX_WORD_LEN);
ysize_++;
while (y_str[pos++] != '\0') {}
}
*/
//先计算YSIZE和最大yname的长度
ysize_ = 0;
max_yname_len = 0;
while(pos < y_str_size) {
ysize_++;
unsigned int yname_len = 0;
while(y_str[pos++] != '\0') {
yname_len++;
}
if(yname_len > max_yname_len) {
max_yname_len = yname_len;
}
}
//分配y_用来存放yname
y_ = (char **)calloc(ysize_, sizeof(char *));
if(NULL == y_) {
ul_writelog(UL_LOG_FATAL, "[%s]: fail to malloc for yname", __func__);
return -1;
}
for(unsigned int i = 0; i < ysize_; i++) {
y_[i] = (char*)calloc(max_yname_len + 1, sizeof(char));
if(NULL == y_[i]) {
ul_writelog(UL_LOG_FATAL, "[%s]: fail to malloc for yname[%d]", __func__, i);
return -1;
}
}
//重新扫描复制yname
pos = 0;
unsigned int ycount = 0;
while(pos < y_str_size) {
bzero(y_[ycount], max_yname_len + 1);
strncpy(y_[ycount], y_str + pos, max_yname_len);
while(y_str[pos++] != '\0') {
}
ycount++;
}
//debug
/*
for(unsigned int i = 0; i < ysize_; i++){
fprintf(stderr, "y[%d]=[%s]\n", i, y_[i]);
}
*/
// fprintf(stderr,"ysize = %d\n",ysize_);
// load unigram templs and bigram templs
//模型使用字符串数组代替vector<string>存储，并为减少函数调用开销，CrfTag在标注中获得模板指针
unigram_templs_num = 0;
unigram_templs_ = (char **)calloc(MAX_TEMPLS_NUM, sizeof(char *));
if(NULL == unigram_templs_) {
ul_writelog(UL_LOG_FATAL, "[%s]: fail to malloc for unigram templs", __func__);
return -1;
}
bigram_templs_num = 0;
bigram_templs_ = (char **)calloc(MAX_TEMPLS_NUM, sizeof(char *));
if(NULL == bigram_templs_) {
ul_writelog(UL_LOG_FATAL, "[%s]: fail to malloc for bigram templs", __func__);
return -1;
}
unsigned int tmpl_str_size;
read_static<unsigned int>(&ptr, &tmpl_str_size);
char *tmpl_str = read_ptr(&ptr, tmpl_str_size);
pos = 0;
while (pos < tmpl_str_size) {
char *v = tmpl_str + pos;
if (v[0] == '\0') {
++pos;
} else if (v[0] == 'U') {
if(unigram_templs_num > MAX_TEMPLS_NUM) {
ul_writelog(UL_LOG_FATAL, "[%s]: too many unigram templs, unigram templs num=%d", __func__, unigram_templs_num);
return -1;
}
unigram_templs_[unigram_templs_num] = (char *)calloc(MAX_TEMPLS_LEN, sizeof(char));
if(NULL == unigram_templs_[unigram_templs_num]) {
ul_writelog(UL_LOG_FATAL, "[%s]: fail to malloc for unigram templs", __func__);
return -1;
}
strncpy(unigram_templs_[unigram_templs_num], v, MAX_TEMPLS_LEN);
ul_writelog(UL_LOG_TRACE, "[%s]: unigram_templs[%d](%s)", __func__, unigram_templs_num, unigram_templs_[unigram_templs_num]);
unigram_templs_num++;
} else if (v[0] == 'B') {
if(bigram_templs_num > MAX_TEMPLS_NUM) {
ul_writelog(UL_LOG_FATAL, "[%s]: too many bigram templs, bigram templs num=%d", __func__, bigram_templs_num);
return -1;
}
bigram_templs_[bigram_templs_num] = (char *)calloc(MAX_TEMPLS_LEN, sizeof(char));
if(NULL == bigram_templs_[bigram_templs_num]) {
ul_writelog(UL_LOG_FATAL, "[%s]: fail to malloc for bigram templs", __func__);
return -1;
}
strncpy(bigram_templs_[bigram_templs_num], v, MAX_TEMPLS_LEN);
ul_writelog(UL_LOG_TRACE, "[%s]: bigram_templs[%d](%s)", __func__, bigram_templs_num, bigram_templs_[bigram_templs_num]);
bigram_templs_num++;
} else {
//CHECK_FALSE(true) << "unknown type: " << v;
}
while (tmpl_str[pos++] != '\0') {}
}
//检查是否为单一B模板
if( bigram_templs_num != 1 || strcmp(bigram_templs_[0],"B") !=0 )
{
ul_writelog(UL_LOG_WARNING,"this is not a single bigram templetes.\n");
return -1;
}
//复制双数组da
da_mem_ = (void*)malloc(dsize);
if(da_mem_ == NULL) {
ul_writelog(UL_LOG_WARNING, "[%s]: malloc for da memory fail.", __func__);
return -1;
}
memcpy(da_mem_, ptr, dsize);
ul_writelog(UL_LOG_DEBUG, "[%s]: copy double array to memory, dsize=%d", __func__, dsize);
da_.set_array(da_mem_);
ptr += dsize;
//复制alpha数组，从float转为int，乘上CONV_INT_FACTOR。该因子较大时基本无精度损失
float *alpha_float_ = reinterpret_cast<float *>(ptr);
ptr += sizeof(alpha_float_[0]) * maxid_;
alpha_int_ = (int*)calloc(maxid_ + 1, sizeof(int));
if(alpha_int_ == NULL) {
ul_writelog(UL_LOG_WARNING, "[%s]: malloc for alpha int array fail.maxid=%d", __func__, maxid_);
return -1;
}
int conv_alpha_int;
for(unsigned int i = 0; i < maxid_; i++) {
conv_alpha_int = (int)(alpha_float_[i] * CONV_INT_FACTOR);
alpha_int_[i] = conv_alpha_int;
// fprintf(stderr,"<<>>%d\n",alpha_int_[i]);
}
ul_writelog(UL_LOG_DEBUG, "[%s]: conv alpha from float to int, maxid=%d", __func__, maxid_);
if( version == 100 )
{
if (ptr != mmap_.end())
{
ul_writelog(UL_LOG_FATAL, "[%s]: model file is broken: %s", __func__, filename);
return -1;
}
}
else
{
//my add
read_static<int>(&ptr, &dan_size); //读取单一模板的辅助数据结构的size
read_static<int>(&ptr, &zu_size); //读取组合模板的辅助数据结构的size
dan_array = (int*)calloc(dan_size,sizeof(int));
zu_array = (int*)calloc(zu_size,sizeof(int));
if( dan_array == NULL || zu_array == NULL)
{
ul_writelog(UL_LOG_FATAL, "dan_array or zu_array is NULL");
return -1;
}
ul_writelog(UL_LOG_DEBUG, "%d %d\n",dan_size,zu_size);
memcpy(dan_array,ptr,dan_size*sizeof(int));
ptr+=dan_size*sizeof(int);
memcpy(zu_array,ptr,zu_size*sizeof(int));
ptr+=zu_size*sizeof(int);
if (ptr != mmap_.end())
{
ul_writelog(UL_LOG_FATAL, "[%s]: model file is broken: %s", __func__, filename);
return -1;
}
}
mmap_.close(); //释放mmap占用内存
b_templs_alpha_p = get_b_templs_start(); //获取B指针
if( NULL == b_templs_alpha_p )
{
ul_writelog(UL_LOG_WARNING,"b_templs_alpha_p is NULL.\n");
return -1;
}
path_factor = this_path_factor;
for(unsigned int i = 0; i < ysize_ * ysize_; i++) {
*(b_templs_alpha_p + i) = int(*(b_templs_alpha_p + i) * path_factor * cost_factor_);
/*
fprintf(stderr, "%d ", *(b_templs_alpha_p + i));
if((i+1) % ysize_ == 0){
fprintf(stderr, "\n");
}
*/
}
ul_writelog(UL_LOG_DEBUG, "[%s]: set path factor to %f", __func__, path_factor);
model_init_stat = MODEL_INIT_FINISH;
return 0;
}
