Exemplo n.º 1
0
/* kid */
int xmap_kid(XMAP *xmap, int64_t key)
{
    int id = -1, k = 0, x = 0, old = 0, root = 0;
    if(xmap && key)
    {
        MUTEX_LOCK(xmap->mutex);
        k = DBKMASK(key);
        if((root = xmap->state->masks[k].root) > 0)
        {
            x = xmap->state->id_max + 1;
            if(mmtree64_try_insert(xmap->tree64, root, key, x, &old) > 0)
            {
                if(old > 0)
                {
                    id = old;
                }
                else
                {
                    id = ++(xmap->state->id_max);
                    xmap->state->masks[k].total++;
                    CHECK_XMMETAIO(xmap, id);
                }
            }
            else
            {
                FATAL_LOGGER(xmap->logger, "Try_insert(%lld,%d) failed", LL64(key), x);
            }
        }
        MUTEX_UNLOCK(xmap->mutex);
    }
    return id;
}
Exemplo n.º 2
0
/* set connection */
int conn_set(CONN *conn)
{
    short flag = 0;
    if(conn && conn->fd > 0 )
    {
        fcntl(conn->fd, F_SETFL, O_NONBLOCK);
        if(conn->evbase && conn->event)
        {
            flag = E_READ|E_PERSIST;
            if(conn->status == CONN_STATUS_READY) flag |= E_WRITE;
            conn->event->set(conn->event, conn->fd, flag, (void *)conn, &conn_event_handler);
            conn->evbase->add(conn->evbase, conn->event);
            DEBUG_LOGGER(conn->logger, "setting connection[%s:%d] via %d", 
                    conn->ip, conn->port, conn->fd);
            return 0;
        }
        else
        {
            FATAL_LOGGER(conn->logger, "Connection[%08x] fd[%d] evbase or"
                    "initialize event failed, %s", conn, conn->fd, strerror(errno));	
            /* Terminate connection */
            CONN_TERMINATE(conn);
        }
    }	
    return -1;	
}
Exemplo n.º 3
0
/* write handler */
int conn_write_handler(CONN *conn)
{
    int ret = -1, n = 0;
    CONN_CHECK_RET(conn, ret);
    CHUNK *cp = NULL;

    if(conn && conn->send_queue && QTOTAL(conn->send_queue) > 0)
    {
        DEBUG_LOGGER(conn->logger, "Ready for send data to %s:%d via %d "
                "qtotal:%d qhead:%d qcount:%d",
                conn->ip, conn->port, conn->fd, QTOTAL(conn->send_queue),
                QHEAD(conn->send_queue), QCOUNT(conn->send_queue));   
        if(QUEUE_HEAD(conn->send_queue, PCHUNK, &cp) == 0)
        {
            DEBUG_LOGGER(conn->logger, "Ready for send data to %s:%d via %d qtotal:%d pcp:%08x",
                    conn->ip, conn->port, conn->fd, QTOTAL(conn->send_queue), cp);   
            if((n = CHUNK_WRITE(cp, conn->fd)) > 0)
            {
                conn->sent_data_total += n;
                DEBUG_LOGGER(conn->logger, "Sent %d byte(s) (total sent %lld) "
                        "to %s:%d via %d leave %lld", n, conn->sent_data_total, 
                        conn->ip, conn->port, conn->fd, CK_LEFT(cp));
                /* CONN TIMER sample */
                TIMER_SAMPLE(conn->timer);
                if(CHUNK_STATUS(cp) == CHUNK_STATUS_OVER )
                {
                    if(QUEUE_POP(conn->send_queue, PCHUNK, &cp) == 0)
                    {
                        DEBUG_LOGGER(conn->logger, "Completed chunk[%08x] and clean it leave %d",
                                cp, QTOTAL(conn->send_queue));
                        CK_CLEAN(cp);
                    }
                }
                ret = 0;
            }
            else
            {
                FATAL_LOGGER(conn->logger, "Sending data to %s:%d via %d failed, %s",
                        conn->ip, conn->port, conn->fd, strerror(errno));
                /* Terminate connection */
                CONN_TERMINATE(conn);
            }
        }
        if(QTOTAL(conn->send_queue) <= 0)
        {
            conn->event->del(conn->event, E_WRITE);
        }
    }
    return ret;
}
Exemplo n.º 4
0
/* push index */
int qindex_push_index(QINDEX *qindex, int64_t key, IBDATA *block)
{
    DOCHEADER *docheader = NULL;
    XPACKET *xpackets = NULL;
    int mid = 0, old = 0;

    if(qindex && block->ndata > 0 && (docheader = (DOCHEADER *)block->data)) 
    {
        MUTEX_LOCK(qindex->mutex);
        if(docheader->size == block->ndata && (xpackets = (XPACKET *)(qindex->xpacketio.map))
            && (mid = mmtree64_try_insert(qindex->idmap, qindex->state->rootid, 
                            key, (qindex->state->id_max+1), &old)) > 0)
        {
            if(old > 0) mid = old;
            else mid = ++(qindex->state->id_max);
            db_set_data(PDB(qindex->db), mid, block->data, block->ndata);
            if(mid <= qindex->state->xpackettotal)
            {
                if(xpackets[mid].crc != docheader->crc)
                    qindex_update(qindex, mid, 1);
                else
                    qindex_update(qindex, mid, 0);
                xpackets[mid].status = docheader->status;
                xpackets[mid].crc = docheader->crc;
                ACCESS_LOGGER(qindex->logger, "update-index{gloablid:%lld mid:%d total:%d}", LL(docheader->globalid), mid, qindex->state->xpackettotal);
            }
            else
            {
                qindex->state->xpackettotal = mid;
                CHECK_XPACKETIO(qindex);
                if((xpackets = (XPACKET *)(qindex->xpacketio.map)))
                {
                    xpackets[mid].status = docheader->status;
                    xpackets[mid].crc = docheader->crc;
                    ACCESS_LOGGER(qindex->logger, "new-index{gloablid:%lld mid:%d}", LL(docheader->globalid), mid);
                }
            }
        }
        else
        {
            FATAL_LOGGER(qindex->logger, "Invalid document size:%d ndata:%d id:%lld crc:%d", docheader->size, block->ndata, LL(docheader->globalid), docheader->crc);
            _exit(-1);
        }
        MUTEX_UNLOCK(qindex->mutex);
    }
    return mid;
}
Exemplo n.º 5
0
CONN *http_newconn(int id, char *ip, int port, int is_ssl)
{
    CONN *conn = NULL;

    if(running_status && ip && port > 0)
    {
        if(is_ssl) service->session.flags |= SB_USE_SSL;
        if((conn = service->newconn(service, -1, -1, ip, port, NULL)))
        {
            conn->c_id = id;
        }
        else
        {
            FATAL_LOGGER(logger, "new_conn(%s:%d) falied, %s", ip, port, strerror(errno));
        }
    }
    return conn;
}
Exemplo n.º 6
0
/* new mindex */
MINDEX *mindex_new(QINDEX *qindex)
{
    MINDEX *mindex = NULL;
    if(strlen(qindex->qtask_server_host) > 0 && qindex->qtask_server_port > 0
            && (qindex->qtask_commitid > 0 || qindex->qtask_queueid > 0)
            && (mindex = xmm_mnew(sizeof(MINDEX))))
    {
        /* set mtask */
        mtask_set(&(mindex->mtask), qindex->qtask_server_host, qindex->qtask_server_port,
                qindex->qtask_commitid, qindex->qtask_queueid);
        /* set db connection*/
        if(dbase_set(&(mindex->db), qindex->db_host, qindex->db_port) != 0
                || dbase_connect(&(mindex->db)) != 0)
        {
            FATAL_LOGGER(qindex->logger, "connect_source_db(%s:%d) failed, %s", qindex->db_host, qindex->db_port, strerror(errno));
        }
    }
    return mindex;
}
Exemplo n.º 7
0
CONN *http_newconn(int id, char *ip, int port, int is_ssl)
{
    CONN *conn = NULL;

    if(running_status && ip && port > 0)
    {
        if(is_ssl) service->session.is_use_SSL = 1;
        if((conn = service->newconn(service, -1, -1, ip, port, NULL)))
        {
            conn->c_id = id;
            conn->start_cstate(conn);
            //service->newtransaction(service, conn, id);
            //usleep(10);
        }
        else
        {
            FATAL_LOGGER(logger, "new_conn(%s:%d) falied, %s", ip, port, strerror(errno));
        }
    }
    return conn;
}
Exemplo n.º 8
0
/* read handler */
int conn_read_handler(CONN *conn)
{
    int ret = -1, n = -1;
    CONN_CHECK_RET(conn, ret);

    if(conn)
    {
        /* Receive OOB */
        if((n = MB_RECV(conn->oob, conn->fd, MSG_OOB)) > 0)
        {
            conn->recv_oob_total += n;
            DEBUG_LOGGER(conn->logger, "Received %d bytes OOB total %lld from %s:%d via %d",
                    n, conn->recv_oob_total, conn->ip, conn->port, conn->fd);
            conn->oob_handler(conn);
            /* CONN TIMER sample */
            TIMER_SAMPLE(conn->timer);
            return (ret = 0);
        }
        /* Receive to chunk with chunk_read_state before reading to buffer */
        CONN_CHUNK_READ(conn, n);
        /* Receive normal data */
        if((n = MB_READ(conn->buffer, conn->fd)) <= 0)
        {
            FATAL_LOGGER(conn->logger, "Reading %d bytes left:%d data from %s:%d via %d failed, %s",
                    n, MB_LEFT(conn->buffer), conn->ip, conn->port, conn->fd, strerror(errno));
            /* Terminate connection */
            CONN_TERMINATE(conn);
            return (ret = 0);
        }
        /* CONN TIMER sample */
        TIMER_SAMPLE(conn->timer);
        conn->recv_data_total += n;
        DEBUG_LOGGER(conn->logger, "Received %d bytes data total %lld from %s:%d via %d",
                n, conn->recv_data_total, conn->ip, conn->port, conn->fd);
        conn->packet_reader(conn);
        ret = 0;
    }
    return ret;
}
Exemplo n.º 9
0
/* packet reader */
int conn_packet_reader(CONN *conn)
{
    int len = -1, i = 0;
    CB_DATA *data = NULL;
    char *p = NULL, *e = NULL;
    int packet_type = 0;

    CONN_CHECK_RET(conn, -1);

    if(conn)
    {
        data = PCB(conn->buffer);
        packet_type = conn->session.packet_type;
        DEBUG_LOGGER(conn->logger, "Reading packet type[%d]", packet_type);
        /* Remove invalid packet type */
        if(!(packet_type & PACKET_ALL))
        {
            FATAL_LOGGER(conn->logger, "Unkown packet_type[%d] on %s:%d via %d",
                    packet_type, conn->ip, conn->port, conn->fd);
            /* Terminate connection */
            CONN_TERMINATE(conn);
        }
        /* Read packet with customized function from user */
        if(packet_type == PACKET_CUSTOMIZED && conn->session.packet_reader)
        {
            len = conn->session.packet_reader(conn, data);
            DEBUG_LOGGER(conn->logger,
                    "Reading packet with customized function[%08x] length[%d] on %s:%d via %d",
                    conn->session.packet_reader, len, conn->ip, conn->port, conn->fd);
            goto end;
        }
        /* Read packet with certain length */
        else if(packet_type == PACKET_CERTAIN_LENGTH
                && MB_NDATA(conn->buffer) >= conn->session.packet_length)
        {
            len = conn->session.packet_length;
            DEBUG_LOGGER(conn->logger, "Reading packet with certain length[%d] on %s:%d via %d",
                    len, conn->ip, conn->port, conn->fd);
            goto end;
        }
        /* Read packet with delimiter */
        else if(packet_type == PACKET_DELIMITER && conn->session.packet_delimiter
                && conn->session.packet_delimiter_length > 0)
        {
            p = MB_DATA(conn->buffer);
            e = MB_END(conn->buffer);
            i = 0;
            while(p < e && i < conn->session.packet_delimiter_length )
            {
                if(((char *)conn->session.packet_delimiter)[i++] != *p++) i = 0;
                if(i == conn->session.packet_delimiter_length)
                {
                    len = p - MB_DATA(conn->buffer);
                    break;
                }
            }
            DEBUG_LOGGER(conn->logger, "Reading packet with delimiter[%d] "
                    "length[%d] on %s:%d via %d", conn->session.packet_delimiter_length, 
                    len, conn->ip, conn->port, conn->fd);
            goto end;
        }
        return len;
end:
        /* Copy data to packet from buffer */
        if(len > 0)
        {
            MB_RESET(conn->packet);
            MB_PUSH(conn->packet, MB_DATA(conn->buffer), len);
            MB_DEL(conn->buffer, len);
            /* For packet handling */
            conn->s_state = S_STATE_PACKET_HANDLING;
            conn->push_message(conn, MESSAGE_PACKET);
        }
    }
    return len;
}
Exemplo n.º 10
0
/* 
 * initialize wtable 
 * */
WTABLE *wtable_init(char *dir)
{
    char path[WT_PATH_MAX], *p = NULL;
    struct stat st = {0};
    WTABLE *w = NULL;
    int n = 0;

    if(dir && (w = (WTABLE *)xmm_mnew(sizeof(WTABLE))))
    {
        n = sprintf(path, "%s/%s", dir, WT_LOG_NAME);
        force_mkdir(path);
        p = path;
        LOGGER_INIT(w->logger, p);
        /* state */
        n = sprintf(path, "%s/%s", dir, WT_STATE_NAME);
        if((w->statefd = open(path, O_CREAT|O_RDWR, 0644)) <= 0
                || fstat(w->statefd, &st) != 0)
        {
            if(w->statefd > 0) close(w->statefd);
            FATAL_LOGGER(w->logger, "open state file[%s] failed, %s",
                    path, strerror(errno));
            _exit(-1);
        }
        else
        {
            if(st.st_size < sizeof(WSTATE)
                    && ftruncate(w->statefd, sizeof(WSTATE)) != 0)
            {
                _exit(-1);
            }
            if((w->state = (WSTATE *)mmap(NULL, sizeof(WSTATE),
                            PROT_READ|PROT_WRITE, MAP_SHARED, w->statefd, 0)) == NULL
                    || w->state == (void *)-1)
            {
                FATAL_LOGGER(w->logger, "mmap state failed, %s", strerror(errno));
                _exit(-1);
            }
            if(st.st_size < sizeof(WSTATE))
                memset(((char *)w->state + st.st_size), 0, sizeof(WSTATE) - st.st_size);
        }
        w->workers = w->state->workers;
        memset(w->workers, 0, sizeof(WORKER) * W_WORKER_MAX);
        w->state->nworkers = 0;
        w->state->conn_total = 0;
        n = sprintf(path, "%s/%s", dir, WT_MDB_DIR);
        w->mdb = db_init(path, DB_USE_MMAP);
        /* mmtrie */
        n = sprintf(path, "%s/%s", dir, WT_MAP_NAME);
        if((w->map = mmtrie_init(path)) == NULL) _exit(-1);
        /* appmap */
        n = sprintf(path, "%s/%s", dir, WT_APPS_NAME);
        if((w->appmap = mmtree64_init(path)) == NULL) _exit(-1);
        mmtree64_use_all(w->appmap);
        /* logger & mutex & mtree & mqueue */
        if((w->queue = mqueue_init()) == NULL) _exit(-1);
        if((w->mtree = mtree_init()) == NULL) _exit(-1);
        w->whitelist = mtree_new_tree(w->mtree);
        //MQ(w->queue)->logger = w->logger;
        MUTEX_INIT(w->mutex);
    }
    return w;
}
Exemplo n.º 11
0
/* initialize XMAP */
XMAP *xmap_init(char *basedir)
{
    char path[XM_PATH_MAX];
    struct stat st = {0};
    XMAP *xmap = NULL;
    int i = 0, k = 0, v = 0;
    unsigned int id  = 0;

    if(basedir && (xmap = (XMAP *)xmm_mnew(sizeof(XMAP))))
    {
        //xmap->mtrie = mtrie_init();
        MUTEX_INIT(xmap->mutex);
        /* logger */
        sprintf(path, "%s/%s", basedir, "xmap.log");
        xmap_mkdir(path);
        LOGGER_INIT(xmap->logger, path);
        /* tree */
        sprintf(path, "%s/%s", basedir, "xmap.tree");
        xmap->tree = mmtree_init(path);
        sprintf(path, "%s/%s", basedir, "xmap.tree64");
        xmap->tree64 = mmtree64_init(path);
        /* kmap */
        sprintf(path, "%s/%s", basedir, "xmap.kmap");
        xmap->kmap = mmtrie_init(path);
        /* queue */
        sprintf(path, "%s/%s", basedir, "xmap.queue");
        xmap->queue = mmqueue_init(path);
        /* db */
        sprintf(path, "%s/%s", basedir, "cache/");
        xmap->db = cdb_init(path, CDB_USE_MMAP);
        /* state */
        sprintf(path, "%s/%s", basedir, "xmap.state");
        if((xmap->stateio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
                && fstat(xmap->stateio.fd, &st) == 0)
        {
            if((xmap->stateio.map = mmap(NULL, sizeof(XMSTATE), PROT_READ|PROT_WRITE,
                                         MAP_SHARED, xmap->stateio.fd, 0)) == NULL
                    || xmap->stateio.map == (void *)-1)
            {
                FATAL_LOGGER(xmap->logger, "mmap state:%s failed, %s", path, strerror(errno));
                _exit(-1);
            }
            xmap->state = (XMSTATE *)(xmap->stateio.map);
            xmap->stateio.end = st.st_size;
            if(st.st_size == 0)
            {
                xmap->stateio.end = xmap->stateio.size = sizeof(XMSTATE);
                if(ftruncate(xmap->stateio.fd, xmap->stateio.end) != 0)
                {
                    FATAL_LOGGER(xmap->logger, "ftruncate state %s failed, %s", path, strerror(errno));
                    _exit(-1);
                }
                memset(xmap->state, 0, sizeof(XMSTATE));
            }
            for(i = 0; i < DBASE_MASK_MAX; i++)
            {
                if(xmap->state->masks[i].root == 0)
                    xmap->state->masks[i].root = mmtree64_new_tree(xmap->tree64);
            }
        }
        /* disk */
        sprintf(path, "%s/%s", basedir, "xmap.disk");
        if((xmap->diskio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
                && fstat(xmap->diskio.fd, &st) == 0)
        {
            if((xmap->diskio.map = mmap(NULL, sizeof(MDISK) * XM_DISK_MAX, PROT_READ|PROT_WRITE,
                                        MAP_SHARED, xmap->diskio.fd, 0)) == NULL
                    || xmap->diskio.map == (void *)-1)
            {
                FATAL_LOGGER(xmap->logger, "mmap disk:%s failed, %s", path, strerror(errno));
                _exit(-1);
            }
            xmap->disks = (MDISK *)(xmap->diskio.map);
            xmap->diskio.end = st.st_size;
        }
        /* meta */
        sprintf(path, "%s/%s", basedir, "xmap.meta");
        if((xmap->metaio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
                && fstat(xmap->metaio.fd, &st) == 0)
        {
            if((xmap->metaio.map = mmap(NULL, sizeof(XMMETA) * XM_META_MAX, PROT_READ|PROT_WRITE,
                                        MAP_SHARED, xmap->metaio.fd, 0)) == NULL
                    || xmap->metaio.map == (void *)-1)
            {
                FATAL_LOGGER(xmap->logger, "mmap meta:%s failed, %s", path, strerror(errno));
                _exit(-1);
            }
            xmap->metas = (XMMETA *)(xmap->metaio.map);
            xmap->metaio.end = st.st_size;
            xmap->start_time = time(NULL);
        }
        if(!xmap->state->qwait) xmap->state->qwait = mmtree_new_tree(xmap->tree);
        if(!xmap->state->qleft) xmap->state->qleft = mmqueue_new(xmap->queue);
        xmap->state->id_wait = 0;
        while((id = mmtree_min(xmap->tree, xmap->state->qwait, &k, &v)))
        {
            mmtree_remove(xmap->tree, xmap->state->qwait, id, NULL, NULL);
        }
        while(mmqueue_pop(xmap->queue, xmap->state->qleft, (int *)&id) > 0);
    }
    return xmap;
}
Exemplo n.º 12
0
/* add index */
int ibase_index(IBASE *ibase, int docid, IBDATA *block)
{
    char *term = NULL, buf[IB_BUF_SIZE], *data = NULL, *p = NULL, 
         *pp = NULL, *nexts = NULL;
    int i = 0, termid = 0, n = 0, k = 0, ndocid = 0, ret = -1, ndata = 0, secid = 0, 
        *intlist = NULL, *np = NULL, last_docid = 0;
    DOCHEADER *docheader = NULL;
    int64_t *longlist = NULL;
    double *doublelist = NULL;
    IHEADER *iheader = NULL;
    STERM *termlist = NULL;
    MDB *index = NULL, *posting = NULL;
    off_t size = 0;

    if((docheader = (DOCHEADER *)block->data) 
            && (secid = docheader->secid) >= 0 && docheader->secid < IB_SEC_MAX)
    {
        index = (MDB *)(ibase->mindex[docheader->secid]);
        posting = (MDB *)(ibase->mposting[docheader->secid]);
        if(docheader->dbid != -1)
        {
            ibase->state->dtotal++;
            ibase->state->ttotal += (int64_t)docheader->terms_total;
        }
        if(ibase->state->used_for == IB_USED_FOR_INDEXD)
        {
            /* add to source */
            if(((off_t)docid * (off_t)sizeof(IHEADER)) >= ibase->state->headers[secid].end)
            {
                ibase->state->headers[secid].old = ibase->state->headers[secid].end;
                size = (off_t)((docid / IB_HEADERS_BASE) + 1) 
                    * (off_t)IB_HEADERS_BASE * (off_t)sizeof(IHEADER);
                ret = ftruncate(ibase->state->headers[secid].fd, size);
                ibase->state->headers[secid].end = size;
                memset(ibase->state->headers[secid].map + ibase->state->headers[secid].old, 0, 
                        ibase->state->headers[secid].end -  ibase->state->headers[secid].old);
            }
        }
        /* index */
        //end = block->data + block->ndata;
        termlist = (STERM *)(block->data + sizeof(DOCHEADER) 
                + docheader->nfields * sizeof(XFIELD));
        term = block->data + docheader->textblock_off;
        nexts = block->data + docheader->nexts_off;
        for(i = 0; i < docheader->nterms; i++)
        {
            termid = termlist[i].termid;
            //find/add termid
            if(termid > 0 && termlist[i].term_len > 0 && termlist[i].nexts_size >= 0 
                && (ret=mmtrie_add((MMTRIE *)ibase->mmtrie,term,termlist[i].term_len,termid))>0)
            {
                if(ibase->state->index_status != IB_INDEX_DISABLED)
                {
                    last_docid = 0;
                    ndocid = -1;
                    if(mdb_get_tag(index, termid, &last_docid) == 0)
                    {
                        ndocid = docid - last_docid;
                        if(ndocid <= 0) goto term_state_update;
                    }
                    else
                    {
                        ndocid = docid;
                    }
                    p = pp = buf;
                    if((ndata = ((sizeof(int) * 5))) > IB_BUF_SIZE)
                        p = pp = data = (char *)xmm_new(ndata);
                    /* compress index */
                    if(ibase->state->compression_status != IB_COMPRESSION_DISABLED)
                    {
                        n = ndocid; np = &n; ZVBCODE(np, p);
                        n = termlist[i].term_count; np = &n;  ZVBCODE(np, p);
                        n = i; np = &n; ZVBCODE(np, p);
                        n = termlist[i].bit_fields; np = &n; ZVBCODE(np, p);
                        n = termlist[i].nexts_size; np = &n; ZVBCODE(np, p);
                    }
                    else
                    {
                        memcpy(p, &docid, sizeof(int));p += sizeof(int);
                        memcpy(p, &(termlist[i].term_count), sizeof(int));p += sizeof(int);
                        memcpy(p, &i, sizeof(int));p += sizeof(int);
                        memcpy(p, &(termlist[i].bit_fields), sizeof(int));p += sizeof(int);
                        memcpy(p, &(termlist[i].nexts_size), sizeof(int));p += sizeof(int);
                    }
                    if(termlist[i].nexts_size > 0)
                    {
                        if(mdb_add_data(posting, termid, nexts, termlist[i].nexts_size) <= 0) 
                        {
                            FATAL_LOGGER(ibase->logger, "index posting term[%d] failed, %s", 
                                    termid, strerror(errno));
                            _exit(-1);
                        }
                        //WARN_LOGGER(ibase->logger, "docid:%lld termid:%d nexts_size:%d", IBLL(docheader->globalid), termid, termlist[i].nexts_size);
                    }
                    if(mdb_add_data(index, termid, pp, (p - pp)) <= 0)
                    {
                        FATAL_LOGGER(ibase->logger, "index term[%d] failed, %s", 
                                termid, strerror(errno));
                        if(data){free(data); data = NULL;}
                        _exit(-1);
                    }
                    mdb_set_tag(index, termid, docid);
                    if(data){xmm_free(data, ndata); data = NULL;}
                }
term_state_update:
                MUTEX_LOCK(ibase->mutex_termstate);
                if(termid > ibase->state->termid){ADD_TERMSTATE(ibase, termid);}
                if(ndocid > 0 )
                {
                    ((TERMSTATE *)(ibase->termstateio.map))[termid].len=termlist[i].term_len;
                    ((TERMSTATE *)(ibase->termstateio.map))[termid].total++;
                }
                MUTEX_UNLOCK(ibase->mutex_termstate);
            }
            else 
            {
                FATAL_LOGGER(ibase->logger, "Invalid ret:%d term[%d]{%.*s} termid:%d in doc:%d", ret, i, termlist[i].term_len, term, termid, docid);
                _exit(-1);
            }
            term += termlist[i].term_len;
            nexts += termlist[i].nexts_size;
        }
        if(ibase->state->used_for == IB_USED_FOR_INDEXD)
        {
            /* index int */
            if((n = ibase->state->int_index_fields_num) > 0 
                    && (intlist = (int *)(block->data + docheader->intblock_off)))
            {
                n += IB_INT_OFF;
                k = 0;
                for(i = IB_INT_OFF; i < n; i++)
                {
                    IMAP_SET(ibase->state->mfields[secid][i], docid, intlist[k]);
                    k++;
                }
            }
            /* index long */
            if((n = ibase->state->long_index_fields_num) > 0 
                    && (longlist = (int64_t *)(block->data + docheader->longblock_off)))
            {
                n += IB_LONG_OFF;
                k = 0;
                for(i = IB_LONG_OFF; i < n; i++)
                {
                    LMAP_SET(ibase->state->mfields[secid][i], docid, longlist[k]);
                    k++;
                }
            }
            /* index double */
            if((n = ibase->state->double_index_fields_num) > 0 
                    && (doublelist = (double *)(block->data + docheader->doubleblock_off)))
            {
                n += IB_DOUBLE_OFF;
                k = 0;
                for(i = IB_DOUBLE_OFF; i < n; i++)
                {
                    DMAP_SET(ibase->state->mfields[secid][i], docid, doublelist[k]);
                    k++;
                }
            }
            if((iheader = PIHEADER(ibase, docheader->secid, docid)))
            {
                iheader->status      = docheader->status;
                iheader->terms_total = docheader->terms_total;
                iheader->crc         = docheader->crc;
                iheader->category    = docheader->category;
                iheader->slevel      = docheader->slevel;
                iheader->rank        = docheader->rank;
                iheader->globalid    = docheader->globalid;
                //WARN_LOGGER(ibase->logger, "iheader->category:%p docheader->category:%p", (void *)iheader->category, (void *)docheader->category);
            }
        }
        ret = 0;
    }
    return ret;
}
Exemplo n.º 13
0
/* add document */
int ibase_add_document(IBASE *ibase, IBDATA *block)
{
    int ret = -1, localid = 0, secid = 0, n = 0, newid = 0;
    DOCHEADER *docheader = NULL;
    MHEADER *mheader = NULL;
    IHEADER *iheader = NULL;
    char line[IB_LINE_MAX];
    off_t size = 0;

    if(ibase && block && block->ndata > 0 && (docheader = (DOCHEADER *)(block->data)) 
            && (secid = docheader->secid) >= 0 && secid < IB_SEC_MAX
            && ibase_check_index_state(ibase, docheader) == 0
            && docheader->globalid && (n = sprintf(line, "%lld", IBLL(docheader->globalid))))
    {
        if((localid = mmtrie_xadd(ibase->docmap, line, n)) > 0)
        {
            MUTEX_LOCK(ibase->mutex);
            if(localid <= ibase->state->docid) 
            {
                if(ibase->state->used_for == IB_USED_FOR_INDEXD)
                {
                    if((mheader = PMHEADER(ibase, localid))
                        && (iheader = PIHEADER(ibase, mheader->secid, mheader->docid)))
                    {
                        /* disabled */
                        if(docheader->status < 0)
                        {
                            mheader->status = -1;
                            iheader->status = -1;
                            DEBUG_LOGGER(ibase->logger, "Update docid:%d globalid:%lld status:%d", localid, IBLL(docheader->globalid), docheader->status);
                            ret = 0;
                        }
                        else if(docheader->crc != mheader->crc) 
                        {
                            ibase_del_index(ibase, secid, localid);
                            newid = ++(ibase->state->ids[secid]);
                            DEBUG_LOGGER(ibase->logger, "ready_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
                            mheader->docid = newid;
                            mheader->crc = docheader->crc;
                            ret = ibase_index(ibase, newid, block);
                            DEBUG_LOGGER(ibase->logger, "over_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
                        }
                        else
                        {
                            DEBUG_LOGGER(ibase->logger, "update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
                            ret = ibase_update_index(ibase, mheader->docid, block);
                            DEBUG_LOGGER(ibase->logger, "over_update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
                        }
                    }
                }
                else
                {
                    docheader->dbid = -1;
                    ret = ibase_index(ibase, localid, block);
                }
            }
            else
            {
                if(ibase->state->used_for == IB_USED_FOR_INDEXD)
                {
                    if(((off_t)localid * (off_t)sizeof(MHEADER)) >= ibase->mheadersio.end)
                    {
                        ibase->mheadersio.old = ibase->mheadersio.end;
                        size = (off_t)((localid / IB_HEADERS_BASE) + 1) 
                            * (off_t)IB_HEADERS_BASE * (off_t)sizeof(MHEADER);
                        ret = ftruncate(ibase->mheadersio.fd, size);
                        ibase->mheadersio.end = size;
                        memset(ibase->mheadersio.map + ibase->mheadersio.old, 0, 
                                ibase->mheadersio.end -  ibase->mheadersio.old);
                    }
                    if((mheader = PMHEADER(ibase, localid)))
                    {
                        mheader->status = 0;
                        mheader->docid = ++(ibase->state->ids[secid]);
                        mheader->secid = docheader->secid;
                        mheader->crc = docheader->crc;
                        DEBUG_LOGGER(ibase->logger, "new_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
                        ret = ibase_index(ibase, mheader->docid, block);
                    }
                }
                else
                {
                    ret = ibase_index(ibase, localid, block);
                }
                ibase->state->docid = localid;
            }
            MUTEX_UNLOCK(ibase->mutex);
            if(ibase->state->used_for != IB_USED_FOR_QPARSERD 
                    && ibase->state->mmsource_status != IB_MMSOURCE_NULL && localid > 0)
            {
                ACCESS_LOGGER(ibase->logger, "docid:%lld/%d c_size:%d c_zsize:%d size:%d", docheader->globalid, localid, docheader->content_size, docheader->content_zsize, docheader->nexts_off);
                docheader->size = docheader->nexts_off;
                ret = db_set_data(PDB(ibase->source), localid, block->data, docheader->size);
            }
            ret = 0;
        }
        else
        {
            FATAL_LOGGER(ibase->logger, "new_index[%lld]{crc:%d rank:%f slevel:%d category:%lld} failed, %s", IBLL(docheader->globalid), docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category), strerror(errno));

        }
    }
    return ret;
}
Exemplo n.º 14
0
/* read upindex */
int qindex_read_upindex(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
    int ret = -1, mid = -1, nodeid = -1, id = 0, k = 0, n = 0, 
        left = 0, prev = 0, next = 0, *px = NULL;
    DOCHEADER *docheader = NULL;
    XPACKET *xpackets = NULL;
    QTNODE *nodes = NULL;
    QTASK *tasks = NULL;
    char *p = NULL;

    if(qindex && taskid >= 0 && data && taskid < Q_NODE_MAX * Q_TASKS_MAX
            && len && count && *len > 0)
    {
        MUTEX_LOCK(qindex->mutex);
        *count = 0;
        k = taskid % Q_TASKS_MAX;
        if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX  && (nodes = qindex->state->nodes)
                && (tasks = qindex->state->nodes[nodeid].tasks) && tasks[k].status > 0
                && (xpackets =  (XPACKET *)(qindex->xpacketio.map)))
        {
            if(tasks[k].nupdates > 0 && tasks[k].upid != tasks[k].upover)
            {
                if((*len = db_read_data(PDB(qindex->update), taskid, p)) > 0)
                {
                    *count =  tasks[k].upcount;
                    ret = mid = tasks[k].upid;
                }
                else
                {
                    tasks[k].upid = tasks[k].upover = 0;
                }
            }
            else if(tasks[k].nqueue > 0 && tasks[k].mmqid > 0)
            {
                tasks[k].upcount = 0;
                left = *len;
                p = data;
                id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
                do
                {
                    if(id > 0 && mid > 0)
                    {
                        if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[mid].nodeid != nodeid)
                        {
                            prev = xpackets[mid].prev;
                            next = xpackets[mid].next;
                            FATAL_LOGGER(qindex->logger, "Invalid rootid:%d id:%d prev:%d[nodeid:%d] next:%d[nodeid:%d] xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d nqueue:%d", tasks[k].mmqid, id, prev, xpackets[prev].nodeid, next, xpackets[next].nodeid, mid, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port, nodeid, tasks[k].nqueue);
                            tasks[k].mmqid = 0;tasks[k].nqueue = 0;
                            break;
                        }
                        if(db_get_data_len(PDB(qindex->db), mid) > (left-Q_LEFT_LEN)) break;
                        tasks[k].upid = mid;
                        px = (int *)p;
                        p += sizeof(int);
                        docheader = (DOCHEADER *)p;
                        if((n = db_read_data(PDB(qindex->db), mid, p)) > sizeof(DOCHEADER)) 
                        {
                            ACCESS_LOGGER(qindex->logger, "update_index(%d) globalid:%lld mid:%d status:%d nodeid:%d task[%s:%d]", id,  LL(docheader->globalid), mid, xpackets[mid].status, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port);
                            left -= n + sizeof(int);
                            p += n;
                            *px = n;
                            tasks[k].upcount++;
                        }
                        else
                        {
                            p -= sizeof(int);
                            FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
                            mid = -1;
                            //_exit(-1);
                            break;
                        }
                        mmqueue_pop(MMQ(qindex->mmqueue),tasks[k].mmqid, &mid);
                        tasks[k].nqueue--;
                        id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
                    }
                    else 
                    {
                        FATAL_LOGGER(qindex->logger, "Invalid qid:%d mid:%d", id, mid);
                        tasks[k].mmqid = 0;tasks[k].nqueue = 0;
                        mid = -1;
                        break;
                    }
                }while(tasks[k].nqueue > 0 && left > Q_LEFT_LEN && id > 0 && mid > 0);
                if((*count = tasks[k].upcount) > 0 && (*len -= left) > 0
                        && db_set_data(PDB(qindex->update), taskid, data, *len) >= 0)
                {
                    ret = tasks[k].upid;
                }
            }
            else ret = -1;
        }
        else ret = -2;
        MUTEX_UNLOCK(qindex->mutex);
    }
    return ret;
}
Exemplo n.º 15
0
/* read index */
int qindex_read_index(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
    int id = -1, nodeid = 0, x = 0, k = 0, *px = NULL, left = 0, last = 0, n = 0;
    DOCHEADER *docheader = NULL;
    XPACKET *xpackets = NULL;
    QTNODE *nodes = NULL;
    QTASK *tasks = NULL;
    char *p = NULL;

    if(qindex && qindex->state && taskid >= 0 && taskid < (Q_NODE_MAX * Q_TASKS_MAX)
            && data && len && *len > 0 && count) 
    {
        MUTEX_LOCK(qindex->mutex);
        *count = 0;
        k = taskid % Q_TASKS_MAX;
        if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX 
                && (xpackets = (XPACKET *)(qindex->xpacketio.map))
                && (nodes = qindex->state->nodes) && (tasks = nodes[nodeid].tasks) 
                && tasks[k].status > 0)
        {
            tasks[k].count = 0;
            //check limit 
            if(nodes[nodeid].limit > 0  && tasks[k].nxpackets >= nodes[nodeid].limit) goto end;
            //check last over
            if(tasks[k].popid != tasks[k].over) 
            {
                tasks[k].popid = tasks[k].last;
            }
            else 
                tasks[k].last = tasks[k].popid;
            p = data;
            left = *len;
            while(left > Q_LEFT_LEN)
            {
                last = tasks[k].popid;
                if(nodes[nodeid].type == Q_NODE_DOCD || nodes[nodeid].type == Q_NODE_PARSERD)
                {
                    //if(tasks[k].popid == 0) tasks[k].popid = 1;
                    if((tasks[k].popid+1) < qindex->state->xpackettotal)
                    {
                        if(tasks[k].popid > 0) id = ++(tasks[k].popid);
                        else id = tasks[k].popid = 1;
                        if(id > qindex->state->docpopid) 
                            qindex->state->docpopid = id;
                        if(id > nodes[nodeid].total) 
                            nodes[nodeid].total = id;
                    }
                    else 
                    {
                        id = -1;
                        break;
                    }
                }
                else if(nodes[nodeid].type == Q_NODE_INDEXD)
                {
                    if((x = tasks[k].popid) == nodes[nodeid].last)
                    {
                        if(qindex->state->popid == 0) qindex->state->popid = 1;
                        if((id = qindex->state->popid) > 0 && id <= qindex->state->docpopid)
                        {
                            if(nodes[nodeid].total == 0) 
                            {
                                nodes[nodeid].last = nodes[nodeid].first = id;
                            }
                            else
                            {
                                xpackets[x].next = id;
                                xpackets[id].prev = x;
                            }
                            xpackets[id].nodeid = nodeid;
                            //xpackets[id].status = 1;
                            tasks[k].popid = id;
                            nodes[nodeid].last = id;
                            nodes[nodeid].total++;
                            qindex->state->popid++;
                        }
                        else id = -1;
                    }
                    else
                    {
                        if(nodes[nodeid].total > 0 && tasks[k].popid < nodes[nodeid].last)
                        {
                            if(tasks[k].nxpackets == 0 && tasks[k].count == 0)
                            {
                                tasks[k].popid = id = nodes[nodeid].first;
                            }
                            else
                            {
                                x = tasks[k].popid;
                                id = xpackets[x].next;
                                tasks[k].popid = id;
                            }
                        }
                        else id = -1;
                    }
                }
                else id = -1;
                if(id > 0)
                {
                    if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[id].nodeid != nodeid)
                    {
                        FATAL_LOGGER(qindex->logger, "Invalid xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d", id, xpackets[id].nodeid, tasks[k].ip, tasks[k].port, nodeid);
                        _exit(-1);
                    }
                    if((n = db_get_data_len(PDB(qindex->db), id)) <= (left - Q_LEFT_LEN))
                    {
                        px = (int *)p;
                        p += sizeof(int);
                        docheader = (DOCHEADER *)p;
                        if((n = db_read_data(PDB(qindex->db), id, p)) > sizeof(DOCHEADER)) 
                        {
                            if(docheader->size < 0 || docheader->size != n) 
                            {
                                FATAL_LOGGER(qindex->logger, "Invalid data id:%d size:%d n:%d", id, docheader->size, n);
                                break;
                            }
                            //docheader->globalid = id;
                            *px = n;
                            left -= n + sizeof(int);
                            p += n;
                            tasks[k].count++;
                        }
                        else
                        {
                            p -= sizeof(int);
                            FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
                            //_exit(-1);
                            break;
                        }
                        last = tasks[k].popid;
                    }
                    else 
                    {
                        tasks[k].popid = last;
                        break;
                    }
                }
                else 
                {
                    break;
                }
            }
end:
            *len -= left;
            if((*count = tasks[k].count) > 0) 
            {
                id = tasks[k].popid = last ;
            }
            else id = -1;
        }
        else id = -2;
        MUTEX_UNLOCK(qindex->mutex);
    }
    return id;
}
Exemplo n.º 16
0
/* working for get_data() and push_index() */
int qindex_work(QINDEX *qindex, MINDEX *mindex)
{
    int ret = -1, i = 0, x = 0 , n = 0, mid = 0;
    int64_t *xid = NULL, key = 0, vlong = 0;
    BELEMENT *root = NULL, *sub = NULL;
    DOCHEADER *docheader = NULL;
    STERM *termlist = NULL;
    IBDATA block = {0};
    char *term = NULL;
    BRES *res = NULL;


    if(qindex && mindex && mindex_check(mindex) == 0)
    {
        if(mtask_pop(&(mindex->mtask)) > 0 && (xid = (int64_t *)mindex->mtask.packet)
                && mindex->mtask.length > sizeof(int64_t))
        {
            n = mindex->mtask.length/sizeof(int64_t);
            brequest_reset(&(mindex->request));
            brequest_append_keys(&(mindex->request), xid, n);
            brequest_finish(&(mindex->request));
            if((res = dbase_get_records(&(mindex->db), &(mindex->request))))
            {
                x = 0;
                while((root = dbase_next_record(res, root, &key))) 
                {
                    ACCESS_LOGGER(qindex->logger, "index_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
                    vlong = 0;
                    if((sub = belement_find(root, qindex->db_key_name)))
                    {
                        belement_v_long(sub, &vlong);
                    }
                    if(key != vlong)
                    {
                        ACCESS_LOGGER(qindex->logger, "Invalid record[%d/%d] packetid:%d key:%lld/%lld", x, n, mindex->mtask.packetid, LLI(vlong), LLI(key));
                        return ret;
                    }
                    if((sub = belement_find(root, qindex->db_index_block_name))
                            && (block.ndata = belement_v_blob(sub, &(block.data))) > 0)
                    {
                        docheader = (DOCHEADER *)block.data;
                        termlist = (STERM *)(block.data + sizeof(DOCHEADER)
                                + docheader->nfields * sizeof(XFIELD));
                        term = block.data + docheader->textblock_off;
                        for(i = 0; i < docheader->nterms; i++)
                        {
                            termlist[i].termid = mmtrie_xadd(qindex->xdict, term, termlist[i].term_len); 
                            term += termlist[i].term_len;
                        }
                        mid = qindex_push_index(qindex, key, &block);
                        ACCESS_LOGGER(qindex->logger, "over_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
                    }
                    else
                    {
                        FATAL_LOGGER(qindex->logger, "Invalid data feilds[%s]",  qindex->db_index_block_name);
                        return ret;
                    }
                    x++;
                }
                ret = 0;
                mtask_finish(&(mindex->mtask), 0);
            }
            else
            {
                FATAL_LOGGER(qindex->logger, "get_records(%d) failed, %s", mindex->mtask.packetid, strerror(errno));
                return ret;
            }
        }
    }
    return ret;
}