/* add task */
int qindex_add_task(QINDEX *qindex, int nodeid, char *ip, int port)
{
int taskid = -1, i = 0, n = 0;
char taskstr[Q_LINE_MAX];
QTNODE *nodes = NULL;
QTASK *tasks = NULL;
if(qindex && ip && port > 0 && nodeid < Q_NODE_MAX
&& (nodes = qindex->state->nodes) && nodes[nodeid].status > 0
&& nodes[nodeid].ntasks < Q_TASKS_MAX
&& (n = sprintf(taskstr, "%d:%s:%d", nodes[nodeid].type, ip, port)) > 0
&& (taskid = (mmtrie_get(qindex->namemap, taskstr, n) - 1)) < 0
&& (tasks = nodes[nodeid].tasks))
{
MUTEX_LOCK(qindex->mutex);
for(i = 0; i < Q_TASKS_MAX; i++)
{
if(tasks[i].status == 0)
{
tasks[i].status = 1;
strcpy(tasks[i].ip, ip);
tasks[i].port = port;
taskid = nodeid * Q_TASKS_MAX + i;
mmtrie_add(qindex->namemap, taskstr, n, taskid+1);
//fprintf(stdout, "%s::%d nodeid:%d taskid:%d\n", __FILE__, __LINE__, nodeid, taskid);
iqueue_push(qindex->queue, taskid);
nodes[nodeid].ntasks++;
tasks[i].mmqid = mmqueue_new(MMQ(qindex->mmqueue));
break;
}
}
MUTEX_UNLOCK(qindex->mutex);
}
return taskid;
}
/* push task to queue */
int qindex_push_task(QINDEX *qindex, int taskid)
{
int id = -1, nodeid = -1;
QTASK *tasks = NULL;
if(qindex && qindex->queue && taskid >= 0 && taskid < (Q_NODE_MAX * Q_TASKS_MAX)
&& (nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX)
{
MUTEX_LOCK(qindex->mutex);
id = taskid % Q_TASKS_MAX;
if((tasks = qindex->state->nodes[nodeid].tasks)
&& tasks[id].status > 0)
{
iqueue_push(qindex->queue, taskid);
id = taskid;
}
MUTEX_UNLOCK(qindex->mutex);
}
return id;
}
static void queue_test_basic_operation(iobject* queue) {
iqueue_clear(queue);
dbg_assert(iqueue_size(queue) == 0);
dbg_assert(iqueue_empty(queue));
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)2);
iqueue_push(queue, (void*)(intptr_t)3);
iqueue_push(queue, (void*)(intptr_t)4);
iqueue_push(queue, (void*)(intptr_t)5);
iqueue_push(queue, (void*)(intptr_t)6);
iqueue_push(queue, (void*)(intptr_t)7);
iqueue_push(queue, (void*)(intptr_t)8);
/* now the queue contains { 1, 2, 3, 4, 5, 6, 7, 8 } */
/* test pop */
dbg_assert(iqueue_size(queue) == 8);
{
/* now the queue contains { 1, 2, 3, 4, 5, 6, 7, 8 } */
intptr_t counter = 1;
intptr_t element;
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)1);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)8);
for (counter = 1; counter <= 8; counter ++) {
element = (intptr_t)iqueue_pop(queue);
dbg_assert(element == counter);
}
}
/* test intersect push and pop */
{
intptr_t element = 0;
dbg_assert(iqueue_empty(queue));
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)2);
/* push two element, now is { 1, 2 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 2);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)1);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)2);
element = (intptr_t)iqueue_pop(queue);
/* remove the front one, now is { 2 } */
dbg_assert(element == 1);
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 1);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)2);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)2);
iqueue_push(queue, (void*)(intptr_t)3);
iqueue_push(queue, (void*)(intptr_t)4);
/* push another 2 element, now is { 2, 3, 4 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 3);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)2);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)4);
/* continous pop all of them */
element = (intptr_t)iqueue_pop(queue);
/* remove the front one, now is { 3, 4 } */
dbg_assert(element == 2);
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 2);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)3);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)4);
element = (intptr_t)iqueue_pop(queue);
/* remove the front one, now is { 4 } */
dbg_assert(element == 3);
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 1);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)4);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)4);
element = (intptr_t)iqueue_pop(queue);
/* remove the front one, now is {} */
dbg_assert(element == 4);
dbg_assert(iqueue_empty(queue) == true);
dbg_assert(iqueue_size (queue) == 0);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)NULL);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)NULL);
/* trying to pop empty queue */
element = (intptr_t)iqueue_pop(queue);
dbg_assert(element == (intptr_t)NULL);
dbg_assert(iqueue_empty(queue) == true);
dbg_assert(iqueue_size (queue) == 0);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)NULL);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)NULL);
/* trying to push an element after pop empty queue */
iqueue_push(queue, (void*)(intptr_t)1);
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 1);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)1);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)1);
}
/* test unique element */
{
intptr_t element = 0;
iqueue_clear(queue);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
/* now the queue is { 1, 1, 1 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 3);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)1);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)1);
element = (intptr_t)iqueue_pop(queue);
/* remove the front one, now is { 1, 1 } */
dbg_assert(element == 1);
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 2);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)1);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
/* now the queue is { 1, 1, 1 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 3);
dbg_assert(iqueue_front(queue) == (void*)(intptr_t)1);
dbg_assert(iqueue_back (queue) == (void*)(intptr_t)1);
iqueue_clear(queue);
}
}
static void queue_test_basic_itr_operation(iobject* queue) {
iqueue_clear(queue);
dbg_assert(iqueue_size(queue) == 0);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)2);
iqueue_push(queue, (void*)(intptr_t)3);
iqueue_push(queue, (void*)(intptr_t)4);
/* now the queue contains { 1, 2, 3, 4 } */
dbg_assert(iqueue_size(queue) == 4);
{
iterator itr = iqueue_itr_create(queue, itr_begin);
const_iterator end = iqueue_itr_end(queue);
intptr_t current = 1;
/* traverse the queue */
for (; !itr_equals(itr, end); itr_to_next(itr)) {
dbg_assert(itr_get_ref(itr) == (void*)current);
current ++;
}
/* test itr_assign */
iqueue_itr_assign(queue, itr, itr_begin);
dbg_assert(itr_get_ref(itr) == (void*)(intptr_t)1);
iqueue_itr_assign(queue, itr, itr_end);
itr_to_prev(itr);
dbg_assert(itr_get_ref(itr) == (void*)(intptr_t)4);
itr_destroy(itr);
}
iqueue_clear(queue);
dbg_assert(iqueue_size(queue) == 0);
/* test unique data */
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
iqueue_push(queue, (void*)(intptr_t)1);
/* now the queue contains eight 1s, { 1, 1, 1, 1, 1, 1, 1, 1 } */
dbg_assert(iqueue_size(queue) == 8);
{
iterator itr = iqueue_itr_create(queue, itr_begin);
const_iterator end = iqueue_itr_end(queue);
/* traverse the queue */
for (; !itr_equals(itr, end); itr_to_next(itr)) {
dbg_assert(itr_get_ref(itr) == (void*)1);
}
itr_destroy(itr);
}
return;
}
static void queue_test_basic_itr_operation(_object* queue, struct test_data_desc* td_desc) {
address *test_data_addr = td_desc->data_repo;
unknown_traits *td_traits = td_desc->data_traits;
iqueue_clear(queue);
dbg_assert(iqueue_size(queue) == 0);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[2]);
iqueue_push(queue, test_data_addr[3]);
iqueue_push(queue, test_data_addr[4]);
/* now the queue contains { 1, 2, 3, 4 } */
dbg_assert(iqueue_size(queue) == 4);
{
iterator itr = iqueue_itr_create(queue, itr_begin);
const_iterator end = iqueue_itr_end(queue);
intptr_t current = 1;
const unknown* x = NULL;
/* traverse the queue */
for (; !itr_equals(itr, end); itr_to_next(itr)) {
x = itr_get_ref(itr);
dbg_assert(td_traits->__equals(x, test_data_addr[current]));
current ++;
}
/* test itr_assign */
iqueue_itr_assign(queue, itr, itr_begin);
x = (int*)itr_get_ref(itr);
dbg_assert(td_traits->__equals(x, test_data_addr[1]));
iqueue_itr_assign(queue, itr, itr_end);
itr_to_prev(itr);
x = (int*)itr_get_ref(itr);
dbg_assert(td_traits->__equals(x, test_data_addr[4]));
itr_destroy(itr);
}
iqueue_clear(queue);
dbg_assert(iqueue_size(queue) == 0);
/* test unique data */
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
/* now the queue contains eight 1s, { 1, 1, 1, 1, 1, 1, 1, 1 } */
dbg_assert(iqueue_size(queue) == 8);
{
iterator itr = iqueue_itr_create(queue, itr_begin);
const_iterator end = iqueue_itr_end(queue);
/* traverse the queue */
for (; !itr_equals(itr, end); itr_to_next(itr)) {
unknown* x = (int*)itr_get_ref(itr);
dbg_assert(td_traits->__equals(x, test_data_addr[1]));
}
itr_destroy(itr);
}
return;
}
static void queue_test_basic_operation(_object* queue, struct test_data_desc* td_desc) {
address *test_data_addr = td_desc->data_repo;
unknown_traits *td_traits = td_desc->data_traits;
iqueue_clear(queue);
dbg_assert(iqueue_size(queue) == 0);
dbg_assert(iqueue_empty(queue));
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[2]);
iqueue_push(queue, test_data_addr[3]);
iqueue_push(queue, test_data_addr[4]);
iqueue_push(queue, test_data_addr[5]);
iqueue_push(queue, test_data_addr[6]);
iqueue_push(queue, test_data_addr[7]);
iqueue_push(queue, test_data_addr[8]);
/* now the queue contains { 1, 2, 3, 4, 5, 6, 7, 8 } */
/* test pop */
dbg_assert(iqueue_size(queue) == 8);
{
/* now the queue contains { 1, 2, 3, 4, 5, 6, 7, 8 } */
intptr_t counter = 1;
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[1]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[8]));
for (counter = 1; counter <= 8; counter ++) {
iqueue_pop(queue);
}
}
/* test intersect push and pop */
{
dbg_assert(iqueue_empty(queue));
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[2]);
/* push two element, now is { 1, 2 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 2);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[1]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[2]));
iqueue_pop(queue);
/* remove the front one, now is { 2 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 1);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[2]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[2]));
iqueue_push(queue, test_data_addr[3]);
iqueue_push(queue, test_data_addr[4]);
/* push another 2 element, now is { 2, 3, 4 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 3);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[2]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[4]));
/* continuous pop all of them */
iqueue_pop(queue);
/* remove the front one, now is { 3, 4 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 2);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[3]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[4]));
iqueue_pop(queue);
/* remove the front one, now is { 4 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 1);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[4]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[4]));
iqueue_pop(queue);
/* remove the front one, now is {} */
dbg_assert(iqueue_empty(queue) == true);
dbg_assert(iqueue_size (queue) == 0);
dbg_assert(iqueue_front(queue) == NULL);
dbg_assert(iqueue_back (queue) == NULL);
/* trying to pop empty queue */
iqueue_pop(queue);
dbg_assert(iqueue_empty(queue) == true);
dbg_assert(iqueue_size (queue) == 0);
dbg_assert(iqueue_front(queue) == NULL);
dbg_assert(iqueue_back (queue) == NULL);
/* trying to push an element after pop empty queue */
iqueue_push(queue, test_data_addr[1]);
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 1);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[1]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[1]));
}
/* test unique element */
{
iqueue_clear(queue);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
iqueue_push(queue, test_data_addr[1]);
/* now the queue is { 1, 1, 1 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 3);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[1]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[1]));
iqueue_pop(queue);
/* remove the front one, now is { 1, 1 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 2);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[1]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[1]));
iqueue_push(queue, test_data_addr[1]);
/* now the queue is { 1, 1, 1 } */
dbg_assert(iqueue_empty(queue) == false);
dbg_assert(iqueue_size (queue) == 3);
dbg_assert(td_traits->__equals(iqueue_front(queue), test_data_addr[1]));
dbg_assert(td_traits->__equals(iqueue_back (queue), test_data_addr[1]));
iqueue_clear(queue);
}
}
/* set document basedir */
int qindex_set_basedir(QINDEX *qindex, char *basedir)
{
int i = 0, j = 0, taskid = 0;
char path[Q_PATH_MAX];
QTASK *tasks = NULL;
struct stat st = {0};
off_t size = 0;
if(qindex)
{
strcpy(qindex->basedir, basedir);
/* log */
sprintf(path, "%s/%s", basedir, QINDEX_LOG_NAME);
pmkdir(path);
LOGGER_INIT(qindex->logger, path);
LOGGER_SET_LEVEL(qindex->logger, qindex->log_level);
/* xdict */
sprintf(path, "%s/%s", basedir, QINDEX_XDICT_NAME);
if((qindex->xdict = mmtrie_init(path)) == NULL)
{
fprintf(stderr, "Initialize xdict(%s) failed, %s\n", path, strerror(errno));
_exit(-1);
return -1;
}
/* mmqueue */
sprintf(path, "%s/%s", basedir, QINDEX_MMQUEUE_NAME);
qindex->mmqueue = mmqueue_init(path);
/* db */
sprintf(path, "%s/%s", basedir, QINDEX_DB_DIR);
qindex->db = db_init(path, 0);
/* update */
sprintf(path, "%s/%s", basedir, QINDEX_UPDATE_DIR);
qindex->update = db_init(path, 0);
/* qstate */
sprintf(path, "%s/%s", basedir, QINDEX_STATE_NAME);
if((qindex->qstatefd = open(path, O_CREAT|O_RDWR, 0644)) <= 0)
{
fprintf(stderr, "open state file(%s) failed, %s\n", path, strerror(errno));
_exit(-1);
}
else
{
fstat(qindex->qstatefd, &st);
if(st.st_size < sizeof(QSTATE))
{
if(ftruncate(qindex->qstatefd, 0) != 0
|| ftruncate(qindex->qstatefd, sizeof(QSTATE)) != 0)_exit(-1);
}
if((qindex->state = (QSTATE *)mmap(NULL, sizeof(QSTATE), PROT_READ|PROT_WRITE,
MAP_SHARED, qindex->qstatefd, 0)) == NULL || qindex->state == (void *)-1)
{
fprintf(stderr, "mmap state failed, %s\n", strerror(errno));
_exit(-1);
}
if(st.st_size < sizeof(QSTATE)) memset(qindex->state, 0, sizeof(QSTATE));
if(qindex->state->nnodes > 0)
{
for(i = 1; i < Q_NODE_MAX; i++)
{
if(qindex->state->nodes[i].status > 0 && qindex->state->nodes[i].ntasks > 0
&& (tasks = qindex->state->nodes[i].tasks))
{
for(j = 0; j < Q_TASKS_MAX; j++)
{
if(tasks[j].status > 0)
{
taskid = i * Q_TASKS_MAX + j;
iqueue_push(qindex->queue, taskid);
}
}
}
}
}
}
/* xpacket */
sprintf(path, "%s/%s", basedir, QINDEX_XPACKET_NAME);
if((qindex->xpacketio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
&& fstat(qindex->xpacketio.fd, &st) == 0)
{
qindex->xpacketio.end = st.st_size;
size = sizeof(XPACKET) * Q_XPACKET_MAX;
if(st.st_size > size) size = st.st_size;
if((qindex->xpacketio.map = (char *)mmap(NULL, size, PROT_READ|PROT_WRITE,
MAP_SHARED, qindex->xpacketio.fd, 0))
&& qindex->xpacketio.map != (void *)-1)
{
qindex->xpacketio.size = size;
}
else
{
fprintf(stderr, "mmap xpacket file (%s) failed, %s\n", path, strerror(errno));
_exit(-1);
}
}
else
{
fprintf(stderr, "open xpacket file(%s) failed, %s\n", path, strerror(errno));
_exit(-1);
}
/* block terms map */
sprintf(path, "%s/%s", basedir, QINDEX_MAP_NAME);
qindex->map = mmtrie_init(path);
/* bsterm file */
sprintf(path, "%s/%s", basedir, QINDEX_BSTERM_NAME);
if((qindex->bstermio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
&& fstat(qindex->bstermio.fd, &st) == 0)
{
qindex->bstermio.end = st.st_size;
size = sizeof(BSTERM) * Q_BSTERM_MAX;
if(st.st_size > size) size = st.st_size;
if((qindex->bstermio.map = (char *)mmap(NULL, size, PROT_READ|PROT_WRITE,
MAP_SHARED, qindex->bstermio.fd, 0))
&& qindex->bstermio.map != (void *)-1)
{
qindex->bstermio.size = size;
}
else
{
fprintf(stderr, "mmap bsterm file (%s) failed, %s\n", path, strerror(errno));
_exit(-1);
}
}
else
{
fprintf(stderr, "open bsterm file(%s) failed, %s\n", path, strerror(errno));
_exit(-1);
}
/* name map */
sprintf(path, "%s/%s", basedir, QINDEX_NODEMAP_NAME);
qindex->namemap = mmtrie_init(path);
/* id map */
sprintf(path, "%s/%s", basedir, QINDEX_IDMAP_NAME);
qindex->idmap = mmtree64_init(path);
if(qindex->state->rootid == 0)
{
qindex->state->rootid = mmtree64_new_tree(qindex->idmap);
}
return 0;
}
return -1;
}
