int
main(int argc, char *argv[])
{
ps_status_t rv = PS_SUCCESS;
int log_level = LOG_LEVEL_NONE;
char *search = NULL, *path = NULL;
size_t search_len = 0;
int i = 0, c = 0;
int number_of_procs = 0, own_rank = 0;
int *slave_nodes = NULL;
unsigned long chunk_size = DEFAULT_CHUNK_SIZE;
ps_searcher_t *searcher = NULL;
ps_search_task_t *task = NULL;
char *result = NULL;
size_t result_len = 0, total_result_len = 0, *all_result_len = NULL;
int search_col = PS_CSV_ALL_COL;
#ifdef TIME_MEASUREMENT
float total_seconds = 0, total_search_time = 0, total_file_io_time = 0, total_setup_time = 0, total_reduce_time = 0;
struct timeval time_start, current_time;
gettimeofday(&time_start, NULL);
memcpy(¤t_time, &time_start, sizeof(struct timeval));
#endif
out_fd = stdout; /*For Logging*/
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &number_of_procs);
MPI_Comm_rank (MPI_COMM_WORLD, &own_rank);
/*set log level until arguments are passed and processed*/
set_log_level(log_level);
if (own_rank == MASTER)
{
opterr = 0;
while ((c = getopt (argc, argv, "hds:f:c:l:")) != -1)
{
switch (c)
{
case 'd':
log_level = LOG_LEVEL_DEBUG;
break;
case 'f':
path = optarg;
break;
case 's':
search = optarg;
search_len = strlen(search);
break;
case 'c':
PS_CHECK_ZERO_GO_ERR( (chunk_size = atol(optarg)), PS_ERROR_WRONG_CHUNK_SIZE);
break;
case 'l':
search_col = atoi(optarg);
break;
case 'h':
break;
default:
if (optopt == 's')
fprintf (stderr, "Option -%c requires an argument.\n", optopt);
else if (isprint (optopt))
fprintf (stderr, "Unknown option `-%c'.\n", optopt);
else
fprintf (stderr,
"Unknown option character `\\x%x'.\n", optopt);
rv = PS_ERROR_WRONG_ARGUMENTS;
goto error;
}
}
}
/*Communicate and set log_level*/
PS_MPI_CHECK_ERR(MPI_Bcast(&log_level, 1, MPI_INT, MASTER, MPI_COMM_WORLD));
set_log_level(log_level);
/*Communicate the token to search for*/
PS_MPI_CHECK_ERR(MPI_Bcast(&search_len, 1, MPI_UNSIGNED_LONG, MASTER, MPI_COMM_WORLD));
if (own_rank != MASTER)
{
PS_MALLOC(search, sizeof(char) * (search_len + 1));
}
PS_MPI_CHECK_ERR(MPI_Bcast(search, search_len + 1, MPI_CHAR, MASTER, MPI_COMM_WORLD));
log_debug("Process %d: search_len:%u search:%s", own_rank, search_len, search);
#ifdef TIME_MEASUREMENT
if(own_rank == MASTER)
{
update_timestamp_and_total_seconds(¤t_time, &total_setup_time);
}
#endif
if (own_rank == MASTER)
{
log_debug("search = %s, path = %s, chunk_size=%lu", search, path, chunk_size);
log_debug("sizeof(ps_search_task_t:%lu", sizeof(ps_search_task_t));
for (i = optind; i < argc; i++)
{
log_debug("Non-option argument %s", argv[i]);
}
/*-------------------Processing of arguments done!-----------*/
log_debug("Number of procs:%d", number_of_procs);
PS_MALLOC( slave_nodes, sizeof(int) * (number_of_procs));
for (i = 0; i < number_of_procs - 1; i++)
{
slave_nodes[i] = i + 1;
}
PS_CHECK_GOTO_ERROR( distribute_path_and_search_range(path,
number_of_procs ,
slave_nodes,
chunk_size,
search_col,
MPI_COMM_WORLD,
&task));
/*Slaves receive path_length and search_task*/
PS_CHECK_GOTO_ERROR(ps_file_searcher_create(&searcher, search, task));
PS_CHECK_GOTO_ERROR(ps_file_searcher_search(searcher, &result, &result_len));
log_debug("Process %d: result_len: %lu", own_rank, result_len);
PS_CHECK_GOTO_ERROR(ps_file_searcher_free(&searcher));
PS_MALLOC(all_result_len, sizeof(size_t) * number_of_procs);
}
else
{
/*Slaves receive path_length and search_task*/
PS_CHECK_GOTO_ERROR(recv_task(&task, own_rank, MASTER, MPI_COMM_WORLD));
PS_CHECK_GOTO_ERROR(ps_file_searcher_create(&searcher, search, task));
PS_CHECK_GOTO_ERROR(ps_file_searcher_search(searcher, &result, &result_len));
log_debug("Process %d: result_len: %lu", own_rank, result_len);
PS_CHECK_GOTO_ERROR(ps_file_searcher_free(&searcher));
}
#ifdef TIME_MEASUREMENT
if(own_rank == MASTER)
{
gettimeofday(¤t_time, NULL);
}
#endif
PS_MPI_CHECK_ERR(MPI_Gather(&result_len, 1, MPI_UNSIGNED_LONG, all_result_len, 1, MPI_UNSIGNED_LONG, MASTER,
MPI_COMM_WORLD));
if(own_rank == MASTER)
{
for(i = 0; i < number_of_procs; i++){
total_result_len += all_result_len[i];
}
log_debug("Process %d: total_result_len:%lu", own_rank, total_result_len);
PS_REALLOC(result, total_result_len);
for(i = 1; i < number_of_procs; i++){
PS_MPI_CHECK_ERR(MPI_Recv(result + all_result_len[i - 1], all_result_len[i], MPI_CHAR,
i, PS_MPI_TAG_RESULT, MPI_COMM_WORLD, MPI_STATUSES_IGNORE));
result_len += all_result_len[i];
}
write(STDOUT_FILENO, result, result_len);
}
else
{
PS_MPI_CHECK_ERR(MPI_Send(result, result_len, MPI_CHAR, MASTER, PS_MPI_TAG_RESULT, MPI_COMM_WORLD));
PS_FREE(search);
}
log_debug("Process %d finished", own_rank);
#ifdef TIME_MEASUREMENT
if(own_rank == MASTER)
{
update_timestamp_and_total_seconds(¤t_time, &total_reduce_time);
}
printf("Process %d: process_search_time: %f, process_file_io_time: %f\n", own_rank
,process_search_time, process_file_io_time);
PS_MPI_CHECK_ERR(MPI_Reduce(&process_search_time, &total_search_time, 1, MPI_FLOAT, MPI_SUM, MASTER,
MPI_COMM_WORLD));
PS_MPI_CHECK_ERR(MPI_Reduce(&process_file_io_time , &total_file_io_time, 1, MPI_FLOAT, MPI_SUM, MASTER,
MPI_COMM_WORLD));
update_timestamp_and_total_seconds(&time_start, &total_seconds);
if(own_rank == MASTER)
{
printf("Total-Time: %.3fs\n"
"\ttotal_setup_time: %.3fs\n"
"\ttotal_reduce_time: %.3fs\n"
"\taverage-io-time: %.3fs\n"
"\taverage-search-time: %.3fs\n"
"processes: %d\n"
"chunksize: %lu Bytes\n",
total_seconds, total_setup_time, total_reduce_time,
total_file_io_time / number_of_procs, total_search_time / number_of_procs,
number_of_procs, chunk_size);
}
#endif
MPI_Finalize();
PS_FREE(slave_nodes);
PS_FREE(result);
PS_FREE(all_result_len);
return EXIT_SUCCESS;
/*-----------------ERROR-Handling------------------------------*/
error:
log_err("Process %d finished with error: %d", own_rank, rv);
MPI_Finalize();
if (own_rank != MASTER)
{
PS_FREE(search);
}
PS_FREE(slave_nodes);
if(searcher)
{
PS_FREE(searcher->task);
}
PS_FREE(result);
PS_FREE(all_result_len);
return rv;
}
void *sync_src(void *arg)
{
sync_thread_arg_t *tmparg = arg;
int sock_fd = tmparg->sock_fd;
io_sync_ctrl_block_t *dcp = tmparg->dcp;
int ret;
//iterate berkeley db, retrieving each record in turn.
struct list_head key_head;
struct hashtable *key_hash;
DBC *cursorp;
DBT dbt_key, dbt_data;
dcp->dbp->cursor(dcp->dbp, NULL, &cursorp, 0);
memset(&dbt_key, 0, sizeof(DBT));
memset(&dbt_data, 0, sizeof(DBT));
pthread_mutex_lock(&(dcp->io_lock));
key_hash = create_hashtable(10000, hash_from_key_fn, keys_equal_fn);
INIT_LIST_HEAD(&key_head);
while ((ret = cursorp->c_get(cursorp, &dbt_key, &dbt_data, DB_NEXT)) == 0) {
key_list_t *tmp = (key_list_t *)malloc(sizeof(key_list_t));
memcpy(tmp->key, dbt_key.data, KEY_SIZE);
list_add_tail(&(tmp->list_node), &key_head);
if (!hashtable_insert(key_hash, tmp->key, tmp)) {
perror("hash insert error!");
return NULL;
}
}
//close
if (cursorp != NULL)
cursorp->c_close(cursorp);
pthread_mutex_unlock(&(dcp->io_lock));
unsigned int max_packet_size = sizeof(msg_sync_t) + SECTION_SIZE;
unsigned char *sock_buf = (unsigned char *)malloc(max_packet_size);
DBT key, data;
MD5_CTX md5_ctx;
unsigned char tmp[BLOCK_INDEX_SIZE];
msg_sync_t msg;
unsigned int bi, fi = 0;
off_t off1 = 0, off2 = 0;
key_list_t *valuep;
void *buf;
if (posix_memalign(&buf, 512, SECTION_SIZE) != 0) {
perror("posix_memalign error");
return NULL;
}
int is_data_end = 0, is_block_end = 1;
while (!is_data_end) {
if (recv_task(sock_fd, sock_buf, max_packet_size) <= 0) {
perror("rece_task error");
return NULL;
}
msg = *((msg_sync_t *)sock_buf);
unsigned char *sock_data;
switch (msg.type) {
//recv section md5 code
case SYNC_SECTION_MD5:
if (is_block_end || off2 >= BLOCK_SIZE) {
perror("wrong section msg");
return NULL;
}
sock_data = sock_buf + sizeof(msg_sync_t);
//read src section data, and calc md5
if (lseek(dcp->fd[fi], off1 + off2, SEEK_SET) == (off_t)-1) {
perror("seek error");
return NULL;
}
if (read(dcp->fd[fi], buf, SECTION_SIZE) != SECTION_SIZE) {
perror("seek error");
return NULL;
}
MD5Calc(&md5_ctx, (unsigned char *)buf, SECTION_SIZE);
#ifdef _SYNC_SRC_DEBUG__
unsigned int kk;
for (kk = 0; kk < 16; ++kk)
printf("%02X", md5_ctx.digest[kk]);
printf("\n");
#endif
off2 += SECTION_SIZE;
// compare to dest md5
if (memcmp(md5_ctx.digest, sock_data, 16) == 0) {
SEND_MSG(0, buf, SYNC_SECTION_SAME, "Can't send msg to client");
} else {
SEND_MSG(SECTION_SIZE, buf, SYNC_SECTION_DATA, "Can't send msg to client");
}
break;
case SYNC_BLOCK_KEY:
sock_data = sock_buf + sizeof(msg_sync_t);
#ifdef _SYNC_SRC_DEBUG__
printf("%u %u ", msg.size, msg.type);
int jj;
for (jj = 0; jj < KEY_SIZE; ++jj) {
printf("%u ", sock_data[jj]);
}
printf("\n");
#endif
if ((valuep = hashtable_remove(key_hash, sock_data)) != NULL) {
list_del(&(valuep->list_node));
free(valuep);
}
//get block index by key
memset(&key, 0, sizeof(DBT));
key.data = sock_data;
key.size = KEY_SIZE;
memset(&data, 0, sizeof(DBT));
data.data = tmp;
data.ulen = BLOCK_INDEX_SIZE;
data.flags = DB_DBT_USERMEM;
ret = dcp->dbp->get(dcp->dbp, NULL, &key, &data, 0);
if (ret == DB_NOTFOUND) {
//reply SYNC_BLOCK_DELETE
SEND_MSG(0, buf, SYNC_BLOCK_DELETE, "Can't send msg to client");
} else if (ret) {
printf("dbp->get error - %d\n", ret);
perror("dbp->get error");
return NULL;
} else {
//reply SYNC_BLOCK_CONTINUE
bi = *((unsigned int *)tmp);
fi = *(unsigned int *)(tmp + 4);
off1 = bi;
off1 = off1 * BLOCK_SIZE;
off2 = 0;
is_block_end = 0;
SEND_MSG(0, buf, SYNC_BLOCK_CONTINUE,
"log_deliver send error -can't send msg to client");
}
break;
case SYNC_BLOCK_END:
if (off2 != BLOCK_SIZE) {
perror("wrong section");
return NULL;
}
is_block_end = 1;
SEND_MSG(0, buf, SYNC_OK, "Can't send msg to client");
break;
case SYNC_DATA_END:
is_data_end = 1;
break;
default:
perror("unknown sync msg code");
return NULL;
}
}
//destroy hashtable but not free key and value
hashtable_destroy(key_hash, 2);
key_list_t *v;
while (!list_empty(&key_head)) {
v = container_of(key_head.next, key_list_t , list_node);
#ifdef _SYNC_SRC_H__
int kk;
for (kk = 0; kk < 8; ++kk) printf("%u ", v->key[kk]);
printf("\n");
#endif
memset(&key, 0, sizeof(DBT));
key.data = v->key;
key.size = KEY_SIZE;
memset(&data, 0, sizeof(DBT));
data.data = tmp;
data.ulen = BLOCK_INDEX_SIZE;
data.flags = DB_DBT_USERMEM;
ret = dcp->dbp->get(dcp->dbp, NULL, &key, &data, 0);
if (ret == DB_NOTFOUND) {
//do something here
//key may be deleted
list_del(key_head.next);
continue;
}
bi = *(unsigned int *)tmp;
fi = *(unsigned int *)(tmp + 4);
//send keys
SEND_MSG(KEY_SIZE, v->key, SYNC_BLOCK_KEY, "Can't send msg to client");
//recv reply
RECV_MSG(SYNC_BLOCK_CONTINUE, "recv key_reply error", "return type error");
//new block, send section data
off1 = bi;
off1 = off1 * BLOCK_SIZE;
for (off2 = 0; off2 < BLOCK_SIZE; off2 += SECTION_SIZE) {
if (lseek(dcp->fd[fi], off1 + off2, SEEK_SET) == (off_t)-1) {
perror("seek error");
return NULL;
}
if (read(dcp->fd[fi], buf, SECTION_SIZE) != SECTION_SIZE) {
perror("read error");
return NULL;
}
//new block, send section data
SEND_MSG(SECTION_SIZE, buf, SYNC_SECTION_DATA, "send section data failed");
RECV_MSG(SYNC_OK, "recv section_reply error", "return type error");
}
//new block, send block end
SEND_MSG(0, buf, SYNC_BLOCK_END, "send block end failed");
RECV_MSG(SYNC_OK, "recv data_end_reply", "return type error");
list_del(key_head.next);
}
//new block, send data end
SEND_MSG(0, buf, SYNC_DATA_END, "send data end failed");
close(sock_fd);
free(sock_buf);
//EXIT:
return NULL;
}
