static void
check_sector(struct tfilter *filter, int type, int rw, uint64_t sec, char *buf)
{
if (sec >= filter->secs)
return;
if (rw) {
if (type == PRE_CHECK)
insert_hash(filter, sec, buf);
else
check_hash(filter, sec, buf, WRITE_INTEGRITY);
} else if (type == POST_CHECK) {
check_hash(filter, sec, buf, READ_INTEGRITY);
insert_hash(filter, sec, buf);
}
}
static void dissector_init_ether_types(void)
{
FILE *fp;
char buff[512], *ptr;
struct ether_type *et;
void **pos;
fp = fopen("/etc/netsniff-ng/ether.conf", "r");
if (!fp)
panic("No /etc/netsniff-ng/ether.conf found!\n");
memset(buff, 0, sizeof(buff));
while (fgets(buff, sizeof(buff), fp) != NULL) {
buff[sizeof(buff) - 1] = 0;
et = xmalloc(sizeof(*et));
ptr = buff;
ptr = skips(ptr);
ptr = getuint(ptr, &et->id);
ptr = skips(ptr);
ptr = skipchar(ptr, ',');
ptr = skips(ptr);
ptr = strtrim_right(ptr, '\n');
ptr = strtrim_right(ptr, ' ');
et->type = xstrdup(ptr);
et->next = NULL;
pos = insert_hash(et->id, et, ð_ether_types);
if (pos) {
et->next = *pos;
*pos = et;
}
memset(buff, 0, sizeof(buff));
}
fclose(fp);
}
void
exec_command_vni_create (char * str, int sock)
{
u_int8_t vnibuf[3];
char cmd[CONTROL_CMD_BUF_LEN];
char vni[CONTROL_CMD_BUF_LEN];
struct vxlan_instance * vins;
if (sscanf (str, "%s %s", cmd, vni) < 2) {
write (sock, CONTROL_ERRMSG, sizeof (CONTROL_ERRMSG));
return;
}
strtovni (vni, vnibuf);
if ((vins = search_hash (&vxlan.vins_tuple, vnibuf)) != NULL) {
char errbuf[] = "this vxlan instance exists\n";
write (sock, errbuf, strlen (errbuf));
return;
}
vins = create_vxlan_instance (vnibuf);
insert_hash (&vxlan.vins_tuple, vins, vnibuf);
init_vxlan_instance (vins);
vxlan.vins_num++;
char msgbuf[] = "created\n";
write (sock, msgbuf, strlen (msgbuf));
return;
}
int main()
{
int initial_num, query_num;
unsigned int roll_number = 0;
char name[26];
int i = 0;
node_p *hash_arry = malloc(MAX_LIMIT*sizeof(node_p *));
memset(hash_arry, 0, MAX_LIMIT*sizeof(node_p *));
scanf("%d", &initial_num);
for (i = 0; i < initial_num; i++) {
scanf("%u", &roll_number);
scanf("%s", name);
insert_hash(hash_arry, roll_number, name);
}
scanf("%d", &query_num);
for (i = 0; i < query_num; i++) {
scanf("%u", &roll_number);
search_hash(hash_arry, roll_number);
}
return 0;
}
static bool
add_user_into_multicast_group( struct in_addr maddr, unsigned int *n_users ) {
assert( multicast_groups != NULL );
assert( n_users != NULL );
if ( !IN_MULTICAST( ntohl( maddr.s_addr ) ) ) {
return false;
}
lock_multicast_group_table();
unsigned int *in_use = search_hash( &multicast_groups->groups, &maddr );
if ( in_use == NULL ) {
in_use = malloc( sizeof( unsigned int ) );
assert( in_use != NULL );
*in_use = 0;
insert_hash( &multicast_groups->groups, in_use, &maddr );
}
( *in_use )++;
*n_users = *in_use;
unlock_multicast_group_table();
return true;
}
static int register_user_by_sockaddr(struct sockaddr_storage *sa,
size_t sa_len,
struct curve25519_proto *proto)
{
void **pos;
struct sockaddr_map_entry *entry;
unsigned int hash = hash_name((char *) sa, sa_len);
rwlock_wr_lock(&sockaddr_map_lock);
entry = xzmalloc(sizeof(*entry));
entry->sa = xmemdupz(sa, sa_len);
entry->sa_len = sa_len;
entry->proto = proto;
pos = insert_hash(hash, entry, &sockaddr_mapper);
if (pos) {
entry->next = (*pos);
(*pos) = entry;
}
rwlock_unlock(&sockaddr_map_lock);
return 0;
}
static void add_to_known_names(const char *path,
const unsigned char *peeled,
int prio,
const unsigned char *sha1)
{
struct commit_name *e = find_commit_name(peeled);
struct tag *tag = NULL;
if (replace_name(e, prio, sha1, &tag)) {
if (!e) {
void **pos;
e = xmalloc(sizeof(struct commit_name));
hashcpy(e->peeled, peeled);
pos = insert_hash(hash_sha1(peeled), e, &names);
if (pos) {
e->next = *pos;
*pos = e;
} else {
e->next = NULL;
}
}
e->tag = tag;
e->prio = prio;
e->name_checked = 0;
hashcpy(e->sha1, sha1);
e->path = path;
}
}
static void dissector_init_oui(void)
{
FILE *fp;
char buff[512], *ptr;
struct vendor_id *ven;
void **pos;
fp = fopen("/etc/netsniff-ng/oui.conf", "r");
if (!fp)
panic("No /etc/netsniff-ng/oui.conf found!\n");
memset(buff, 0, sizeof(buff));
while (fgets(buff, sizeof(buff), fp) != NULL) {
buff[sizeof(buff) - 1] = 0;
ven = xmalloc(sizeof(*ven));
ptr = buff;
ptr = skips(ptr);
ptr = getuint(ptr, &ven->id);
ptr = skips(ptr);
ptr = skipchar(ptr, ',');
ptr = skips(ptr);
ptr = strtrim_right(ptr, '\n');
ptr = strtrim_right(ptr, ' ');
ven->vendor = xstrdup(ptr);
ven->next = NULL;
pos = insert_hash(ven->id, ven, ð_oui);
if (pos) {
ven->next = *pos;
*pos = ven;
}
memset(buff, 0, sizeof(buff));
}
fclose(fp);
}
static void dissector_init_ports_tcp(void)
{
FILE *fp;
char buff[512], *ptr;
struct port_tcp *ptcp;
void **pos;
fp = fopen("/etc/netsniff-ng/tcp.conf", "r");
if (!fp)
panic("No /etc/netsniff-ng/tcp.conf found!\n");
memset(buff, 0, sizeof(buff));
while (fgets(buff, sizeof(buff), fp) != NULL) {
buff[sizeof(buff) - 1] = 0;
ptcp = xmalloc(sizeof(*ptcp));
ptr = buff;
ptr = skips(ptr);
ptr = getuint(ptr, &ptcp->id);
ptr = skips(ptr);
ptr = skipchar(ptr, ',');
ptr = skips(ptr);
ptr = strtrim_right(ptr, '\n');
ptr = strtrim_right(ptr, ' ');
ptcp->port = xstrdup(ptr);
ptcp->next = NULL;
pos = insert_hash(ptcp->id, ptcp, ð_ports_tcp);
if (pos) {
ptcp->next = *pos;
*pos = ptcp;
}
memset(buff, 0, sizeof(buff));
}
fclose(fp);
}
static void *_hash_tbl_insert_item(QSP_ARG_DECL Container *cnt_p,Item *ip)
{
int stat;
stat=insert_hash(ip,cnt_p->cnt_htp);
if( stat == 0 )
return ip;
return NULL;
}
static void dissector_init_ports(enum ports which)
{
FILE *fp;
char buff[128], *ptr, *file;
struct hash_table *table;
struct port *p;
void **pos;
switch (which) {
case PORTS_UDP:
file = "/etc/netsniff-ng/udp.conf";
table = ð_ports_udp;
break;
case PORTS_TCP:
file = "/etc/netsniff-ng/tcp.conf";
table = ð_ports_tcp;
break;
case PORTS_ETHER:
file = "/etc/netsniff-ng/ether.conf";
table = ð_ether_types;
break;
default:
bug();
}
fp = fopen(file, "r");
if (!fp)
panic("No %s found!\n", file);
memset(buff, 0, sizeof(buff));
while (fgets(buff, sizeof(buff), fp) != NULL) {
buff[sizeof(buff) - 1] = 0;
ptr = buff;
p = xmalloc(sizeof(*p));
p->id = strtol(ptr, &ptr, 0);
if ((ptr = strstr(buff, ", ")))
ptr += strlen(", ");
ptr = strtrim_right(ptr, '\n');
ptr = strtrim_right(ptr, ' ');
p->port = xstrdup(ptr);
p->next = NULL;
pos = insert_hash(p->id, p, table);
if (pos) {
p->next = *pos;
*pos = p;
}
memset(buff, 0, sizeof(buff));
}
fclose(fp);
}
void dissector_init_oui(void)
{
FILE *fp;
char buff[128], *ptr, *end;
struct vendor_id *v;
void **pos;
if (initialized)
return;
fp = fopen(ETCDIRE_STRING "/oui.conf", "r");
if (!fp)
panic("No oui.conf found!\n");
memset(buff, 0, sizeof(buff));
while (fgets(buff, sizeof(buff), fp) != NULL) {
buff[sizeof(buff) - 1] = 0;
ptr = buff;
v = xmalloc(sizeof(*v));
v->id = strtol(ptr, &end, 0);
/* not a valid line, skip */
if (v->id == 0 && end == ptr) {
xfree(v);
continue;
}
ptr = strstr(buff, ", ");
/* likewise */
if (!ptr) {
xfree(v);
continue;
}
ptr += strlen(", ");
ptr = strtrim_right(ptr, '\n');
ptr = strtrim_right(ptr, ' ');
v->vendor = xstrdup(ptr);
v->next = NULL;
pos = insert_hash(v->id, v, &oui);
if (pos) {
v->next = *pos;
*pos = v;
}
memset(buff, 0, sizeof(buff));
}
fclose(fp);
initialized = true;
}
//checks a specific move from specific location on the board. If move viable and doesn't lead to duplicate board,
//new board shows this move and 1 is returned. If no moves found, returns 0
int check_move (movement direction, int i, int j, hash *hash_array, Node *start, Node *latest_node, int parent_board [BOARD_HEIGHT][BOARD_WIDTH], int new_board[BOARD_HEIGHT][BOARD_WIDTH]){
if (valid_move (direction, i, j, new_board)){
move_piece (direction, i, j, new_board);
latest_node->identifier = allocate_board_identifier (new_board);
if (insert_hash (hash_array, latest_node, latest_node->identifier, new_board) == 1){
copy_board (new_board, parent_board);
}
else{
return 1;
}
}
return 0;
}
int
install_acl_entry (struct vxlan_instance * vins, struct acl_entry ae)
{
struct acl_entry * e;
if (vins == NULL)
return -1;
if (CHECK_VNI (vins->vni, ae.vni) < 0)
return -1;
e = (struct acl_entry *) malloc (sizeof (struct acl_entry));
memset (e, 0, sizeof (struct acl_entry));
memcpy (e, &ae, sizeof (ae));
switch (e->type) {
case ACL_TYPE_MAC :
insert_hash (&vins->acl_mac, e, &e->term_mac);
break;
case ACL_TYPE_IP4 :
insert_hash (&vins->acl_ip4, e, &e->term_ip4);
break;
case ACL_TYPE_IP6 :
insert_hash (&vins->acl_ip6, e, &e->term_ip6);
break;
case ACL_TYPE_RA :
vins->acl_mask |= ACL_MASK_RA;
break;
case ACL_TYPE_RS :
vins->acl_mask |= ACL_MASK_RS;
break;
default :
return -1;
}
return 1;
}
int main(){
char* names[] = {"Abhijeet", "Ramdev", "Kamdev", "Shatmatdev", "This is it",
"You are shit", "This is what I want to become", "I am a token",
"Hallelujah", "I am the king of the world", "How I met you mother?",
"Kamdev", "Luftwaffe"};
printf("\t%lu\n",sizeof(names));
for(int i = 0; i < (sizeof(names))/sizeof(names[0]); ++i){
insert_hash(names[i]);
}
print_hash_table();
delete_hash_table();
print_hash_table();
}
int
main(int argc, char **argv)
{
char buf[WORD_BUF];
while(scanf("%s", buf) != EOF) {
insert_hash(buf);
}
int i;
struct HNode *node;
for(i=0; i<NHASH; ++i)
for(node=Hash[i]; node!=NULL; node=node->next)
printf("%s\t%d\n", node->word, node->count);
return 0;
}
int search_hash(int num0[6])
{
int h = get_hash(num0);
if(hashtable[h] != -1)
{
int t = hashtable[h];
while(t != -1)
{
if(cmp(node[t].num, num0))
return 1;
t = node[t].next;
}
}
insert_hash(num0, h);
return 0;
}
int
fdb_add_entry (struct hash * fdb, u_int8_t * mac, struct in_addr vtep)
{
struct fdb_entry * entry;
struct sockaddr_in saddr_in;
entry = (struct fdb_entry *) malloc (sizeof (struct fdb_entry));
memset (entry, 0, sizeof (struct fdb_entry));
saddr_in.sin_family = AF_INET;
saddr_in.sin_port = htons (VXLAN_PORT);
saddr_in.sin_addr = vtep;
memcpy (&entry->vtep_addr, &saddr_in, sizeof (saddr_in));
entry->ttl = FDB_CACHE_TTL;
return insert_hash (fdb, entry, mac);
}
static int register_user_by_socket(int fd, struct curve25519_proto *proto)
{
void **pos;
struct sock_map_entry *entry;
rwlock_wr_lock(&sock_map_lock);
entry = xzmalloc(sizeof(*entry));
entry->fd = fd;
entry->proto = proto;
pos = insert_hash(entry->fd, entry, &sock_mapper);
if (pos) {
entry->next = (*pos);
(*pos) = entry;
}
rwlock_unlock(&sock_map_lock);
return 0;
}
unsigned int register_socket(int fd)
{
void **pos;
struct map_entry *entry;
rwlock_wr_lock(&map_lock);
entry = xzmalloc(sizeof(*entry));
entry->fd = fd;
entry->cpu = get_appropriate_cpu();
cpu_assigned[entry->cpu]++;
pos = insert_hash(entry->fd, entry, &mapper);
if (pos) {
entry->next = (*pos);
(*pos) = entry;
}
rwlock_unlock(&map_lock);
return entry->cpu;
}
int main()
{
Tree *tree = createTree();
Hashtable *table5, *table50, *table500;
FILE *fp50, *fp500, *fp5mil;
clock_t tstart, tend;
time_t t;
srand((unsigned) time(&t));
double favg;
int data = 0, i = 0, search, value, found;
int array50k[50000], *array500k, *array5mil;
array500k = malloc(500001 * sizeof(int));
array5mil = malloc(5000001 * sizeof(int));
if(tree == NULL)
printf("Memory allocation failed...");
fp50 = fopen("50tus.txt", "r");
fp500 = fopen("500tus.txt", "r");
fp5mil = fopen("5mil.txt", "r");
if(fp50 == NULL || fp500 == NULL|| fp5mil == NULL)
{
fprintf(stderr, "error\n");
return 1;
}
system("color 3");
table5 = create_table(50000000);
table50 = create_table(500000);
table500 = create_table(5000000);
/*50k*/
printf("\n50k insert, balance and search!\n");
printf("Insert: 50k ");
tstart = clock(); // start
for(i = 0; i < 50000; ++i)
{
fscanf(fp50,"%d\n", &data);
insert(tree, data);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
tree_to_arr(tree);
printf("Searching 100x in 50k ");
tstart = clock(); // start
find_tree(tree);
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
clear_tree(tree);
/*500k*/
printf("\n500k insert, balance and search!\n");
printf("Insert: 500k ");
tstart = clock(); // start
for(i = 0; i < 500000; ++i)
{
fscanf(fp500,"%d\n", &data);
insert(tree, data);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
tree_to_arr(tree);
printf("Searching 100x in 500k ");
tstart = clock(); // start
find_tree(tree);
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
clear_tree(tree);
/*5mille*/
printf("\n5 million insert, balance and search!\n");
printf("Insert: 5 million ");
tstart = clock(); // start
for(i = 0; i < 5000000; ++i)
{
fscanf(fp5mil,"%d\n", &data);
insert(tree, data);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
tree_to_arr(tree);
printf("Searching 100x in 5 million ");
tstart = clock(); // start
find_tree(tree);
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
clear_tree(tree);
/*50k sekvens*/
printf("\nSekvential insert 50k!\n");
tstart = clock(); // start
for(i = 0; i<50000; i++)
{
fscanf(fp50, "%d\n", &array50k[i]);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
printf("Sekvential search 50k, 100x!\n");
tstart = clock(); // start
for(i = 0; i<100; i++)
{
search = randomGen();
sekvential_search(array50k, search, 50000);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
printf("Sekvential search 50k, 100x(o)!\n");
tstart = clock(); // start
for(i = 0; i<100; i++)
{
search = randomGen();
sekvent_search(array50k, search, 50000);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
/*500k sekvens*/
printf("\nSekvential insert 500k!\n");
tstart = clock(); // start
for(i = 0; i<500000; i++)
{
fscanf(fp500, "%d\n", &array500k[i]);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
printf("Sekvential search 500k, 100x!\n");
tstart = clock(); // start
for(i = 0; i<100; i++)
{
search = randomGen();
sekvential_search(array500k, search, 500000);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
printf("Sekvential search 500k, 100x(o)!\n");
tstart = clock(); // start
for(i = 0; i<100; i++)
{
search = randomGen();
sekvent_search(array500k, search, 500000);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
/*5mille seq*/
printf("\nSekvential insert 5 million!\n");
tstart = clock(); // start
for(i = 0; i<5000000; i++)
{
fscanf(fp5mil, "%d\n", &array5mil[i]);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
printf("Sekvential search 5 million, 100x!\n");
tstart = clock(); // start
for(i = 0; i<100; i++)
{
search = randomGen();
sekvential_search(array5mil, search, 5000000);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
printf("Sekvential search 5 million, 100x(o)!\n");
tstart = clock(); // start
for(i = 0; i<100; i++)
{
search = randomGen();
sekvent_search(array5mil, search, 5000000);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
/*Hash 500k*/
printf("\nHash insert 50k!\n");
tstart = clock(); // start
for(i = 0; i<50000; i++){
fscanf(fp500, "%d\n", &value);
insert_hash(table50, value);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
/*Hash 500k*/
printf("\nHash insert 500k!\n");
tstart = clock(); // start
for(i = 0; i<500000; i++){
fscanf(fp500, "%d\n", &value);
insert_hash(table500, value);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
/*Hash 5 mille*/
printf("\nHash insert 5 million!\n");
tstart = clock(); // start
for(i = 0; i<5000000; i++){
fscanf(fp5mil, "%d\n", &value);
insert_hash(table5, value);
}
tend = clock(); // end
favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
printf("Avg. execution time: %g sec\n",favg);
destroy_table(table5);
destroy_table(table50);
destroy_table(table500);
destroyTree(tree);
return 0;
}
void insert_comment(char *comment) { /* Insert comments into Comment Table */
insert_hash( comment_table, comment);
}
void insert_id(char *text) { /* Populate Symbol Table */
insert_hash(id_table, text);
}
int mongo_write(const char *path, const char *buf, size_t size,
off_t offset, struct fuse_file_info *fi)
{
struct inode * e;
int res;
size_t reallen;
int32_t realend = size, blk_offset = 0;
const off_t write_end = size + offset;
char * lock;
bson doc, cond;
mongo * conn = get_conn();
uint8_t hash[20];
time_t now = time(NULL);
e = (struct inode*)fi->fh;
if((res = get_cached_inode(path, e)) != 0)
return res;
if(e->mode & S_IFDIR)
return -EISDIR;
/* Uncomment this for incredibly slow length calculations.
for(;realend >= 0 && buf[realend] == '\0'; realend--);
realend++;
for(blk_offset = 0; blk_offset < realend && buf[blk_offset] == 0; blk_offset++);
blk_offset -= blk_offset > 0 ? 1 : 0;
* The code below uses SSE4 instructions to find the first/last
* zero bytes by doing 16-byte comparisons at a time. This should give
* a ~16 speed boost on blocks with lots of zero bytes over the dumb
* method above.
*/
if(size >= 16) {
__m128i zero = _mm_setzero_si128();
lock = (char*)buf + size - 16;
while(lock >= buf) {
__m128i x = _mm_loadu_si128((__m128i*)lock);
res = _mm_movemask_epi8(_mm_cmpeq_epi8(zero, x));
if(res == 0xffff) {
lock -= 16;
continue;
}
realend = lock - buf + fls(res ^ 0xffff);
break;
}
if(lock <= buf)
realend = 0;
lock = (char*)buf;
while(lock - buf < realend) {
__m128i x = _mm_loadu_si128((__m128i*)lock);
res = _mm_movemask_epi8(_mm_cmpeq_epi8(zero, x));
if(res == 0xffff) {
lock += 16;
continue;
}
blk_offset = lock - buf + ffs(res ^ 0xffff) - 1;
break;
}
}
reallen = realend - blk_offset;
if(reallen == 0) {
pthread_mutex_lock(&e->wr_lock);
res = insert_empty(&e->wr_extent, offset, size);
goto end;
}
#ifdef __APPLE__
CC_SHA1(buf, size, hash);
#else
SHA1(buf, size, hash);
#endif
bson_init(&cond);
bson_append_binary(&cond, "_id", 0, (char*)hash, sizeof(hash));
bson_finish(&cond);
bson_init(&doc);
bson_append_start_object(&doc, "$setOnInsert");
char * comp_out = get_compress_buf();
size_t comp_size = snappy_max_compressed_length(reallen);
if((res = snappy_compress(buf + blk_offset, reallen,
comp_out, &comp_size)) != SNAPPY_OK) {
fprintf(stderr, "Error compressing input: %d\n", res);
return -EIO;
}
bson_append_binary(&doc, "data", 0, comp_out, comp_size);
bson_append_int(&doc, "offset", blk_offset);
bson_append_int(&doc, "size", size);
bson_append_time_t(&doc, "created", now);
bson_append_finish_object(&doc);
bson_finish(&doc);
res = mongo_update(conn, blocks_name, &cond, &doc,
MONGO_UPDATE_UPSERT, NULL);
bson_destroy(&doc);
bson_destroy(&cond);
if(res != MONGO_OK) {
fprintf(stderr, "Error committing block %s\n", conn->lasterrstr);
return -EIO;
}
pthread_mutex_lock(&e->wr_lock);
res = insert_hash(&e->wr_extent, offset, size, hash);
end:
if(write_end > e->size)
e->size = write_end;
if(now - e->wr_age > 3) {
res = serialize_extent(e, e->wr_extent);
if(res != 0) {
pthread_mutex_unlock(&e->wr_lock);
return res;
}
e->wr_age = now;
}
pthread_mutex_unlock(&e->wr_lock);
if(res != 0)
return res;
res = update_filesize(e, write_end);
if(res != 0)
return res;
return size;
}
void deal_page(link_t *link)
{
int sockfd;
char *host, *url;
requset_t *req;
host = link->host;
url = link->link;
if (NULL == (req = get_with_host_url(host, url))) {
quit_spider();
exit(1);
}
sockfd = get_fd_from_host(host);
if (0 > sockfd) { /* 负数表示没有打开过 */
printf("connecting to %s ...\n", host);
if (-1 == (sockfd = create_connect(host, PORT))) {
fprintf(stderr, "%s: Open host error with port %d!\n", host, PORT);
quit_spider();
exit(1);
}
printf("connected to %s!!\n", host);
if (0 != add_to_sers(host, sockfd)) {
fprintf(stderr, "Error: `SERVERS` is too small\n");
quit_spider();
exit(1);
}
}
/*
* 0x1: 发送一个http请求
*/
printf("writing...\n");
printf("writed. totle %lu\n", write(sockfd, req->re, req->len));
/*
* 0x2: 处理应答,重点
*/
int readlen;
int crlf_offset;
char buffer[BUF_SIZE], downlink[512];
char *href;
char *tag_a_href;
char savesour[100];
int sour_type;
link_t *curr_link;
int tmp_hash;
int filesize, readed;
printf("reading...\n");
readlen = read(sockfd, buffer, BUF_SIZE);
crlf_offset = strstr(buffer, CRLFEND)-buffer+4; /* 计算实际数据的偏移量 */
readed = readlen-crlf_offset;
filesize = file_size(buffer);
#ifdef DEBUG
printf("%s", buffer);
#endif
printf("readed. %d/%d: %.2f%%\n", readed, filesize, (float)readed/filesize*100);
for (;;) {
/* 处理一次读取 */
while (NULL != (href = get_next_href(buffer+crlf_offset))) {
tag_a_href = get_link(href);
/* 没有找到，或链接有错，或已经存在了的链接 */
/* NOTE 暂时修改下is_good_link来只读取.htm文档 */
if (!is_good_link(tag_a_href) ||
is_exist_hash(link_hash, tmp_hash = hash_value(tag_a_href)))
continue;
printf("-find link '%s', nice\n", tag_a_href);
insert_hash(link_hash, tmp_hash);
curr_link = calloc(sizeof(link_t), 1);
if (NULL == curr_link) {
quit_spider();
exit(1);
}
if (is_this_server_link(tag_a_href))
curr_link->host = strdup(link->host);
else
curr_link->host = get_host_from_link(tag_a_href);
curr_link->link = get_url_from_link(tag_a_href);
if (NULL == curr_link->link || NULL == curr_link->host)
free_link(curr_link);
#ifdef DEBUG
else
printf("成功构造:%s %s\n", curr_link->host, curr_link->link);
#endif
/* 判断是不是资源文件 */
sour_type = sources_type(tag_a_href);
if (sour_type >= 0) {
/* TODO 添加自动建立目录函数 */
char *tmp;
/* 构造下载链接和保存名字 */
strcpy(downlink, curr_link->host);
strcat(downlink, curr_link->link);
strcpy(savesour, SAVEPATH);
tmp = get_name_from_link(downlink);
strcat(savesour, tmp);
free(tmp);
printf("%s: downloading...\n", downlink);
switch (imbed_download(downlink, savesour)) {
case -1:
printf("%s: 连接服务器失败\n", curr_link->host);
break;
case -2:
printf("写到文件失败\n");
break;
case -3:
printf("%s: 资源链接错误\n", curr_link->link);
break;
default:
printf("%s: 下载成功(%d)\n", savesour, filesize);
}
free_link(curr_link);
} else {
g_queue_push_tail(link_que, (gpointer)curr_link);
}
free(tag_a_href);
}
crlf_offset = 0; /* 以后都不会有偏移量了 */
if (readed >= filesize)
break;
printf("reading...\n");
readlen = read(sockfd, buffer, BUF_SIZE);
readed += readlen;
printf("readed. %d/%d: %.2f%%\n", readed, filesize, (float)readed/filesize*100);
#ifdef DEBUG
printf("%s", buffer);
#endif
}
sleep(1); /* 处于人道化考虑，暂停2s */
}
int main(int argc, char *argv[]) {
clock_t startTime, endTime;
startTime = clock();
int p, my_rank;
/* initialize MPI stuff */
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD,&p);
MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
srand(time(NULL));
int row_num, nz, col_num, i;
FILE *fp;
fp = fopen("crs48x48.txt", "r");
fscanf(fp, "%d", &nz);
while (fgetc(fp) != '\n');
fscanf(fp, "%d", &row_num);
while (fgetc(fp) != '\n');
fscanf(fp, "%d", &col_num);
while (fgetc(fp) != '\n');
printf("%d => NZ = %d\n",my_rank, nz);
FILE *fpseed;
int seed[p];
//int *column_partition = (int *)malloc(sizeof(int)*col_num);
int *column_ptr;
int *hash_weights;
int num_cols_per_process[p];
int *YPartition = (int*)calloc(row_num, sizeof(int));
int *YmaxPartition = (int*)calloc(row_num, sizeof(int));
const int nitems = 2;
int blocklengths[2] = {1, 1};
MPI_Datatype types[2] = {MPI_INT, MPI_INT};
MPI_Datatype mpi_pair;
MPI_Aint offsets[2];
offsets[0] = offsetof(pair, col);
offsets[1] = offsetof(pair, nz);
MPI_Type_create_struct(nitems, blocklengths, offsets, types, &mpi_pair);
MPI_Type_commit(&mpi_pair);
//printf("datatype created\n");
pair A_partition_column[p];
pair *A_partition[p];
pair *my_columns;
column_ptr = (int *)malloc(sizeof(int) * (col_num+1));
// I need how many non-zeros in each column in the matrix data
for (i=0; i <= col_num; i++) {
fscanf(fp, "%d", &column_ptr[i]);
while (fgetc(fp) != '\n');
}
//column_ptr[i] = nz;
if (my_rank == 0) {
fpseed = fopen("seed48x48.txt", "r");
for(i=0; i<p; i++)
{
fscanf(fpseed, "%d\n", &seed[i]);
printf("seed[%d]: %d\n", i, seed[i]);
}
fclose(fpseed);
int i;
int prime_arr[prime_arr_len] = { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97,
101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181,
191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281,
283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397,
401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503,
509, 521, 523, 541
};
hash_weights = (int *)malloc(sizeof(int)*row_num);
genHashWeightsArr(hash_weights, row_num, prime_arr);
/*for (i=0; i<row_num; i++) {
printf("hashweights[%d]: %d\n",i, hash_weights[i]);
}*/
int *current_column_rows = (int *)malloc(sizeof(int)*row_num);
//printf("check 1\n");
HASHTABLE hash_columns;
hash_columns = createHashtable(p, row_num);
// read row_arr and insert in the hashtable for each column
int j,c, flag;
//insert seed cols
for (c=0; c<p; c++) {
j = seed[c];
int nz_in_current_col = column_ptr[j+1] - column_ptr[j];
fseek(fp, col_block_size*(col_num+1) + init_block_size*3 + rowVal_block_size*(column_ptr[j]), SEEK_SET);
//printf("inserting\n");
for (i=0; i<nz_in_current_col; i++) {
fscanf(fp, "%d,", ¤t_column_rows[i]);
while (fgetc(fp) != '\n');
}
hash_columns = insert_hash(hash_columns, current_column_rows, nz_in_current_col, j, p, hash_weights, row_num, c);
}
printf("\nSeeds:\n");
print_hash(hash_columns, p);
fseek(fp, col_block_size*(col_num+1) + init_block_size*3, SEEK_SET);
//#pragma omp parallel for private(fp, j) num_threads(8)
for (j=0; j<col_num; j++) {
int nz_in_current_col = column_ptr[j+1] - column_ptr[j];
flag =1;
for (i=0; i<nz_in_current_col; i++) {
fscanf(fp, "%d,", ¤t_column_rows[i]);
while (fgetc(fp) != '\n');
}
//current_column_rows[i] = -1;
/*if (j==0)
{
for (i=0; i<nz_in_current_col; i++) {
printf("cur col[%d]: %d\n",i, current_column_rows[i]);
}
*/
for(c =0 ; c<p; c++)
{
if(seed[c] == j)
{
flag = 0;
}
}
if(flag == 1)
{
hash_columns = insert_hash(hash_columns, current_column_rows, nz_in_current_col, j, p, hash_weights, row_num, -1);
}
//}
}
// Load balancing
//printf("inserted in hash\n");
print_hash(hash_columns, p);
// Generate a column-wise index storing the partition alloted to each column
NODE temp;
int max;
#pragma omp parallel for num_threads(p)
for (i=0; i<p; i++) {
max = 0;
A_partition_column[i].col = hash_columns->col_counts[i];
A_partition[i] = (pair *)malloc(sizeof(pair)*A_partition_column[i].col);
temp = hash_columns->buckets[i];
for (j = 0; j < A_partition_column[i].col; j++) {
A_partition[i][j].col = temp->col_index;
A_partition[i][j].nz = temp->col_nz;
if (temp->col_nz > max) {
max = temp->col_nz;
}
temp = temp->next;
}
for (j=0; j<row_num; j++) {
if(hash_columns->row_indices[i][j] > YmaxPartition[j]) {
YmaxPartition[j] = hash_columns->row_indices[i][j];
YPartition[j] = i;
}
}
A_partition_column[i].nz = max;
}
}
// Broadcast the column-wise partition array
MPI_Bcast(A_partition_column, p, mpi_pair, 0, MPI_COMM_WORLD);
if (my_rank == 0) {
my_columns = *A_partition;
}
else {
my_columns = (pair *)malloc(sizeof(struct _pair)*A_partition_column[my_rank].col);
}
if (my_rank == 0) {
for (i=1; i<p; i++) {
MPI_Send(A_partition[i], A_partition_column[i].col, mpi_pair, i, 0, MPI_COMM_WORLD);
}
}
else {
MPI_Recv(my_columns, A_partition_column[my_rank].col, mpi_pair, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
MPI_Bcast(YPartition, row_num, MPI_INT, 0, MPI_COMM_WORLD);
//check what recvd in mycolumns
/*for(i=0; i<A_partition_column[my_rank].col; i++)
{
printf("Rank %d , Col no: %d, myNz : %d\n", my_rank, my_columns[i].col, my_columns[i].nz);
}*/
//partition_fp = (FILE **)malloc(sizeof(FILE*)*p);
FILE *my_output;
char f_name[20];
int colIndex, myNz, rowIndex, j;
float val;
char *buffer;
float *Y;
Y = (float*)calloc(row_num, sizeof(float));
//Read X
FILE *fp2;
fp2 = fopen("Xvector_algo2.txt", "r");
int *X;
X = (int*)malloc(sizeof(int)*A_partition_column[my_rank].col);
printf("Rank %d recvd %d columns\n", my_rank, A_partition_column[my_rank].col);
#pragma omp parallel for private(fp2, colIndex, i)
for(i=0; i<A_partition_column[my_rank].col; i++)
{
colIndex = my_columns[i].col;
fseek(fp2, colIndex*vector_block_size, SEEK_SET);
fscanf(fp2, "%d\n", &X[i]);
}
fclose(fp2);
/*for(i=0; i<A_partition_column[my_rank].col; i++)
{
printf("Rank %d ::, X[%d] = %d\n",my_rank, i, X[i]);
}*/
//for each column in A_partition_column[my_rank]...
//Read non zeroes and multiply (computing local Y)...
#pragma omp parallel for private(fp, colIndex, myNz, rowIndex, val)
for(i=0; i<A_partition_column[my_rank].col; i++)
{
//printf("proc: %d, Operating on col %d \n", my_rank, colIndex);
colIndex = my_columns[i].col;
myNz = my_columns[i].nz;
//seek to non-zeroes corresponding to this column in file
fseek(fp, col_block_size*(col_num+1) + init_block_size*3 + rowVal_block_size*(column_ptr[colIndex]), SEEK_SET);
//fread(buffer, myNz*rowVal_block_size,1,fp);
//for each non zero...
for(j=0; j<myNz; j++)
{
fscanf(fp, "%d, %f", &rowIndex, &val);
while (fgetc(fp) != '\n');
if(rowIndex>=row_num)
{
//printf("\n\n***********ERROR %d\n\n\n\n", rowIndex);
}
#pragma omp atomic
Y[rowIndex]+= X[i]*val;
}
}
//printf("end of loop: %d\n", my_rank);
pairF *sendOthers[p];
int numRowsInPartition[p], part;
//numRowsInPartition = (int*) malloc(sizeof(int)*p);
#pragma omp parallel for
for(i=0; i<row_num; i++)
{
numRowsInPartition[i] = 0;
}
/* for(i=0; i<row_num; i++)
{
printf("YPartition[%d] = %d\n", i, YPartition[i]);
}
*/
#pragma omp parallel for
for(i=0; i<row_num; i++)
{
part = YPartition[i];
#pragma omp atomic
numRowsInPartition[part]++;
}
if (my_rank == 0) {
for(i=0;i < p; i++)
{
printf("Rank %d got %d rows of Y vector\n", i, numRowsInPartition[i]);
}
}
//make the arrays that have to be sent to other processes.
//pair arrays that store rowIndex and val.
//allocate!
for(i=0; i<p;i++)
{
//if(i!=my_rank)
//{
sendOthers[i] = (pairF*)malloc(sizeof(pairF)*numRowsInPartition[i]);
//}
}
int *current = (int*) calloc(p, sizeof(int));
int other, other_pos;
//populate!
for(i=0; i<row_num; i++)
{
other = YPartition[i];
//if(other!=my_rank)
//{
other_pos = current[other];
sendOthers[other][other_pos].row = i;
sendOthers[other][other_pos].val = Y[i];
current[other]++;
//}
}
//write to respective files
FILE *partition_fp[p];
//open output files
for (i=0; i< p; i++)
{
sprintf(f_name, "%d", i);
//printf("open file %d\n", i);
partition_fp[i] = fopen(f_name, "a");
}
//FILE *fp21 = fopen("hehe.txt", "a");
for(i=0; i<p; i++)
{
if(i!=my_rank)
{
other = i;
for(j=0; j< numRowsInPartition[other]; j++)
{
if(sendOthers[other][j].val!=0){
fprintf(partition_fp[other], "%d, %f, process %d\n",sendOthers[other][j].row, sendOthers[other][j].val, my_rank);
}
}
}
}
//read from respective files and add!
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < p; ++i)
{
fclose(partition_fp[i]);
}
//printf("all files closed by rank %d\n", my_rank);
sprintf(f_name, "%d", my_rank);
partition_fp[my_rank] = fopen(f_name, "r");
strcat(f_name, "_output.txt");
my_output = fopen(f_name, "w");
while (fscanf(partition_fp[my_rank], "%d, %f", &rowIndex, &val) > 0) // expect 1 successful conversion
{
while (fgetc(partition_fp[my_rank]) != '\n');
//update local y
//printf("\n****\nRank %d read value %f\n\n", my_rank, val);
Y[rowIndex]+=val;
}
for(i=0; i<numRowsInPartition[my_rank]; i++)
{
rowIndex = sendOthers[my_rank][i].row;
sendOthers[my_rank][i].val = Y[rowIndex];
//these are the final values!
fprintf(my_output, "%d, %f, process %d\n",sendOthers[my_rank][i].row, sendOthers[my_rank][i].val, my_rank);
}
fclose(partition_fp[my_rank]);
fclose(my_output);
endTime = clock();
printf("\nrank = %d, Time taken: %lf\n", my_rank, (double)(endTime - startTime)/CLOCKS_PER_SEC);
MPI_Finalize();
return 0;
}
