示例#1
0
static void
check_sector(struct tfilter *filter, int type, int rw, uint64_t sec, char *buf)
{
	if (sec >= filter->secs)
		return;

	if (rw) {
		if (type == PRE_CHECK)
			insert_hash(filter, sec, buf);
		else
			check_hash(filter, sec, buf, WRITE_INTEGRITY);
	} else if (type == POST_CHECK) {
		check_hash(filter, sec, buf, READ_INTEGRITY);
		insert_hash(filter, sec, buf);
	}
}
示例#2
0
static void dissector_init_ether_types(void)
{
	FILE *fp;
	char buff[512], *ptr;
	struct ether_type *et;
	void **pos;

	fp = fopen("/etc/netsniff-ng/ether.conf", "r");
	if (!fp)
		panic("No /etc/netsniff-ng/ether.conf found!\n");
	memset(buff, 0, sizeof(buff));
	while (fgets(buff, sizeof(buff), fp) != NULL) {
		buff[sizeof(buff) - 1] = 0;
		et = xmalloc(sizeof(*et));
		ptr = buff;
		ptr = skips(ptr);
		ptr = getuint(ptr, &et->id);
		ptr = skips(ptr);
		ptr = skipchar(ptr, ',');
		ptr = skips(ptr);
		ptr = strtrim_right(ptr, '\n');
		ptr = strtrim_right(ptr, ' ');
		et->type = xstrdup(ptr);
		et->next = NULL;
		pos = insert_hash(et->id, et, &eth_ether_types);
		if (pos) {
			et->next = *pos;
			*pos = et;
		}
		memset(buff, 0, sizeof(buff));
	}

	fclose(fp);
}
示例#3
0
void
exec_command_vni_create (char * str, int sock)
{
	u_int8_t vnibuf[3];
	char cmd[CONTROL_CMD_BUF_LEN];
	char vni[CONTROL_CMD_BUF_LEN];
	struct vxlan_instance * vins;
	
	if (sscanf (str, "%s %s", cmd, vni) < 2) {
		write (sock, CONTROL_ERRMSG, sizeof (CONTROL_ERRMSG));
		return;
	}
	
	strtovni (vni, vnibuf);

	if ((vins = search_hash (&vxlan.vins_tuple, vnibuf)) != NULL) {
		char errbuf[] = "this vxlan instance exists\n";
		write (sock, errbuf, strlen (errbuf));
		return;
	}
	
	vins = create_vxlan_instance (vnibuf);
	insert_hash (&vxlan.vins_tuple, vins, vnibuf);
	init_vxlan_instance (vins);
	vxlan.vins_num++;

	char msgbuf[] = "created\n";
	write (sock, msgbuf, strlen (msgbuf));
	return;
}
示例#4
0
int main()
{
    int initial_num, query_num;
    unsigned int roll_number = 0;
    char name[26];
    int i = 0;
    node_p *hash_arry = malloc(MAX_LIMIT*sizeof(node_p *));
    memset(hash_arry, 0, MAX_LIMIT*sizeof(node_p *));

    scanf("%d", &initial_num);

    for (i = 0; i < initial_num; i++) {
        scanf("%u", &roll_number);
        scanf("%s", name);
        insert_hash(hash_arry, roll_number, name);
    }

    scanf("%d", &query_num);
    for (i = 0; i < query_num; i++) {
        scanf("%u", &roll_number);
        search_hash(hash_arry, roll_number);
    }

    return 0;
}
示例#5
0
static bool
add_user_into_multicast_group( struct in_addr maddr, unsigned int *n_users ) {
  assert( multicast_groups != NULL );
  assert( n_users != NULL );

  if ( !IN_MULTICAST( ntohl( maddr.s_addr ) ) ) {
    return false;
  }

  lock_multicast_group_table();

  unsigned int *in_use = search_hash( &multicast_groups->groups, &maddr );
  if ( in_use == NULL ) {
    in_use = malloc( sizeof( unsigned int ) );
    assert( in_use != NULL );
    *in_use = 0;
    insert_hash( &multicast_groups->groups, in_use, &maddr );
  }
  ( *in_use )++;

  *n_users = *in_use;

  unlock_multicast_group_table();

  return true;
}
示例#6
0
static int register_user_by_sockaddr(struct sockaddr_storage *sa,
				     size_t sa_len,
				     struct curve25519_proto *proto)
{
	void **pos;
	struct sockaddr_map_entry *entry;
	unsigned int hash = hash_name((char *) sa, sa_len);

	rwlock_wr_lock(&sockaddr_map_lock);

	entry = xzmalloc(sizeof(*entry));
	entry->sa = xmemdupz(sa, sa_len);
	entry->sa_len = sa_len;
	entry->proto = proto;

	pos = insert_hash(hash, entry, &sockaddr_mapper);
	if (pos) {
		entry->next = (*pos);
		(*pos) = entry;
	}

	rwlock_unlock(&sockaddr_map_lock);

	return 0;
}
示例#7
0
文件: describe.c 项目: OpenSpan/git
static void add_to_known_names(const char *path,
			       const unsigned char *peeled,
			       int prio,
			       const unsigned char *sha1)
{
	struct commit_name *e = find_commit_name(peeled);
	struct tag *tag = NULL;
	if (replace_name(e, prio, sha1, &tag)) {
		if (!e) {
			void **pos;
			e = xmalloc(sizeof(struct commit_name));
			hashcpy(e->peeled, peeled);
			pos = insert_hash(hash_sha1(peeled), e, &names);
			if (pos) {
				e->next = *pos;
				*pos = e;
			} else {
				e->next = NULL;
			}
		}
		e->tag = tag;
		e->prio = prio;
		e->name_checked = 0;
		hashcpy(e->sha1, sha1);
		e->path = path;
	}
}
示例#8
0
static void dissector_init_oui(void)
{
	FILE *fp;
	char buff[512], *ptr;
	struct vendor_id *ven;
	void **pos;

	fp = fopen("/etc/netsniff-ng/oui.conf", "r");
	if (!fp)
		panic("No /etc/netsniff-ng/oui.conf found!\n");
	memset(buff, 0, sizeof(buff));
	while (fgets(buff, sizeof(buff), fp) != NULL) {
		buff[sizeof(buff) - 1] = 0;
		ven = xmalloc(sizeof(*ven));
		ptr = buff;
		ptr = skips(ptr);
		ptr = getuint(ptr, &ven->id);
		ptr = skips(ptr);
		ptr = skipchar(ptr, ',');
		ptr = skips(ptr);
		ptr = strtrim_right(ptr, '\n');
		ptr = strtrim_right(ptr, ' ');
		ven->vendor = xstrdup(ptr);
		ven->next = NULL;
		pos = insert_hash(ven->id, ven, &eth_oui);
		if (pos) {
			ven->next = *pos;
			*pos = ven;
		}
		memset(buff, 0, sizeof(buff));
	}

	fclose(fp);
}
示例#9
0
static void dissector_init_ports_tcp(void)
{
	FILE *fp;
	char buff[512], *ptr;
	struct port_tcp *ptcp;
	void **pos;

	fp = fopen("/etc/netsniff-ng/tcp.conf", "r");
	if (!fp)
		panic("No /etc/netsniff-ng/tcp.conf found!\n");
	memset(buff, 0, sizeof(buff));
	while (fgets(buff, sizeof(buff), fp) != NULL) {
		buff[sizeof(buff) - 1] = 0;
		ptcp = xmalloc(sizeof(*ptcp));
		ptr = buff;
		ptr = skips(ptr);
		ptr = getuint(ptr, &ptcp->id);
		ptr = skips(ptr);
		ptr = skipchar(ptr, ',');
		ptr = skips(ptr);
		ptr = strtrim_right(ptr, '\n');
		ptr = strtrim_right(ptr, ' ');
		ptcp->port = xstrdup(ptr);
		ptcp->next = NULL;
		pos = insert_hash(ptcp->id, ptcp, &eth_ports_tcp);
		if (pos) {
			ptcp->next = *pos;
			*pos = ptcp;
		}
		memset(buff, 0, sizeof(buff));
	}

	fclose(fp);
}
示例#10
0
static void *_hash_tbl_insert_item(QSP_ARG_DECL  Container *cnt_p,Item *ip)
{
	int stat;
	stat=insert_hash(ip,cnt_p->cnt_htp);
	if( stat == 0 )
		return ip;
	return NULL;
}
示例#11
0
static void dissector_init_ports(enum ports which)
{
	FILE *fp;
	char buff[128], *ptr, *file;
	struct hash_table *table;
	struct port *p;
	void **pos;

	switch (which) {
	case PORTS_UDP:
		file = "/etc/netsniff-ng/udp.conf";
		table = &eth_ports_udp;
		break;
	case PORTS_TCP:
		file = "/etc/netsniff-ng/tcp.conf";
		table = &eth_ports_tcp;
		break;
	case PORTS_ETHER:
		file = "/etc/netsniff-ng/ether.conf";
		table = &eth_ether_types;
		break;
	default:
		bug();
	}

	fp = fopen(file, "r");
	if (!fp)
		panic("No %s found!\n", file);

	memset(buff, 0, sizeof(buff));

	while (fgets(buff, sizeof(buff), fp) != NULL) {
		buff[sizeof(buff) - 1] = 0;
		ptr = buff;

		p = xmalloc(sizeof(*p));
		p->id = strtol(ptr, &ptr, 0);

		if ((ptr = strstr(buff, ", ")))
			ptr += strlen(", ");
		ptr = strtrim_right(ptr, '\n');
		ptr = strtrim_right(ptr, ' ');

		p->port = xstrdup(ptr);
		p->next = NULL;

		pos = insert_hash(p->id, p, table);
		if (pos) {
			p->next = *pos;
			*pos = p;
		}

		memset(buff, 0, sizeof(buff));
	}

	fclose(fp);
}
示例#12
0
文件: oui.c 项目: 0x0mar/netsniff-ng
void dissector_init_oui(void)
{
	FILE *fp;
	char buff[128], *ptr, *end;
	struct vendor_id *v;
	void **pos;

	if (initialized)
		return;

	fp = fopen(ETCDIRE_STRING "/oui.conf", "r");
	if (!fp)
		panic("No oui.conf found!\n");

	memset(buff, 0, sizeof(buff));

	while (fgets(buff, sizeof(buff), fp) != NULL) {
		buff[sizeof(buff) - 1] = 0;
		ptr = buff;

		v = xmalloc(sizeof(*v));
		v->id = strtol(ptr, &end, 0);
		/* not a valid line, skip */
		if (v->id == 0 && end == ptr) {
			xfree(v);
			continue;
		}

		ptr = strstr(buff, ", ");
		/* likewise */
		if (!ptr) {
			xfree(v);
			continue;
		}

		ptr += strlen(", ");
		ptr = strtrim_right(ptr, '\n');
		ptr = strtrim_right(ptr, ' ');

		v->vendor = xstrdup(ptr);
		v->next = NULL;

		pos = insert_hash(v->id, v, &oui);
		if (pos) {
			v->next = *pos;
			*pos = v;
		}

		memset(buff, 0, sizeof(buff));
	}

	fclose(fp);
	initialized = true;
}
示例#13
0
//checks a specific move from specific location on the board. If move viable and doesn't lead to duplicate board, 
//new board shows this move and 1 is returned. If no moves found, returns 0
int check_move (movement direction, int i, int j, hash *hash_array, Node *start, Node *latest_node, int parent_board [BOARD_HEIGHT][BOARD_WIDTH], int new_board[BOARD_HEIGHT][BOARD_WIDTH]){
    if (valid_move (direction, i, j, new_board)){
        move_piece (direction, i, j, new_board);
        latest_node->identifier = allocate_board_identifier (new_board);
        if (insert_hash (hash_array, latest_node, latest_node->identifier, new_board) == 1){
            copy_board (new_board, parent_board);
        }
        else{
            return 1;
        }
    }
    return 0;
}
示例#14
0
int
install_acl_entry (struct vxlan_instance * vins, struct acl_entry ae)
{
        struct acl_entry * e;

        if (vins == NULL)
                return -1;

        if (CHECK_VNI (vins->vni, ae.vni) < 0)
                return -1;

        e = (struct acl_entry *) malloc (sizeof (struct acl_entry));
        memset (e, 0, sizeof (struct acl_entry));
        memcpy (e, &ae, sizeof (ae));

        switch (e->type) {
        case ACL_TYPE_MAC :
                insert_hash (&vins->acl_mac, e, &e->term_mac);
                break;
        case ACL_TYPE_IP4 :
                insert_hash (&vins->acl_ip4, e, &e->term_ip4);
                break;
        case ACL_TYPE_IP6 :
                insert_hash (&vins->acl_ip6, e, &e->term_ip6);
                break;
        case ACL_TYPE_RA :
                vins->acl_mask |= ACL_MASK_RA;
                break;
        case ACL_TYPE_RS :
                vins->acl_mask |= ACL_MASK_RS;
                break;

        default :
                return -1;
        }

        return 1;
}
示例#15
0
int main(){

    char* names[] = {"Abhijeet", "Ramdev", "Kamdev", "Shatmatdev", "This is it",
    "You are shit", "This is what I want to become", "I am a token",
    "Hallelujah", "I am the king of the world", "How I met you mother?",
    "Kamdev", "Luftwaffe"};

    printf("\t%lu\n",sizeof(names));
    for(int i = 0; i < (sizeof(names))/sizeof(names[0]); ++i){
        insert_hash(names[i]);
    }
    print_hash_table();
    delete_hash_table();
    print_hash_table(); 
}
示例#16
0
int
main(int argc, char **argv)
{
	char buf[WORD_BUF];
	while(scanf("%s", buf) != EOF) {
		insert_hash(buf);
	}

	int i;
	struct HNode *node;
	for(i=0; i<NHASH; ++i) 
		for(node=Hash[i]; node!=NULL; node=node->next) 
			printf("%s\t%d\n", node->word, node->count);

	return 0;
}
示例#17
0
int search_hash(int num0[6])
{
    int h = get_hash(num0);
    if(hashtable[h] != -1)
    {
        int t = hashtable[h];
        while(t != -1)
        {
            if(cmp(node[t].num, num0))
                return 1;
            t = node[t].next;
        }
    }
    insert_hash(num0, h);
    return 0;
}
示例#18
0
文件: fdb.c 项目: k-akashi/hogelan
int
fdb_add_entry (struct hash * fdb, u_int8_t * mac, struct in_addr vtep)
{
	struct fdb_entry * entry;
	struct sockaddr_in saddr_in;
	
	entry = (struct fdb_entry *) malloc (sizeof (struct fdb_entry));
	memset (entry, 0, sizeof (struct fdb_entry));
	
	saddr_in.sin_family = AF_INET;
	saddr_in.sin_port = htons (VXLAN_PORT);
	saddr_in.sin_addr = vtep;

	memcpy (&entry->vtep_addr, &saddr_in, sizeof (saddr_in));
	entry->ttl = FDB_CACHE_TTL;
	
	return insert_hash (fdb, entry, mac);
}
示例#19
0
static int register_user_by_socket(int fd, struct curve25519_proto *proto)
{
	void **pos;
	struct sock_map_entry *entry;

	rwlock_wr_lock(&sock_map_lock);

	entry = xzmalloc(sizeof(*entry));
	entry->fd = fd;
	entry->proto = proto;

	pos = insert_hash(entry->fd, entry, &sock_mapper);
	if (pos) {
		entry->next = (*pos);
		(*pos) = entry;
	}

	rwlock_unlock(&sock_map_lock);

	return 0;
}
示例#20
0
unsigned int register_socket(int fd)
{
	void **pos;
	struct map_entry *entry;

	rwlock_wr_lock(&map_lock);

	entry = xzmalloc(sizeof(*entry));
	entry->fd = fd;
	entry->cpu = get_appropriate_cpu();

	cpu_assigned[entry->cpu]++;

	pos = insert_hash(entry->fd, entry, &mapper);
	if (pos) {
		entry->next = (*pos);
		(*pos) = entry;
	}

	rwlock_unlock(&map_lock);

	return entry->cpu;
}
int main()
{
    Tree *tree = createTree();
    Hashtable *table5, *table50, *table500;
    FILE *fp50, *fp500, *fp5mil;
    clock_t tstart, tend;
    time_t t;
    srand((unsigned) time(&t));
    double favg;
    int data = 0, i = 0, search, value, found;
    int array50k[50000], *array500k, *array5mil;
    array500k = malloc(500001 * sizeof(int));
    array5mil = malloc(5000001 * sizeof(int));
    if(tree == NULL)
        printf("Memory allocation failed...");
    fp50 = fopen("50tus.txt", "r");
    fp500 = fopen("500tus.txt", "r");
    fp5mil = fopen("5mil.txt", "r");
    if(fp50 == NULL || fp500 == NULL|| fp5mil == NULL)
    {
        fprintf(stderr, "error\n");
        return 1;
    }
    system("color 3");
    table5 = create_table(50000000);
    table50 = create_table(500000);
    table500 = create_table(5000000);

    /*50k*/
    printf("\n50k insert, balance and search!\n");
    printf("Insert: 50k ");
    tstart = clock(); // start
    for(i = 0; i < 50000; ++i)
    {
        fscanf(fp50,"%d\n", &data);
        insert(tree, data);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    tree_to_arr(tree);
    printf("Searching 100x in 50k ");
    tstart = clock(); // start
    find_tree(tree);
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    clear_tree(tree);

    /*500k*/
    printf("\n500k insert, balance and search!\n");
    printf("Insert: 500k ");
    tstart = clock(); // start
    for(i = 0; i < 500000; ++i)
    {
        fscanf(fp500,"%d\n", &data);
        insert(tree, data);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    tree_to_arr(tree);
    printf("Searching 100x in 500k ");
    tstart = clock(); // start
    find_tree(tree);
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    clear_tree(tree);

    /*5mille*/
    printf("\n5 million insert, balance and search!\n");
    printf("Insert: 5 million ");
    tstart = clock(); // start
    for(i = 0; i < 5000000; ++i)
    {
        fscanf(fp5mil,"%d\n", &data);
        insert(tree, data);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    tree_to_arr(tree);
    printf("Searching 100x in 5 million ");
    tstart = clock(); // start
    find_tree(tree);
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    clear_tree(tree);

    /*50k sekvens*/
    printf("\nSekvential insert 50k!\n");
    tstart = clock(); // start
    for(i = 0; i<50000; i++)
    {
        fscanf(fp50, "%d\n", &array50k[i]);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    printf("Sekvential search 50k, 100x!\n");
    tstart = clock(); // start
    for(i = 0; i<100; i++)
    {
        search = randomGen();
        sekvential_search(array50k, search, 50000);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    printf("Sekvential search 50k, 100x(o)!\n");
    tstart = clock(); // start
    for(i = 0; i<100; i++)
    {
        search = randomGen();
        sekvent_search(array50k, search, 50000);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);

    /*500k sekvens*/
    printf("\nSekvential insert 500k!\n");
    tstart = clock(); // start
    for(i = 0; i<500000; i++)
    {
        fscanf(fp500, "%d\n", &array500k[i]);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    printf("Sekvential search 500k, 100x!\n");
    tstart = clock(); // start
    for(i = 0; i<100; i++)
    {
        search = randomGen();
        sekvential_search(array500k, search, 500000);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    printf("Sekvential search 500k, 100x(o)!\n");
    tstart = clock(); // start
    for(i = 0; i<100; i++)
    {
        search = randomGen();
        sekvent_search(array500k, search, 500000);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);

    /*5mille seq*/
    printf("\nSekvential insert 5 million!\n");
    tstart = clock(); // start
    for(i = 0; i<5000000; i++)
    {
        fscanf(fp5mil, "%d\n", &array5mil[i]);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    printf("Sekvential search 5 million, 100x!\n");
    tstart = clock(); // start
    for(i = 0; i<100; i++)
    {
        search = randomGen();
        sekvential_search(array5mil, search, 5000000);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);
    printf("Sekvential search 5 million, 100x(o)!\n");
    tstart = clock(); // start
    for(i = 0; i<100; i++)
    {
        search = randomGen();
        sekvent_search(array5mil, search, 5000000);
    }
    tend = clock(); // end
    favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
    printf("Avg. execution time: %g sec\n",favg);

    /*Hash 500k*/
    printf("\nHash insert 50k!\n");
    tstart = clock(); // start
    for(i = 0; i<50000; i++){
        fscanf(fp500, "%d\n", &value);
        insert_hash(table50, value);
    }
        tend = clock(); // end
        favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
        printf("Avg. execution time: %g sec\n",favg);

    /*Hash 500k*/
    printf("\nHash insert 500k!\n");
    tstart = clock(); // start
    for(i = 0; i<500000; i++){
        fscanf(fp500, "%d\n", &value);
        insert_hash(table500, value);
    }
        tend = clock(); // end
        favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
        printf("Avg. execution time: %g sec\n",favg);

    /*Hash 5 mille*/
    printf("\nHash insert 5 million!\n");
    tstart = clock(); // start
    for(i = 0; i<5000000; i++){
        fscanf(fp5mil, "%d\n", &value);
        insert_hash(table5, value);
    }
        tend = clock(); // end
        favg = ((double)(tend - tstart))/CLOCKS_PER_SEC;
        printf("Avg. execution time: %g sec\n",favg);

    destroy_table(table5);
    destroy_table(table50);
    destroy_table(table500);
    destroyTree(tree);
    return 0;
}
示例#22
0
void insert_comment(char *comment)  {  /* Insert comments into Comment Table */
	insert_hash( comment_table, comment);
}
示例#23
0
void insert_id(char *text)  {  /* Populate Symbol Table */
	insert_hash(id_table, text);
}
示例#24
0
int mongo_write(const char *path, const char *buf, size_t size,
                off_t offset, struct fuse_file_info *fi)
{
    struct inode * e;
    int res;
    size_t reallen;
    int32_t realend = size, blk_offset = 0;
    const off_t write_end = size + offset;
    char * lock;
    bson doc, cond;
    mongo * conn = get_conn();
    uint8_t hash[20];
    time_t now = time(NULL);

    e = (struct inode*)fi->fh;
    if((res = get_cached_inode(path, e)) != 0)
        return res;

    if(e->mode & S_IFDIR)
        return -EISDIR;

    /* Uncomment this for incredibly slow length calculations.
    for(;realend >= 0 && buf[realend] == '\0'; realend--);
    realend++;
    for(blk_offset = 0; blk_offset < realend && buf[blk_offset] == 0; blk_offset++);
    blk_offset -= blk_offset > 0 ? 1 : 0;
    * The code below uses SSE4 instructions to find the first/last
    * zero bytes by doing 16-byte comparisons at a time. This should give
    * a ~16 speed boost on blocks with lots of zero bytes over the dumb
    * method above.
    */
    
    if(size >= 16) {
        __m128i zero = _mm_setzero_si128();
        lock = (char*)buf + size - 16;
        while(lock >= buf) {
            __m128i x = _mm_loadu_si128((__m128i*)lock);
            res = _mm_movemask_epi8(_mm_cmpeq_epi8(zero, x));
            if(res == 0xffff) {
                lock -= 16;
                continue;
            }
            realend = lock - buf + fls(res ^ 0xffff);
            break;
        }
        if(lock <= buf)
            realend = 0;

        lock = (char*)buf;
        while(lock - buf < realend) {
            __m128i x = _mm_loadu_si128((__m128i*)lock);
            res = _mm_movemask_epi8(_mm_cmpeq_epi8(zero, x));
            if(res == 0xffff) {
                lock += 16;
                continue;
            }
            blk_offset = lock - buf + ffs(res ^ 0xffff) - 1;
            break;
        }
    }

    reallen = realend - blk_offset;
    if(reallen == 0) {
        pthread_mutex_lock(&e->wr_lock);
        res = insert_empty(&e->wr_extent, offset, size);
        goto end;
    }

#ifdef __APPLE__
    CC_SHA1(buf, size, hash);
#else
    SHA1(buf, size, hash);
#endif

    bson_init(&cond);
    bson_append_binary(&cond, "_id", 0, (char*)hash, sizeof(hash));
    bson_finish(&cond);

    bson_init(&doc);
    bson_append_start_object(&doc, "$setOnInsert");
    char * comp_out = get_compress_buf();
    size_t comp_size = snappy_max_compressed_length(reallen);
    if((res = snappy_compress(buf + blk_offset, reallen,
        comp_out, &comp_size)) != SNAPPY_OK) {
        fprintf(stderr, "Error compressing input: %d\n", res);
        return -EIO;
    }
    bson_append_binary(&doc, "data", 0, comp_out, comp_size);
    bson_append_int(&doc, "offset", blk_offset);
    bson_append_int(&doc, "size", size);
    bson_append_time_t(&doc, "created", now);
    bson_append_finish_object(&doc);
    bson_finish(&doc);

    res = mongo_update(conn, blocks_name, &cond, &doc,
        MONGO_UPDATE_UPSERT, NULL);
    bson_destroy(&doc);
    bson_destroy(&cond);

    if(res != MONGO_OK) {
        fprintf(stderr, "Error committing block %s\n", conn->lasterrstr);
        return -EIO;
    }

    pthread_mutex_lock(&e->wr_lock);
    res = insert_hash(&e->wr_extent, offset, size, hash);

end:
    if(write_end > e->size)
        e->size = write_end;

    if(now - e->wr_age > 3) {
        res = serialize_extent(e, e->wr_extent);
        if(res != 0) {
            pthread_mutex_unlock(&e->wr_lock);
            return res;    
        }
        e->wr_age = now;
    }
    pthread_mutex_unlock(&e->wr_lock);
    if(res != 0)
        return res;

    res = update_filesize(e, write_end);

    if(res != 0)
        return res;
    return size;
}
示例#25
0
文件: spider.c 项目: leetking/spider
void deal_page(link_t *link)
{
	int sockfd;
	char *host, *url;
	requset_t *req;

	host = link->host;
	url = link->link;
	if (NULL == (req = get_with_host_url(host, url))) {
		quit_spider();
		exit(1);
	}

	sockfd = get_fd_from_host(host);
	if (0 > sockfd) {	/* 负数表示没有打开过 */
		printf("connecting to %s ...\n", host);
		if (-1 == (sockfd = create_connect(host, PORT))) {
			fprintf(stderr, "%s: Open host error with port %d!\n", host, PORT);
			quit_spider();
			exit(1);
		}
		printf("connected to %s!!\n", host);
		if (0 != add_to_sers(host, sockfd)) {
			fprintf(stderr, "Error: `SERVERS` is too small\n");
			quit_spider();
			exit(1);
		}
	}

	/*
	 * 0x1: 发送一个http请求
	 */
	printf("writing...\n");
	printf("writed. totle %lu\n", write(sockfd, req->re, req->len));
	
	/*
	 * 0x2: 处理应答,重点
	 */
	int readlen;
	int crlf_offset;
	char buffer[BUF_SIZE], downlink[512];
	char *href;
	char *tag_a_href;
	char savesour[100];
	int sour_type;
	link_t *curr_link;
	int tmp_hash;
	int filesize, readed;
	
	printf("reading...\n");
	readlen = read(sockfd, buffer, BUF_SIZE);
	crlf_offset = strstr(buffer, CRLFEND)-buffer+4;	/* 计算实际数据的偏移量 */
	readed = readlen-crlf_offset;
	filesize = file_size(buffer);
#ifdef DEBUG
	printf("%s", buffer);
#endif
	printf("readed. %d/%d: %.2f%%\n", readed, filesize, (float)readed/filesize*100);
	for (;;) {
		/* 处理一次读取 */
		while (NULL != (href = get_next_href(buffer+crlf_offset))) {
			tag_a_href = get_link(href);
			/* 没有找到,或链接有错,或已经存在了的链接 */
			/* NOTE 暂时修改下is_good_link来只读取.htm文档 */
			if (!is_good_link(tag_a_href) ||
					is_exist_hash(link_hash, tmp_hash = hash_value(tag_a_href)))
				continue;
			printf("-find link '%s', nice\n", tag_a_href);
			insert_hash(link_hash, tmp_hash);

			curr_link = calloc(sizeof(link_t), 1);
			if (NULL == curr_link) {
				quit_spider();
				exit(1);
			}

			if (is_this_server_link(tag_a_href))
				curr_link->host = strdup(link->host);
			else
				curr_link->host = get_host_from_link(tag_a_href);
			curr_link->link = get_url_from_link(tag_a_href);
			if (NULL == curr_link->link || NULL == curr_link->host)
				free_link(curr_link);
#ifdef DEBUG
			else
				printf("成功构造:%s %s\n", curr_link->host, curr_link->link);
#endif

			/* 判断是不是资源文件 */
			sour_type = sources_type(tag_a_href);
			if (sour_type >= 0) {
				/* TODO 添加自动建立目录函数 */
				char *tmp;
				/* 构造下载链接和保存名字 */
				strcpy(downlink, curr_link->host);
				strcat(downlink, curr_link->link);
				strcpy(savesour, SAVEPATH);
				tmp = get_name_from_link(downlink);
				strcat(savesour, tmp);
				free(tmp);
				printf("%s: downloading...\n", downlink);
				switch (imbed_download(downlink, savesour)) {
					case -1:
						printf("%s: 连接服务器失败\n", curr_link->host);
						break;
					case -2:
						printf("写到文件失败\n");
						break;
					case -3:
						printf("%s: 资源链接错误\n", curr_link->link);
						break;
					default:
						printf("%s: 下载成功(%d)\n", savesour, filesize);
				}
				free_link(curr_link);
			} else {
				g_queue_push_tail(link_que, (gpointer)curr_link);
			}
			free(tag_a_href);
		}

		crlf_offset = 0;	/* 以后都不会有偏移量了 */
		if (readed >= filesize)
			break;
		printf("reading...\n");
		readlen = read(sockfd, buffer, BUF_SIZE);
		readed += readlen;
		printf("readed. %d/%d: %.2f%%\n", readed, filesize, (float)readed/filesize*100);
#ifdef DEBUG
		printf("%s", buffer);
#endif
	}

	sleep(1);	/* 处于人道化考虑,暂停2s */
}
int main(int argc, char *argv[]) {
	
	clock_t startTime, endTime;
	startTime = clock();

	int p, my_rank;

	/* initialize MPI stuff */
	MPI_Init(&argc, &argv);
	MPI_Comm_size(MPI_COMM_WORLD,&p);
	MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);	

	srand(time(NULL));
	
	int row_num, nz, col_num, i;

	FILE *fp;
	fp = fopen("crs48x48.txt", "r");
	fscanf(fp, "%d", &nz);
	while (fgetc(fp) != '\n');

	fscanf(fp, "%d", &row_num);
	while (fgetc(fp) != '\n');

	fscanf(fp, "%d", &col_num);
	while (fgetc(fp) != '\n');

	printf("%d => NZ = %d\n",my_rank, nz);

	FILE *fpseed;
	int seed[p];
	

	//int *column_partition = (int *)malloc(sizeof(int)*col_num);
	int *column_ptr;
	int *hash_weights;

	int num_cols_per_process[p];
	int *YPartition = (int*)calloc(row_num, sizeof(int));
	int *YmaxPartition = (int*)calloc(row_num, sizeof(int));
	
	const int nitems 		=  2;
    int blocklengths[2] 	= {1, 1};
    MPI_Datatype types[2] 	= {MPI_INT, MPI_INT};
    MPI_Datatype mpi_pair;
    MPI_Aint offsets[2];

    offsets[0] = offsetof(pair, col);
    offsets[1] = offsetof(pair, nz);

    MPI_Type_create_struct(nitems, blocklengths, offsets, types, &mpi_pair);
    MPI_Type_commit(&mpi_pair);

    //printf("datatype created\n");
	pair A_partition_column[p];
	pair *A_partition[p];

	pair *my_columns;

	column_ptr = (int *)malloc(sizeof(int) * (col_num+1));
	// I need how many non-zeros in each column in the matrix data
	for (i=0; i <= col_num; i++) {
		fscanf(fp, "%d", &column_ptr[i]);
		while (fgetc(fp) != '\n');
	}
	//column_ptr[i] = nz;

	if (my_rank == 0) {	
		fpseed = fopen("seed48x48.txt", "r");
		for(i=0; i<p; i++)
		{
			fscanf(fpseed, "%d\n", &seed[i]);
			printf("seed[%d]: %d\n", i, seed[i]);
		}
		fclose(fpseed);

		int i;
		int prime_arr[prime_arr_len] = {  2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97,
											101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181,
											191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281,
											283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397,
											401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503,
											509, 521, 523, 541 
										 };

	
		hash_weights = (int *)malloc(sizeof(int)*row_num);
		genHashWeightsArr(hash_weights, row_num, prime_arr);

		/*for (i=0; i<row_num; i++) {
			printf("hashweights[%d]: %d\n",i,  hash_weights[i]);
		}*/

		int *current_column_rows = (int *)malloc(sizeof(int)*row_num);	
		//printf("check 1\n");
		HASHTABLE hash_columns;
		hash_columns = createHashtable(p, row_num);

		// read row_arr and insert in the hashtable for each column
		int j,c, flag;

		//insert seed cols
		for (c=0; c<p; c++) {
			j = seed[c];
			int nz_in_current_col = column_ptr[j+1] - column_ptr[j];
		
			fseek(fp, col_block_size*(col_num+1) + init_block_size*3 + rowVal_block_size*(column_ptr[j]), SEEK_SET);
			//printf("inserting\n");
			for (i=0; i<nz_in_current_col; i++) {
				fscanf(fp, "%d,", &current_column_rows[i]);
				while (fgetc(fp) != '\n');
			}
			
			hash_columns = insert_hash(hash_columns, current_column_rows, nz_in_current_col, j, p, hash_weights, row_num, c);
		}

		printf("\nSeeds:\n");
		print_hash(hash_columns, p);

		fseek(fp, col_block_size*(col_num+1) + init_block_size*3, SEEK_SET);

		//#pragma omp parallel for private(fp, j) num_threads(8)
		for (j=0; j<col_num; j++) {
			int nz_in_current_col = column_ptr[j+1] - column_ptr[j];
			flag =1;
			for (i=0; i<nz_in_current_col; i++) {
				fscanf(fp, "%d,", &current_column_rows[i]);
				while (fgetc(fp) != '\n');
			}
			//current_column_rows[i] = -1;
			/*if (j==0)
			{
				for (i=0; i<nz_in_current_col; i++) {
					printf("cur col[%d]: %d\n",i,  current_column_rows[i]);
				}
			*/	
			for(c =0 ; c<p; c++)
			{
				if(seed[c] == j)
				{
					flag = 0;
				}
			}
			
			if(flag == 1)
			{
				hash_columns = insert_hash(hash_columns, current_column_rows, nz_in_current_col, j, p, hash_weights, row_num, -1);
			}
				
			//}
		}
		// Load balancing
		//printf("inserted in hash\n");
		print_hash(hash_columns, p);

		// Generate a column-wise index storing the partition alloted to each column
		NODE temp;
		int max;

		#pragma omp parallel for num_threads(p)
		for (i=0; i<p; i++) {
			max = 0;
			A_partition_column[i].col = hash_columns->col_counts[i];
			A_partition[i] = (pair *)malloc(sizeof(pair)*A_partition_column[i].col);

			temp = hash_columns->buckets[i];
			for (j = 0; j < A_partition_column[i].col; j++) {
				A_partition[i][j].col = temp->col_index;
				A_partition[i][j].nz = temp->col_nz;
				if (temp->col_nz > max) {
					max = temp->col_nz;
				}
				temp = temp->next;
			}

			for (j=0; j<row_num; j++) {
				if(hash_columns->row_indices[i][j] > YmaxPartition[j]) {
					YmaxPartition[j] = hash_columns->row_indices[i][j];
					YPartition[j] = i;
				}
			}
			
			A_partition_column[i].nz = max;
		}		
	}

	// Broadcast the column-wise partition array
	MPI_Bcast(A_partition_column, p, mpi_pair, 0, MPI_COMM_WORLD);

	if (my_rank == 0) {
		my_columns = *A_partition;
	}
	else {
		my_columns = (pair *)malloc(sizeof(struct _pair)*A_partition_column[my_rank].col);
	}

	if (my_rank == 0) {
		for (i=1; i<p; i++) {
			MPI_Send(A_partition[i], A_partition_column[i].col, mpi_pair, i, 0, MPI_COMM_WORLD);
		}
	}
	else {
		MPI_Recv(my_columns, A_partition_column[my_rank].col, mpi_pair, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);			
	}

	MPI_Bcast(YPartition, row_num, MPI_INT, 0, MPI_COMM_WORLD);
	

	//check what recvd in mycolumns
	/*for(i=0; i<A_partition_column[my_rank].col; i++)
	{
		printf("Rank %d , Col no: %d, myNz : %d\n", my_rank, my_columns[i].col, my_columns[i].nz);
	}*/

	//partition_fp = (FILE **)malloc(sizeof(FILE*)*p);
	FILE *my_output;
	char f_name[20];


	int colIndex, myNz, rowIndex, j;
	float val;
	char *buffer;
	float *Y;
	
	Y = (float*)calloc(row_num, sizeof(float));

	
	//Read X
	FILE *fp2;
	fp2 = fopen("Xvector_algo2.txt", "r");
	int *X;
	X = (int*)malloc(sizeof(int)*A_partition_column[my_rank].col);
	
	printf("Rank %d recvd %d columns\n", my_rank, A_partition_column[my_rank].col);

	#pragma omp parallel for private(fp2, colIndex, i)
	for(i=0; i<A_partition_column[my_rank].col; i++)
	{
		colIndex = my_columns[i].col;
		fseek(fp2, colIndex*vector_block_size, SEEK_SET);
		fscanf(fp2, "%d\n", &X[i]);	
	}
  	fclose(fp2);

  	/*for(i=0; i<A_partition_column[my_rank].col; i++)
	{
		printf("Rank %d ::, X[%d] = %d\n",my_rank, i, X[i]);	
	}*/

	//for each column in A_partition_column[my_rank]...
	//Read non zeroes and multiply (computing local Y)...
	#pragma omp parallel for private(fp, colIndex, myNz, rowIndex, val)
	for(i=0; i<A_partition_column[my_rank].col; i++)
	{
		//printf("proc: %d, Operating on col %d \n", my_rank, colIndex);
		colIndex = my_columns[i].col;
		myNz = my_columns[i].nz;
		
		
		//seek to non-zeroes corresponding to this column in file
		fseek(fp, col_block_size*(col_num+1) + init_block_size*3 + rowVal_block_size*(column_ptr[colIndex]), SEEK_SET);
		
		//fread(buffer, myNz*rowVal_block_size,1,fp);
		//for each non zero...
		for(j=0; j<myNz; j++)
		{
			fscanf(fp, "%d, %f", &rowIndex, &val);
			while (fgetc(fp) != '\n');
			if(rowIndex>=row_num)
			{
				//printf("\n\n***********ERROR %d\n\n\n\n", rowIndex);
			}
			#pragma omp atomic
			Y[rowIndex]+= X[i]*val;
		}
	}
	//printf("end of loop: %d\n", my_rank);

	pairF *sendOthers[p];
	int numRowsInPartition[p], part;
	//numRowsInPartition = (int*) malloc(sizeof(int)*p);
	
	#pragma omp parallel for 
	for(i=0; i<row_num; i++)
	{
		numRowsInPartition[i] = 0;
	}

/*	for(i=0; i<row_num; i++)
	{
		printf("YPartition[%d] = %d\n", i, YPartition[i]);
	}
*/
	#pragma omp parallel for
	for(i=0; i<row_num; i++)
	{
		part = YPartition[i];
		#pragma omp atomic
		numRowsInPartition[part]++;
	}
	
	if (my_rank == 0) {
		for(i=0;i < p; i++)
		{
			printf("Rank %d got %d rows of Y vector\n", i, numRowsInPartition[i]);
		}
	}

	//make the arrays that have to be sent to other processes. 
	//pair arrays that store rowIndex and val.
	//allocate!
	for(i=0; i<p;i++)
	{
		//if(i!=my_rank)
		//{
			sendOthers[i] = (pairF*)malloc(sizeof(pairF)*numRowsInPartition[i]);
		//}
	}
	
	int *current = (int*) calloc(p, sizeof(int));
	int other, other_pos;

	//populate!
	for(i=0; i<row_num; i++)
	{
		other = YPartition[i];
		//if(other!=my_rank)
		//{
			other_pos = current[other];
			sendOthers[other][other_pos].row = i;
			sendOthers[other][other_pos].val = Y[i];
			current[other]++;
		//}
	}

	//write to respective files
	FILE *partition_fp[p];
	//open output files
	for (i=0; i< p; i++)
	{
		sprintf(f_name, "%d", i);
		//printf("open file %d\n", i);
		partition_fp[i] = fopen(f_name, "a");
	}

	//FILE *fp21 = fopen("hehe.txt", "a");
	for(i=0; i<p; i++)
	{
		if(i!=my_rank)
		{	
			other = i;
			for(j=0; j< numRowsInPartition[other]; j++)
			{
				if(sendOthers[other][j].val!=0){
					fprintf(partition_fp[other], "%d, %f, process %d\n",sendOthers[other][j].row, sendOthers[other][j].val, my_rank);
				}
			}
		}
	}

	//read from respective files and add! 
	MPI_Barrier(MPI_COMM_WORLD);
	
	for (i = 0; i < p; ++i)
	{
		fclose(partition_fp[i]);
	}
	//printf("all files closed by rank %d\n", my_rank);
	sprintf(f_name, "%d", my_rank);
	partition_fp[my_rank] = fopen(f_name, "r");
	strcat(f_name, "_output.txt");
	my_output = fopen(f_name, "w");

	while (fscanf(partition_fp[my_rank], "%d, %f", &rowIndex, &val) > 0) // expect 1 successful conversion
	{
		while (fgetc(partition_fp[my_rank]) != '\n');
		//update local y
		//printf("\n****\nRank %d read value %f\n\n", my_rank, val);
		Y[rowIndex]+=val;
	}

	for(i=0; i<numRowsInPartition[my_rank]; i++)
	{
		rowIndex = sendOthers[my_rank][i].row;
		sendOthers[my_rank][i].val = Y[rowIndex];
		//these are the final values!
		fprintf(my_output, "%d, %f, process %d\n",sendOthers[my_rank][i].row, sendOthers[my_rank][i].val, my_rank);
	}

	fclose(partition_fp[my_rank]);
	fclose(my_output);
	
	endTime = clock();

	printf("\nrank = %d, Time taken: %lf\n", my_rank, (double)(endTime - startTime)/CLOCKS_PER_SEC);
	MPI_Finalize();
	return 0;

}