Exemple #1
0
struct jikukan *_read_mmap(struct silopit *silopit, size_t count)
{
	int i;
	int fd;
	int result;
	int fcount;
	int blk_sizes;
	char file[FILE_PATH_SIZE];
	struct jikukan *merge = NULL;
	struct footer footer;
	int fsize = sizeof(struct footer);

	memset(file, 0, FILE_PATH_SIZE);
	snprintf(file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, silopit->name);

	fd = open(file, O_RDWR, 0644);
	if (fd == -1)
		__PANIC("error opening silopit when read map");

	result = lseek(fd, -fsize, SEEK_END);
	if (result == -1)
		__PANIC("error lseek footer");

	result = read(fd, &footer, fsize);
	if (result != fsize) {
		__PANIC("error reading when read footer process");
	}

	struct inner_block{
		char key[from_be32(footer.max_len)];
		char offset[8];
	};

	struct inner_block *blks;

	fcount = from_be32(footer.count);
	blk_sizes = from_be32(footer.size);

	blks= mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
	if (blks == MAP_FAILED) {
		__PANIC("error map when read process");
		goto out;
	}

	merge = jikukan_new(fcount + count + 1);
	for (i = 0; i < fcount; i++) {
		jikukan_insert(merge, blks[i].key, u64_from_big((unsigned char*)blks[i].offset), ADD);
	}
	
	if (munmap(blks, blk_sizes) == -1)
		__ERROR("Un-mmapping the file");

out:
	close(fd);

	return merge;
}
Exemple #2
0
void _silopit_load(struct silopit *silopit)
{
	int fd, result, all_count = 0;
	DIR *dd;
	struct dirent *de;

	dd = opendir(silopit->basedir);
	while ((de = readdir(dd))) {
		if (strstr(de->d_name, ".silopit")) {
			int fcount = 0, fcrc = 0;
			struct meta_node mn;
			struct footer footer;
			char silopit_file[FILE_PATH_SIZE];
			int fsize = sizeof(struct footer);

			memset(silopit_file, 0, FILE_PATH_SIZE);
			snprintf(silopit_file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, de->d_name);
			
			fd = open(silopit_file, O_RDWR, 0644);
			lseek(fd, -fsize, SEEK_END);
			result = read(fd, &footer, sizeof(struct footer));
			if (result != sizeof(struct footer))
				__PANIC("read footer error");

			fcount = from_be32(footer.count);
			fcrc = from_be32(footer.crc);

			if (fcrc != F_CRC) {
				__PANIC("Crc wrong, silo file maybe broken, crc:<%d>,index<%s>", fcrc, silopit_file);
				close(fd);
				continue;
			}

			if (fcount == 0) {
				close(fd);
				continue;
			}

			_add_hiraishin(silopit, fd, fcount, from_be32(footer.max_len));

			all_count += fcount;
						
			mn.count = fcount;
			memset(mn.end, 0, SILOKATANA_MAX_KEY_SIZE);
			memcpy(mn.end, footer.key, SILOKATANA_MAX_KEY_SIZE);

			memset(mn.index_name, 0, FILE_NAME_SIZE);
			memcpy(mn.index_name, de->d_name, FILE_NAME_SIZE);
			meta_set(silopit->meta, &mn);
		
			close(fd);
		}
	}

	closedir(dd);
	__DEBUG("Load silopit,all entries count:<%d>", all_count);
}
Exemple #3
0
void _sst_load(struct sst *sst)
{
	int fd, result;
	int all_count = 0;
	DIR *dd;
	struct dirent *de;;

	dd = opendir(sst->basedir);
	while ((de = readdir(dd))) {
		if(strstr(de->d_name, ".sst")) {
			int fcount = 0, fcrc = 0;
			struct meta_node mn;
			struct footer footer;
			char sst_file[FILE_PATH_SIZE];
			int fsize = sizeof(struct footer);

			memset(sst_file, 0, FILE_PATH_SIZE);
			snprintf(sst_file, FILE_PATH_SIZE, "%s/%s", sst->basedir, de->d_name);

			fd = open(sst_file, O_RDWR, 0644);
			lseek(fd, -fsize, SEEK_END);
			result = read(fd, &footer, sizeof(struct footer));
			if(result == -1)
				abort();

			fcount = from_be32(footer.count);
			fcrc = from_be32(footer.crc);
			if(fcrc != F_CRC) {
				__DEBUG(LEVEL_ERROR, "Error: crc wrong, crc:<%d>, index<%s>", fcrc, sst_file);
				close(fd);
				continue;
			}

			if(fcount == 0) {
				close(fd);
				continue;
			}

			all_count += fcount;

			mn.count = fcount;
			memset(mn.end, 0, NESSDB_MAX_KEY_SIZE);
			memcpy(mn.end, footer.key, NESSDB_MAX_KEY_SIZE);

			memset(mn.index_name, 0, FILE_NAME_SIZE);
			memcpy(mn.index_name, de->d_name, FILE_NAME_SIZE);
			meta_set(sst->meta, &mn);

			close(fd);
		}
	}
	closedir(dd);
	__DEBUG(LEVEL_DEBUG, "load sst, all entries count:<%d>", all_count);
}
Exemple #4
0
static int
blockdev_malloc_unmap(struct malloc_disk *mdisk,
		      struct spdk_io_channel *ch,
		      struct copy_task *copy_req,
		      struct spdk_scsi_unmap_bdesc *unmap_d,
		      uint16_t bdesc_count)
{
	uint64_t lba, offset, byte_count;
	uint32_t block_count;

	assert(bdesc_count <= MALLOC_MAX_UNMAP_BDESC);

	/*
	 * For now, only support a single unmap descriptor per command. The copy engine API does not
	 * support batch submission of operations.
	 */
	assert(bdesc_count == 1);

	lba = from_be64(&unmap_d[0].lba);
	offset = lba * mdisk->disk.blocklen;
	block_count = from_be32(&unmap_d[0].block_count);
	byte_count = (uint64_t)block_count * mdisk->disk.blocklen;

	if (lba >= mdisk->disk.blockcnt || block_count > mdisk->disk.blockcnt - lba) {
		return -1;
	}

	return spdk_copy_submit_fill(copy_req, ch, mdisk->malloc_buf + offset, 0, byte_count, malloc_done);
}
Exemple #5
0
uint64_t _read_offset(struct sst *sst, struct slice *sk)
{
	int fd;
	int fcount;
	int blk_sizes;
	int result;
	uint64_t off = 0UL;
	char file[FILE_PATH_SIZE];
	struct sst_block *blks;
	struct footer footer;
	int fsize = sizeof(struct footer);

	memset(file, 0, FILE_PATH_SIZE);
	snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name);

	fd = open(file, O_RDWR, 0644);
	result = lseek(fd, -fsize, SEEK_END);
	if(result == -1) {
		abort();
	}

	result = read(fd, &footer, fsize);
	if(result == -1) {
		abort();
	}

	fcount = from_be32(footer.count);
	blk_sizes = fcount*sizeof(struct sst_block);

	blks = mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
	if(blks == MAP_FAILED) {
		__DEBUG(LEVEL_ERROR, "%s", "Error:read_offset, mmapping the file");
		goto out;
	}

	size_t left = 0, right = fcount, i = 0;
	while(left < right) {
		i = (right - left) / 2 + left;
		int cmp = strcmp(sk->data, blks[i].key);
		if(cmp == 0) {
			off = from_be64(blks[i].offset);
			break;
		}

		if(cmp < 0) 
			right = i;
		else 
			left = i + 1;
	}
	if(munmap(blks, blk_sizes) == -1) 
		__DEBUG(LEVEL_ERROR, "%s", "read_offset:un-mmapping the file");

out :
	close(fd);
	return off;
}
Exemple #6
0
struct skiplist *_read_mmap(struct sst *sst, size_t count)
{
	int i;
	int fd;
	int result;
	int fcount;
	int blk_sizes;
	char file[FILE_PATH_SIZE];
	struct sst_block *blks;
	struct skiplist *merge = NULL;
	struct footer footer;
	int fsize = sizeof(struct footer);

	memset(file, 0, FILE_PATH_SIZE);
	snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name);

	fd = open(file, O_RDWR, 0644);
	result = lseek(fd, -fsize, SEEK_END);
	if(result == -1)
		abort();

	result = read(fd, &footer, fsize);
	if(result == -1)
		abort();

	fcount = from_be32(footer.count);
	blk_sizes = fcount * sizeof(struct sst_block);

	blks = mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
	if(blks == MAP_FAILED) {
		__DEBUG(LEVEL_ERROR, "%s", "Error: sst_bloom un-mmapping the file");
		goto out;
	}

	merge = skiplist_new(fcount + count + 1);
	for(i = 0; i < fcount; i++) {
		skiplist_insert(merge, blks[i].key, from_be64(blks[i].offset), ADD);
	}

	if(munmap(blks, blk_sizes) == -1) {
		__DEBUG(LEVEL_ERROR, "%s", "read_map:un-mmapping the file");
	}

out:
	close(fd);
	return merge;
}
Exemple #7
0
int index_get(struct index *idx, struct slice *sk, struct slice *sv)
{
	int ret = 0, value_len, result;
	uint64_t value_off = 0UL;

	struct skipnode *node;
	struct skiplist *cur_list;
	struct skiplist *merge_list;

	/* 
	 * 0) Get from bloomfilter, if bloom_get return 1, next 
	 * 1) First lookup from active memtable 
	 * 2) Then from merge memtable 
	 * 3) Last from sst on-disk indexes 
	 */

	cur_list = idx->list;
	node = skiplist_lookup(cur_list, sk->data);

	if(node) {
		if(node->opt == DEL) {
			ret = -1;
			goto out_get;
		}
		value_off =node->val;
	} else {
		merge_list = idx->park->list;
		if(merge_list) {
			node = skiplist_lookup(merge_list, sk->data);
			if(node && node->opt == ADD)
				value_off = node->val;
		}
	}
	if(value_off == 0UL)
		value_off = sst_getoff(idx->sst, sk);

	if(value_off != 0UL) {
		__be32 be32len;
		lseek(idx->db_rfd, value_off, SEEK_SET);
		result = read(idx->db_rfd, &be32len, sizeof(int));
		if(FILE_ERR(result)) {
			ret = -1;
			goto out_get;
		}

		value_len = from_be32(be32len);
		if(result == sizeof(int)) {
			char *data = malloc(value_len + 1);
			memset(data, 0, value_len+1);
			result = read(idx->db_rfd, data, value_len);
			if(FILE_ERR(result)) {
				free(data);
				ret = -1;
				goto out_get;
			}
			sv->len = value_len;
			sv->data = data;
			return 1;
		}
	} else {
		return 0;
	}
out_get:
	return ret;
}
Exemple #8
0
uint64_t _read_offset(struct silopit *silopit, struct slice *sk)
{
	int fd;
	int fcount;
	int blk_sizes;
	int result;
	uint64_t off = 0UL;
	char file[FILE_PATH_SIZE];
	struct footer footer;
	int fsize = sizeof(struct footer);

	memset(file, 0, FILE_PATH_SIZE);
	snprintf(file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, silopit->name);

	fd = open(file, O_RDWR, 0644);
	if (fd == -1) {
		__ERROR("error opening silopit when read offset process");
		return 0UL;
	}
	
	result = lseek(fd, -fsize, SEEK_END);
	if (result == -1) {
		__ERROR("error lseek when read offset process");
		close(fd);
		return off;
	}

	result = read(fd, &footer, fsize);
	if (result == -1) {
		__ERROR("error reading footer when read offset process");
		close(fd);
		return off;
	}

	int max_len = from_be32(footer.max_len);

	struct inner_block {
		char key[max_len];
		char offset[8];
	};

	struct inner_block *blks;


	fcount = from_be32(footer.count);
	blk_sizes = from_be32(footer.size);

	blks= mmap(0, blk_sizes, PROT_READ, MAP_PRIVATE, fd, 0);
	if (blks == MAP_FAILED) {
		__ERROR("Map_failed when read process");
		close(fd);
		return off;
	}

	size_t left = 0, right = fcount, i = 0;
	while (left < right) {
		i = (right -left) / 2 +left;
		int cmp = strcmp(sk->data, blks[i].key);
		if (cmp == 0) {
			off = u64_from_big((unsigned char*)blks[i].offset);	
			break ;
		}

		if (cmp < 0)
			right = i;
		else
			left = i + 1;
	}
	
	if (munmap(blks, blk_sizes) == -1)
		__ERROR("un-mmapping the file");

	close(fd);
	return off;
}