struct jikukan *_read_mmap(struct silopit *silopit, size_t count) { int i; int fd; int result; int fcount; int blk_sizes; char file[FILE_PATH_SIZE]; struct jikukan *merge = NULL; struct footer footer; int fsize = sizeof(struct footer); memset(file, 0, FILE_PATH_SIZE); snprintf(file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, silopit->name); fd = open(file, O_RDWR, 0644); if (fd == -1) __PANIC("error opening silopit when read map"); result = lseek(fd, -fsize, SEEK_END); if (result == -1) __PANIC("error lseek footer"); result = read(fd, &footer, fsize); if (result != fsize) { __PANIC("error reading when read footer process"); } struct inner_block{ char key[from_be32(footer.max_len)]; char offset[8]; }; struct inner_block *blks; fcount = from_be32(footer.count); blk_sizes = from_be32(footer.size); blks= mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (blks == MAP_FAILED) { __PANIC("error map when read process"); goto out; } merge = jikukan_new(fcount + count + 1); for (i = 0; i < fcount; i++) { jikukan_insert(merge, blks[i].key, u64_from_big((unsigned char*)blks[i].offset), ADD); } if (munmap(blks, blk_sizes) == -1) __ERROR("Un-mmapping the file"); out: close(fd); return merge; }
void _silopit_load(struct silopit *silopit) { int fd, result, all_count = 0; DIR *dd; struct dirent *de; dd = opendir(silopit->basedir); while ((de = readdir(dd))) { if (strstr(de->d_name, ".silopit")) { int fcount = 0, fcrc = 0; struct meta_node mn; struct footer footer; char silopit_file[FILE_PATH_SIZE]; int fsize = sizeof(struct footer); memset(silopit_file, 0, FILE_PATH_SIZE); snprintf(silopit_file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, de->d_name); fd = open(silopit_file, O_RDWR, 0644); lseek(fd, -fsize, SEEK_END); result = read(fd, &footer, sizeof(struct footer)); if (result != sizeof(struct footer)) __PANIC("read footer error"); fcount = from_be32(footer.count); fcrc = from_be32(footer.crc); if (fcrc != F_CRC) { __PANIC("Crc wrong, silo file maybe broken, crc:<%d>,index<%s>", fcrc, silopit_file); close(fd); continue; } if (fcount == 0) { close(fd); continue; } _add_hiraishin(silopit, fd, fcount, from_be32(footer.max_len)); all_count += fcount; mn.count = fcount; memset(mn.end, 0, SILOKATANA_MAX_KEY_SIZE); memcpy(mn.end, footer.key, SILOKATANA_MAX_KEY_SIZE); memset(mn.index_name, 0, FILE_NAME_SIZE); memcpy(mn.index_name, de->d_name, FILE_NAME_SIZE); meta_set(silopit->meta, &mn); close(fd); } } closedir(dd); __DEBUG("Load silopit,all entries count:<%d>", all_count); }
void _sst_load(struct sst *sst) { int fd, result; int all_count = 0; DIR *dd; struct dirent *de;; dd = opendir(sst->basedir); while ((de = readdir(dd))) { if(strstr(de->d_name, ".sst")) { int fcount = 0, fcrc = 0; struct meta_node mn; struct footer footer; char sst_file[FILE_PATH_SIZE]; int fsize = sizeof(struct footer); memset(sst_file, 0, FILE_PATH_SIZE); snprintf(sst_file, FILE_PATH_SIZE, "%s/%s", sst->basedir, de->d_name); fd = open(sst_file, O_RDWR, 0644); lseek(fd, -fsize, SEEK_END); result = read(fd, &footer, sizeof(struct footer)); if(result == -1) abort(); fcount = from_be32(footer.count); fcrc = from_be32(footer.crc); if(fcrc != F_CRC) { __DEBUG(LEVEL_ERROR, "Error: crc wrong, crc:<%d>, index<%s>", fcrc, sst_file); close(fd); continue; } if(fcount == 0) { close(fd); continue; } all_count += fcount; mn.count = fcount; memset(mn.end, 0, NESSDB_MAX_KEY_SIZE); memcpy(mn.end, footer.key, NESSDB_MAX_KEY_SIZE); memset(mn.index_name, 0, FILE_NAME_SIZE); memcpy(mn.index_name, de->d_name, FILE_NAME_SIZE); meta_set(sst->meta, &mn); close(fd); } } closedir(dd); __DEBUG(LEVEL_DEBUG, "load sst, all entries count:<%d>", all_count); }
static int blockdev_malloc_unmap(struct malloc_disk *mdisk, struct spdk_io_channel *ch, struct copy_task *copy_req, struct spdk_scsi_unmap_bdesc *unmap_d, uint16_t bdesc_count) { uint64_t lba, offset, byte_count; uint32_t block_count; assert(bdesc_count <= MALLOC_MAX_UNMAP_BDESC); /* * For now, only support a single unmap descriptor per command. The copy engine API does not * support batch submission of operations. */ assert(bdesc_count == 1); lba = from_be64(&unmap_d[0].lba); offset = lba * mdisk->disk.blocklen; block_count = from_be32(&unmap_d[0].block_count); byte_count = (uint64_t)block_count * mdisk->disk.blocklen; if (lba >= mdisk->disk.blockcnt || block_count > mdisk->disk.blockcnt - lba) { return -1; } return spdk_copy_submit_fill(copy_req, ch, mdisk->malloc_buf + offset, 0, byte_count, malloc_done); }
uint64_t _read_offset(struct sst *sst, struct slice *sk) { int fd; int fcount; int blk_sizes; int result; uint64_t off = 0UL; char file[FILE_PATH_SIZE]; struct sst_block *blks; struct footer footer; int fsize = sizeof(struct footer); memset(file, 0, FILE_PATH_SIZE); snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name); fd = open(file, O_RDWR, 0644); result = lseek(fd, -fsize, SEEK_END); if(result == -1) { abort(); } result = read(fd, &footer, fsize); if(result == -1) { abort(); } fcount = from_be32(footer.count); blk_sizes = fcount*sizeof(struct sst_block); blks = mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(blks == MAP_FAILED) { __DEBUG(LEVEL_ERROR, "%s", "Error:read_offset, mmapping the file"); goto out; } size_t left = 0, right = fcount, i = 0; while(left < right) { i = (right - left) / 2 + left; int cmp = strcmp(sk->data, blks[i].key); if(cmp == 0) { off = from_be64(blks[i].offset); break; } if(cmp < 0) right = i; else left = i + 1; } if(munmap(blks, blk_sizes) == -1) __DEBUG(LEVEL_ERROR, "%s", "read_offset:un-mmapping the file"); out : close(fd); return off; }
struct skiplist *_read_mmap(struct sst *sst, size_t count) { int i; int fd; int result; int fcount; int blk_sizes; char file[FILE_PATH_SIZE]; struct sst_block *blks; struct skiplist *merge = NULL; struct footer footer; int fsize = sizeof(struct footer); memset(file, 0, FILE_PATH_SIZE); snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name); fd = open(file, O_RDWR, 0644); result = lseek(fd, -fsize, SEEK_END); if(result == -1) abort(); result = read(fd, &footer, fsize); if(result == -1) abort(); fcount = from_be32(footer.count); blk_sizes = fcount * sizeof(struct sst_block); blks = mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(blks == MAP_FAILED) { __DEBUG(LEVEL_ERROR, "%s", "Error: sst_bloom un-mmapping the file"); goto out; } merge = skiplist_new(fcount + count + 1); for(i = 0; i < fcount; i++) { skiplist_insert(merge, blks[i].key, from_be64(blks[i].offset), ADD); } if(munmap(blks, blk_sizes) == -1) { __DEBUG(LEVEL_ERROR, "%s", "read_map:un-mmapping the file"); } out: close(fd); return merge; }
int index_get(struct index *idx, struct slice *sk, struct slice *sv) { int ret = 0, value_len, result; uint64_t value_off = 0UL; struct skipnode *node; struct skiplist *cur_list; struct skiplist *merge_list; /* * 0) Get from bloomfilter, if bloom_get return 1, next * 1) First lookup from active memtable * 2) Then from merge memtable * 3) Last from sst on-disk indexes */ cur_list = idx->list; node = skiplist_lookup(cur_list, sk->data); if(node) { if(node->opt == DEL) { ret = -1; goto out_get; } value_off =node->val; } else { merge_list = idx->park->list; if(merge_list) { node = skiplist_lookup(merge_list, sk->data); if(node && node->opt == ADD) value_off = node->val; } } if(value_off == 0UL) value_off = sst_getoff(idx->sst, sk); if(value_off != 0UL) { __be32 be32len; lseek(idx->db_rfd, value_off, SEEK_SET); result = read(idx->db_rfd, &be32len, sizeof(int)); if(FILE_ERR(result)) { ret = -1; goto out_get; } value_len = from_be32(be32len); if(result == sizeof(int)) { char *data = malloc(value_len + 1); memset(data, 0, value_len+1); result = read(idx->db_rfd, data, value_len); if(FILE_ERR(result)) { free(data); ret = -1; goto out_get; } sv->len = value_len; sv->data = data; return 1; } } else { return 0; } out_get: return ret; }
uint64_t _read_offset(struct silopit *silopit, struct slice *sk) { int fd; int fcount; int blk_sizes; int result; uint64_t off = 0UL; char file[FILE_PATH_SIZE]; struct footer footer; int fsize = sizeof(struct footer); memset(file, 0, FILE_PATH_SIZE); snprintf(file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, silopit->name); fd = open(file, O_RDWR, 0644); if (fd == -1) { __ERROR("error opening silopit when read offset process"); return 0UL; } result = lseek(fd, -fsize, SEEK_END); if (result == -1) { __ERROR("error lseek when read offset process"); close(fd); return off; } result = read(fd, &footer, fsize); if (result == -1) { __ERROR("error reading footer when read offset process"); close(fd); return off; } int max_len = from_be32(footer.max_len); struct inner_block { char key[max_len]; char offset[8]; }; struct inner_block *blks; fcount = from_be32(footer.count); blk_sizes = from_be32(footer.size); blks= mmap(0, blk_sizes, PROT_READ, MAP_PRIVATE, fd, 0); if (blks == MAP_FAILED) { __ERROR("Map_failed when read process"); close(fd); return off; } size_t left = 0, right = fcount, i = 0; while (left < right) { i = (right -left) / 2 +left; int cmp = strcmp(sk->data, blks[i].key); if (cmp == 0) { off = u64_from_big((unsigned char*)blks[i].offset); break ; } if (cmp < 0) right = i; else left = i + 1; } if (munmap(blks, blk_sizes) == -1) __ERROR("un-mmapping the file"); close(fd); return off; }